diff --git a/lib/functions/cli/cli-jsoninfo.sh b/lib/functions/cli/cli-jsoninfo.sh index 4ffde1bf56..9e9f3bda30 100644 --- a/lib/functions/cli/cli-jsoninfo.sh +++ b/lib/functions/cli/cli-jsoninfo.sh @@ -106,6 +106,7 @@ function cli_json_info_run() { ### --- inventory --- ### + declare ALL_USERSPACE_INVENTORY_FILE="${BASE_INFO_OUTPUT_DIR}/all_userspace_inventory.json" declare ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE="${BASE_INFO_OUTPUT_DIR}/all_boards_all_branches.json" declare TARGETS_OUTPUT_FILE="${BASE_INFO_OUTPUT_DIR}/all-targets.json" declare IMAGE_INFO_FILE="${BASE_INFO_OUTPUT_DIR}/image-info.json" @@ -115,19 +116,18 @@ function cli_json_info_run() { declare ARTIFACTS_INFO_UPTODATE_FILE="${BASE_INFO_OUTPUT_DIR}/artifacts-info-uptodate.json" declare OUTDATED_ARTIFACTS_IMAGES_FILE="${BASE_INFO_OUTPUT_DIR}/outdated-artifacts-images.json" + # Userspace inventory: RELEASES, and DESKTOPS and their possible ARCH'es, names, and support status. + if [[ ! -f "${ALL_USERSPACE_INVENTORY_FILE}" ]]; then + display_alert "Generating userspace inventory" "all_userspace_inventory.json" "info" + run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/userspace-inventory.py ">" "${ALL_USERSPACE_INVENTORY_FILE}" + fi + # Board/branch inventory. if [[ ! -f "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" ]]; then display_alert "Generating board/branch inventory" "all_boards_all_branches.json" "info" run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/board-inventory.py ">" "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" fi - # @TODO: Release/rootfs inventory? - - # A simplistic all-boards-all-branches target file, for the all-boards-all-branches-targets.json. - # Then just use the same info-gatherer-image to get the image info. - # This will be used as database for the targets-compositor, for example to get "all boards+branches that have kernel < 5.0" or "all boards+branches of meson64 family" etc. - # @TODO: this is a bit heavy; only do it if out-of-date (compared to config/, lib/, extensions/, userpatches/ file mtimes...) - if [[ "${ARMBIAN_COMMAND}" == "inventory" ]]; then display_alert "Done with" "inventory" "info" return 0 @@ -147,12 +147,17 @@ function cli_json_info_run() { export TARGETS_BETA="${BETA}" # Read by the Python script, and injected into every target as "BETA=" param. export TARGETS_REVISION="${REVISION}" # Read by the Python script, and injected into every target as "REVISION=" param. export TARGETS_FILTER_INCLUDE="${TARGETS_FILTER_INCLUDE}" # Read by the Python script; used to "only include" targets that match the given string. - run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/targets-compositor.py "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" "not_yet_releases.json" "${TARGETS_FILE}" ">" "${TARGETS_OUTPUT_FILE}" + run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/targets-compositor.py "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" "${ALL_USERSPACE_INVENTORY_FILE}" "${TARGETS_FILE}" ">" "${TARGETS_OUTPUT_FILE}" unset TARGETS_BETA unset TARGETS_REVISION unset TARGETS_FILTER_INCLUDE fi + if [[ "${ARMBIAN_COMMAND}" == "targets-composed" ]]; then + display_alert "Done with" "targets-dashboard" "info" + return 0 + fi + ### Images. # The image info extractor. diff --git a/lib/functions/cli/commands.sh b/lib/functions/cli/commands.sh index ab9939d292..08ee4866cf 100644 --- a/lib/functions/cli/commands.sh +++ b/lib/functions/cli/commands.sh @@ -28,6 +28,7 @@ function armbian_register_commands() { ["inventory"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["targets"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["targets-dashboard"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run + ["targets-composed"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["debs-to-repo-json"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["gha-matrix"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run ["gha-workflow"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run diff --git a/lib/tools/common/armbian_utils.py b/lib/tools/common/armbian_utils.py index 4b7b0b40bc..cc7d2d3106 100755 --- a/lib/tools/common/armbian_utils.py +++ b/lib/tools/common/armbian_utils.py @@ -202,12 +202,100 @@ def find_armbian_src_path(): if not os.path.exists(core_boards_path): raise Exception("Can't find config/boards") + # userspace stuff + core_distributions_path = os.path.realpath(os.path.join(armbian_src_path, "config", "distributions")) + log.debug(f"Real path to core distributions '{core_distributions_path}'") + # Make sure it exists + if not os.path.exists(core_distributions_path): + raise Exception("Can't find config/distributions") + + core_desktop_path = os.path.realpath(os.path.join(armbian_src_path, "config", "desktop")) + log.debug(f"Real path to core desktop '{core_desktop_path}'") + # Make sure it exists + if not os.path.exists(core_desktop_path): + raise Exception("Can't find config/desktop") + userpatches_boards_path = os.path.realpath(os.path.join(armbian_src_path, "userpatches", "config", "boards")) log.debug(f"Real path to userpatches boards '{userpatches_boards_path}'") has_userpatches_path = os.path.exists(userpatches_boards_path) - return {"armbian_src_path": armbian_src_path, "compile_sh_full_path": compile_sh_full_path, "core_boards_path": core_boards_path, - "userpatches_boards_path": userpatches_boards_path, "has_userpatches_path": has_userpatches_path} + return { + "armbian_src_path": armbian_src_path, "compile_sh_full_path": compile_sh_full_path, "core_boards_path": core_boards_path, + "core_distributions_path": core_distributions_path, "core_desktop_path": core_desktop_path, + "userpatches_boards_path": userpatches_boards_path, "has_userpatches_path": has_userpatches_path + } + + +def read_one_distro_config_file(filename): + # Read the contents of filename passed in and return it as string, trimmed + with open(filename, 'r') as file_handle: + file_contents = file_handle.read() + return file_contents.strip() + + +def split_commas_and_clean_into_list(string): + ret = [] + for item in string.split(","): + item = item.strip() + if item != "": + ret.append(item) + return ret + + +def get_desktop_inventory_for_distro(distro, armbian_paths): + ret = [] + desktops_path = armbian_paths["core_desktop_path"] + envs_path_for_distro = os.path.join(desktops_path, distro, "environments") + if not os.path.exists(envs_path_for_distro): + log.warning(f"Can't find desktop environments for distro '{distro}' at '{envs_path_for_distro}'") + return ret + for env in os.listdir(envs_path_for_distro): + one_env_path = os.path.join(envs_path_for_distro, env) + if not os.path.isdir(one_env_path): + continue + log.debug(f"Processing desktop '{env}' for distro '{distro}'") + support_file_path = os.path.join(one_env_path, "support") + arches_file_path = os.path.join(one_env_path, "architectures") + if not os.path.exists(support_file_path): + log.warning(f"Can't find desktop support file for distro '{distro}' and environment '{env}' at '{support_file_path}'") + continue + if not os.path.exists(arches_file_path): + log.warning(f"Can't find desktop arches file for distro '{distro}' and environment '{env}' at '{arches_file_path}'") + continue + + env_main_info = { + "id": env, + "support": read_one_distro_config_file(support_file_path), + "arches": split_commas_and_clean_into_list(read_one_distro_config_file(arches_file_path)) + } + ret.append(env_main_info) + + return ret + + +def armbian_get_all_userspace_inventory(): + armbian_paths = find_armbian_src_path() + distros_path = armbian_paths["core_distributions_path"] + all_distros = [] + # find and loop over every directory in distros_path, including symlinks + for distro in os.listdir(distros_path): + one_distro_path = os.path.join(distros_path, distro) + if not os.path.isdir(one_distro_path): + continue + log.debug(f"Processing distro '{distro}'") + support_file_path = os.path.join(one_distro_path, "support") + arches_file_path = os.path.join(one_distro_path, "architectures") + name_file_path = os.path.join(one_distro_path, "name") + distro_main_info = { + "id": distro, + "name": read_one_distro_config_file(name_file_path), + "support": read_one_distro_config_file(support_file_path), + "arches": split_commas_and_clean_into_list(read_one_distro_config_file(arches_file_path)), + "desktops": get_desktop_inventory_for_distro(distro, armbian_paths) + } + all_distros.append(distro_main_info) + + return all_distros def armbian_get_all_boards_inventory(): diff --git a/lib/tools/info/targets-compositor.py b/lib/tools/info/targets-compositor.py index 6a4d934218..9faf84ebe2 100644 --- a/lib/tools/info/targets-compositor.py +++ b/lib/tools/info/targets-compositor.py @@ -52,6 +52,10 @@ for board in board_inventory: if board_inventory[board]["BOARD_HAS_VIDEO"]: not_eos_with_video_boards_all_branches.append(data_from_inventory) +userspace_inventory_file = sys.argv[2] +with open(userspace_inventory_file, 'r') as f: + userspace_inventory = json.load(f) + # get the third argv, which is the targets.yaml file. targets_yaml_file = sys.argv[3] # read it as yaml, modern way @@ -61,6 +65,103 @@ with open(targets_yaml_file, 'r') as f: # Keep a running of all the invocations we want to make. invocations_dict: list[dict] = [] + +# userspace inventory is a bit more complex, here's a function +def get_userspace_inventory(opts: dict): + ret = [] + log.info("Processing userspace inventory...") + log.debug(f"Processing userspace inventory options: {opts}") + + # set default opts if not present + if opts is None: + opts = {} + if "arches" not in opts: + opts["arches"] = {"arm64": [{"BOARD": "uefi-arm64", "BRANCH": "current"}]} # default is arm64 only + if "minimal" not in opts: + opts["minimal"] = False + if "cli" not in opts: + opts["cli"] = True # default on, only for CLI + if "cloud" not in opts: + opts["cloud"] = False + if "desktops" not in opts: + opts["desktops"] = False + if "desktop_variations" not in opts: + opts["desktop_variations"] = [[]] + + # loop over the userspace inventory + for userspace in userspace_inventory: + if userspace["support"] == "eos": + log.debug(f"Skipping userspace inventory entry: '{userspace['id']}' has support '{userspace['support']}'") + continue + + if "skip-releases" in opts and userspace["id"] in opts["skip-releases"]: + log.info(f"Skipping userspace inventory entry: '{userspace['id']}' is in skip-releases list.") + continue + + if "only-releases" in opts and userspace["id"] not in opts["only-releases"]: + log.info(f"Skipping userspace inventory entry: '{userspace['id']}' is not in only-releases list.") + continue + + log.info(f"Processing userspace inventory for distro: {userspace['id']}") + + # loop over the wanted wanted_arch'es + for wanted_arch in opts["arches"]: + wanted_bbs_for_arch = opts["arches"][wanted_arch] + log.debug(f"Processing wanted userspace inventory wanted_arch: '{wanted_arch}' - '{wanted_bbs_for_arch}'") + # if the wanted_arch is not in the userspace, skip it completely. + if wanted_arch not in userspace["arches"]: + log.debug(f"Skipping userspace inventory entry: '{userspace['id']}' does not support wanted_arch '{wanted_arch}'") + continue + + if opts["cli"]: + for bb in wanted_bbs_for_arch: + ret.append({**bb, **{"RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "no", "BUILD_DESKTOP": "no"}}) + + if opts["minimal"]: + for bb in wanted_bbs_for_arch: + ret.append({**bb, **{"RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "yes", "BUILD_DESKTOP": "no"}}) + + if opts["cloud"]: # rpardini's cloud images. + for bb in wanted_bbs_for_arch: + ret.append({**bb, **{ + "RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "no", "BUILD_DESKTOP": "no", "CLOUD_IMAGE": "yes" + }}) + + if opts["desktops"]: + # loop over the desktops in userspace; skip any that are eos, or that don't have the wanted arch + for desktop in userspace["desktops"]: + if desktop["support"] == "eos": + log.warning( + f"Skipping userspace inventory desktop: '{desktop['id']}' has support '{desktop['support']} for userspace '{userspace['id']}'") + continue + + if "skip-desktops" in opts and desktop["id"] in opts["skip-desktops"]: + log.info(f"Skipping userspace inventory desktop: '{desktop['id']}' is in skip-desktops list.") + continue + + if "only-desktops" in opts and desktop["id"] not in opts["only-desktops"]: + log.info(f"Skipping userspace inventory desktop: '{desktop['id']}' is not in only-desktops list.") + continue + + if wanted_arch not in desktop["arches"]: + log.debug( + f"Skipping userspace inventory desktop: '{desktop['id']}' does not support wanted_arch '{wanted_arch}' for userspace '{userspace['id']}'") + continue + + # loop over the variants... desktop_variations is a list of lists + for variant in opts["desktop_variations"]: + appgroups_comma = ",".join(variant) + + for bb in wanted_bbs_for_arch: + ret.append({**bb, **{ + "RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "no", "BUILD_DESKTOP": "yes", + "DESKTOP_ENVIRONMENT_CONFIG_NAME": "config_base", # yeah, config_base is hardcoded. + "DESKTOP_APPGROUPS_SELECTED": appgroups_comma, # hopefully empty works + "DESKTOP_ENVIRONMENT": desktop["id"]}}) + + return ret + + # Loop over targets for target_name in targets["targets"]: target_obj = targets["targets"][target_name] @@ -97,10 +198,12 @@ for target_name in targets["targets"]: # Now add to all_items by resolving the "items-from-inventory" key if "items-from-inventory" in target_obj: - # loop over the keys + # loop over the keys, for regular board vs branches inventory for key in target_obj["items-from-inventory"]: to_add = [] - if key == "all": + if key == "userspace": + to_add.extend(get_userspace_inventory(target_obj["items-from-inventory"][key])) + elif key == "all": to_add.extend(all_boards_all_branches) elif key == "not-eos": to_add.extend(not_eos_boards_all_branches) diff --git a/lib/tools/info/userspace-inventory.py b/lib/tools/info/userspace-inventory.py new file mode 100644 index 0000000000..71e2f9ccc3 --- /dev/null +++ b/lib/tools/info/userspace-inventory.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2023 Ricardo Pardini +# This file is a part of the Armbian Build Framework https://github.com/armbian/build/ +# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ +import json +import logging +import os + +import sys + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from common import armbian_utils + +# Prepare logging +armbian_utils.setup_logging() +log: logging.Logger = logging.getLogger("userspace-inventory") + +all = armbian_utils.armbian_get_all_userspace_inventory() +log.info(f"Inventoried {len(all)} userspace combinations.") +print(json.dumps(all, indent=4, sort_keys=True))