pipeline: add userspace inventory capabilities
- digs into config/distributions and config/desktops for info - this produces `output/info/all_userspace_inventory.json` - this is now passed down to the `targets-compositor` in `cli-jsoninfo` - `targets-compositor` now accepts `userspace:` as `items-from-inventory` - extra: add `targets-composed` CLI command, to stop after targets-compositor
This commit is contained in:
Ricardo Pardini
Igor
parent
0d22ef349d
commit
688e415832
@ -106,6 +106,7 @@ function cli_json_info_run() {
|
||||
|
||||
### --- inventory --- ###
|
||||
|
||||
declare ALL_USERSPACE_INVENTORY_FILE="${BASE_INFO_OUTPUT_DIR}/all_userspace_inventory.json"
|
||||
declare ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE="${BASE_INFO_OUTPUT_DIR}/all_boards_all_branches.json"
|
||||
declare TARGETS_OUTPUT_FILE="${BASE_INFO_OUTPUT_DIR}/all-targets.json"
|
||||
declare IMAGE_INFO_FILE="${BASE_INFO_OUTPUT_DIR}/image-info.json"
|
||||
@ -115,19 +116,18 @@ function cli_json_info_run() {
|
||||
declare ARTIFACTS_INFO_UPTODATE_FILE="${BASE_INFO_OUTPUT_DIR}/artifacts-info-uptodate.json"
|
||||
declare OUTDATED_ARTIFACTS_IMAGES_FILE="${BASE_INFO_OUTPUT_DIR}/outdated-artifacts-images.json"
|
||||
|
||||
# Userspace inventory: RELEASES, and DESKTOPS and their possible ARCH'es, names, and support status.
|
||||
if [[ ! -f "${ALL_USERSPACE_INVENTORY_FILE}" ]]; then
|
||||
display_alert "Generating userspace inventory" "all_userspace_inventory.json" "info"
|
||||
run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/userspace-inventory.py ">" "${ALL_USERSPACE_INVENTORY_FILE}"
|
||||
fi
|
||||
|
||||
# Board/branch inventory.
|
||||
if [[ ! -f "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" ]]; then
|
||||
display_alert "Generating board/branch inventory" "all_boards_all_branches.json" "info"
|
||||
run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/board-inventory.py ">" "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}"
|
||||
fi
|
||||
|
||||
# @TODO: Release/rootfs inventory?
|
||||
|
||||
# A simplistic all-boards-all-branches target file, for the all-boards-all-branches-targets.json.
|
||||
# Then just use the same info-gatherer-image to get the image info.
|
||||
# This will be used as database for the targets-compositor, for example to get "all boards+branches that have kernel < 5.0" or "all boards+branches of meson64 family" etc.
|
||||
# @TODO: this is a bit heavy; only do it if out-of-date (compared to config/, lib/, extensions/, userpatches/ file mtimes...)
|
||||
|
||||
if [[ "${ARMBIAN_COMMAND}" == "inventory" ]]; then
|
||||
display_alert "Done with" "inventory" "info"
|
||||
return 0
|
||||
@ -147,12 +147,17 @@ function cli_json_info_run() {
|
||||
export TARGETS_BETA="${BETA}" # Read by the Python script, and injected into every target as "BETA=" param.
|
||||
export TARGETS_REVISION="${REVISION}" # Read by the Python script, and injected into every target as "REVISION=" param.
|
||||
export TARGETS_FILTER_INCLUDE="${TARGETS_FILTER_INCLUDE}" # Read by the Python script; used to "only include" targets that match the given string.
|
||||
run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/targets-compositor.py "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" "not_yet_releases.json" "${TARGETS_FILE}" ">" "${TARGETS_OUTPUT_FILE}"
|
||||
run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${INFO_TOOLS_DIR}"/targets-compositor.py "${ALL_BOARDS_ALL_BRANCHES_INVENTORY_FILE}" "${ALL_USERSPACE_INVENTORY_FILE}" "${TARGETS_FILE}" ">" "${TARGETS_OUTPUT_FILE}"
|
||||
unset TARGETS_BETA
|
||||
unset TARGETS_REVISION
|
||||
unset TARGETS_FILTER_INCLUDE
|
||||
fi
|
||||
|
||||
if [[ "${ARMBIAN_COMMAND}" == "targets-composed" ]]; then
|
||||
display_alert "Done with" "targets-dashboard" "info"
|
||||
return 0
|
||||
fi
|
||||
|
||||
### Images.
|
||||
|
||||
# The image info extractor.
|
||||
|
||||
@ -28,6 +28,7 @@ function armbian_register_commands() {
|
||||
["inventory"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
["targets"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
["targets-dashboard"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
["targets-composed"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
["debs-to-repo-json"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
["gha-matrix"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
["gha-workflow"]="json_info" # implemented in cli_json_info_pre_run and cli_json_info_run
|
||||
|
||||
@ -202,12 +202,100 @@ def find_armbian_src_path():
|
||||
if not os.path.exists(core_boards_path):
|
||||
raise Exception("Can't find config/boards")
|
||||
|
||||
# userspace stuff
|
||||
core_distributions_path = os.path.realpath(os.path.join(armbian_src_path, "config", "distributions"))
|
||||
log.debug(f"Real path to core distributions '{core_distributions_path}'")
|
||||
# Make sure it exists
|
||||
if not os.path.exists(core_distributions_path):
|
||||
raise Exception("Can't find config/distributions")
|
||||
|
||||
core_desktop_path = os.path.realpath(os.path.join(armbian_src_path, "config", "desktop"))
|
||||
log.debug(f"Real path to core desktop '{core_desktop_path}'")
|
||||
# Make sure it exists
|
||||
if not os.path.exists(core_desktop_path):
|
||||
raise Exception("Can't find config/desktop")
|
||||
|
||||
userpatches_boards_path = os.path.realpath(os.path.join(armbian_src_path, "userpatches", "config", "boards"))
|
||||
log.debug(f"Real path to userpatches boards '{userpatches_boards_path}'")
|
||||
has_userpatches_path = os.path.exists(userpatches_boards_path)
|
||||
|
||||
return {"armbian_src_path": armbian_src_path, "compile_sh_full_path": compile_sh_full_path, "core_boards_path": core_boards_path,
|
||||
"userpatches_boards_path": userpatches_boards_path, "has_userpatches_path": has_userpatches_path}
|
||||
return {
|
||||
"armbian_src_path": armbian_src_path, "compile_sh_full_path": compile_sh_full_path, "core_boards_path": core_boards_path,
|
||||
"core_distributions_path": core_distributions_path, "core_desktop_path": core_desktop_path,
|
||||
"userpatches_boards_path": userpatches_boards_path, "has_userpatches_path": has_userpatches_path
|
||||
}
|
||||
|
||||
|
||||
def read_one_distro_config_file(filename):
|
||||
# Read the contents of filename passed in and return it as string, trimmed
|
||||
with open(filename, 'r') as file_handle:
|
||||
file_contents = file_handle.read()
|
||||
return file_contents.strip()
|
||||
|
||||
|
||||
def split_commas_and_clean_into_list(string):
|
||||
ret = []
|
||||
for item in string.split(","):
|
||||
item = item.strip()
|
||||
if item != "":
|
||||
ret.append(item)
|
||||
return ret
|
||||
|
||||
|
||||
def get_desktop_inventory_for_distro(distro, armbian_paths):
|
||||
ret = []
|
||||
desktops_path = armbian_paths["core_desktop_path"]
|
||||
envs_path_for_distro = os.path.join(desktops_path, distro, "environments")
|
||||
if not os.path.exists(envs_path_for_distro):
|
||||
log.warning(f"Can't find desktop environments for distro '{distro}' at '{envs_path_for_distro}'")
|
||||
return ret
|
||||
for env in os.listdir(envs_path_for_distro):
|
||||
one_env_path = os.path.join(envs_path_for_distro, env)
|
||||
if not os.path.isdir(one_env_path):
|
||||
continue
|
||||
log.debug(f"Processing desktop '{env}' for distro '{distro}'")
|
||||
support_file_path = os.path.join(one_env_path, "support")
|
||||
arches_file_path = os.path.join(one_env_path, "architectures")
|
||||
if not os.path.exists(support_file_path):
|
||||
log.warning(f"Can't find desktop support file for distro '{distro}' and environment '{env}' at '{support_file_path}'")
|
||||
continue
|
||||
if not os.path.exists(arches_file_path):
|
||||
log.warning(f"Can't find desktop arches file for distro '{distro}' and environment '{env}' at '{arches_file_path}'")
|
||||
continue
|
||||
|
||||
env_main_info = {
|
||||
"id": env,
|
||||
"support": read_one_distro_config_file(support_file_path),
|
||||
"arches": split_commas_and_clean_into_list(read_one_distro_config_file(arches_file_path))
|
||||
}
|
||||
ret.append(env_main_info)
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def armbian_get_all_userspace_inventory():
|
||||
armbian_paths = find_armbian_src_path()
|
||||
distros_path = armbian_paths["core_distributions_path"]
|
||||
all_distros = []
|
||||
# find and loop over every directory in distros_path, including symlinks
|
||||
for distro in os.listdir(distros_path):
|
||||
one_distro_path = os.path.join(distros_path, distro)
|
||||
if not os.path.isdir(one_distro_path):
|
||||
continue
|
||||
log.debug(f"Processing distro '{distro}'")
|
||||
support_file_path = os.path.join(one_distro_path, "support")
|
||||
arches_file_path = os.path.join(one_distro_path, "architectures")
|
||||
name_file_path = os.path.join(one_distro_path, "name")
|
||||
distro_main_info = {
|
||||
"id": distro,
|
||||
"name": read_one_distro_config_file(name_file_path),
|
||||
"support": read_one_distro_config_file(support_file_path),
|
||||
"arches": split_commas_and_clean_into_list(read_one_distro_config_file(arches_file_path)),
|
||||
"desktops": get_desktop_inventory_for_distro(distro, armbian_paths)
|
||||
}
|
||||
all_distros.append(distro_main_info)
|
||||
|
||||
return all_distros
|
||||
|
||||
|
||||
def armbian_get_all_boards_inventory():
|
||||
|
||||
@ -52,6 +52,10 @@ for board in board_inventory:
|
||||
if board_inventory[board]["BOARD_HAS_VIDEO"]:
|
||||
not_eos_with_video_boards_all_branches.append(data_from_inventory)
|
||||
|
||||
userspace_inventory_file = sys.argv[2]
|
||||
with open(userspace_inventory_file, 'r') as f:
|
||||
userspace_inventory = json.load(f)
|
||||
|
||||
# get the third argv, which is the targets.yaml file.
|
||||
targets_yaml_file = sys.argv[3]
|
||||
# read it as yaml, modern way
|
||||
@ -61,6 +65,103 @@ with open(targets_yaml_file, 'r') as f:
|
||||
# Keep a running of all the invocations we want to make.
|
||||
invocations_dict: list[dict] = []
|
||||
|
||||
|
||||
# userspace inventory is a bit more complex, here's a function
|
||||
def get_userspace_inventory(opts: dict):
|
||||
ret = []
|
||||
log.info("Processing userspace inventory...")
|
||||
log.debug(f"Processing userspace inventory options: {opts}")
|
||||
|
||||
# set default opts if not present
|
||||
if opts is None:
|
||||
opts = {}
|
||||
if "arches" not in opts:
|
||||
opts["arches"] = {"arm64": [{"BOARD": "uefi-arm64", "BRANCH": "current"}]} # default is arm64 only
|
||||
if "minimal" not in opts:
|
||||
opts["minimal"] = False
|
||||
if "cli" not in opts:
|
||||
opts["cli"] = True # default on, only for CLI
|
||||
if "cloud" not in opts:
|
||||
opts["cloud"] = False
|
||||
if "desktops" not in opts:
|
||||
opts["desktops"] = False
|
||||
if "desktop_variations" not in opts:
|
||||
opts["desktop_variations"] = [[]]
|
||||
|
||||
# loop over the userspace inventory
|
||||
for userspace in userspace_inventory:
|
||||
if userspace["support"] == "eos":
|
||||
log.debug(f"Skipping userspace inventory entry: '{userspace['id']}' has support '{userspace['support']}'")
|
||||
continue
|
||||
|
||||
if "skip-releases" in opts and userspace["id"] in opts["skip-releases"]:
|
||||
log.info(f"Skipping userspace inventory entry: '{userspace['id']}' is in skip-releases list.")
|
||||
continue
|
||||
|
||||
if "only-releases" in opts and userspace["id"] not in opts["only-releases"]:
|
||||
log.info(f"Skipping userspace inventory entry: '{userspace['id']}' is not in only-releases list.")
|
||||
continue
|
||||
|
||||
log.info(f"Processing userspace inventory for distro: {userspace['id']}")
|
||||
|
||||
# loop over the wanted wanted_arch'es
|
||||
for wanted_arch in opts["arches"]:
|
||||
wanted_bbs_for_arch = opts["arches"][wanted_arch]
|
||||
log.debug(f"Processing wanted userspace inventory wanted_arch: '{wanted_arch}' - '{wanted_bbs_for_arch}'")
|
||||
# if the wanted_arch is not in the userspace, skip it completely.
|
||||
if wanted_arch not in userspace["arches"]:
|
||||
log.debug(f"Skipping userspace inventory entry: '{userspace['id']}' does not support wanted_arch '{wanted_arch}'")
|
||||
continue
|
||||
|
||||
if opts["cli"]:
|
||||
for bb in wanted_bbs_for_arch:
|
||||
ret.append({**bb, **{"RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "no", "BUILD_DESKTOP": "no"}})
|
||||
|
||||
if opts["minimal"]:
|
||||
for bb in wanted_bbs_for_arch:
|
||||
ret.append({**bb, **{"RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "yes", "BUILD_DESKTOP": "no"}})
|
||||
|
||||
if opts["cloud"]: # rpardini's cloud images.
|
||||
for bb in wanted_bbs_for_arch:
|
||||
ret.append({**bb, **{
|
||||
"RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "no", "BUILD_DESKTOP": "no", "CLOUD_IMAGE": "yes"
|
||||
}})
|
||||
|
||||
if opts["desktops"]:
|
||||
# loop over the desktops in userspace; skip any that are eos, or that don't have the wanted arch
|
||||
for desktop in userspace["desktops"]:
|
||||
if desktop["support"] == "eos":
|
||||
log.warning(
|
||||
f"Skipping userspace inventory desktop: '{desktop['id']}' has support '{desktop['support']} for userspace '{userspace['id']}'")
|
||||
continue
|
||||
|
||||
if "skip-desktops" in opts and desktop["id"] in opts["skip-desktops"]:
|
||||
log.info(f"Skipping userspace inventory desktop: '{desktop['id']}' is in skip-desktops list.")
|
||||
continue
|
||||
|
||||
if "only-desktops" in opts and desktop["id"] not in opts["only-desktops"]:
|
||||
log.info(f"Skipping userspace inventory desktop: '{desktop['id']}' is not in only-desktops list.")
|
||||
continue
|
||||
|
||||
if wanted_arch not in desktop["arches"]:
|
||||
log.debug(
|
||||
f"Skipping userspace inventory desktop: '{desktop['id']}' does not support wanted_arch '{wanted_arch}' for userspace '{userspace['id']}'")
|
||||
continue
|
||||
|
||||
# loop over the variants... desktop_variations is a list of lists
|
||||
for variant in opts["desktop_variations"]:
|
||||
appgroups_comma = ",".join(variant)
|
||||
|
||||
for bb in wanted_bbs_for_arch:
|
||||
ret.append({**bb, **{
|
||||
"RELEASE": userspace["id"], "USERSPACE_ARCH": wanted_arch, "BUILD_MINIMAL": "no", "BUILD_DESKTOP": "yes",
|
||||
"DESKTOP_ENVIRONMENT_CONFIG_NAME": "config_base", # yeah, config_base is hardcoded.
|
||||
"DESKTOP_APPGROUPS_SELECTED": appgroups_comma, # hopefully empty works
|
||||
"DESKTOP_ENVIRONMENT": desktop["id"]}})
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
# Loop over targets
|
||||
for target_name in targets["targets"]:
|
||||
target_obj = targets["targets"][target_name]
|
||||
@ -97,10 +198,12 @@ for target_name in targets["targets"]:
|
||||
|
||||
# Now add to all_items by resolving the "items-from-inventory" key
|
||||
if "items-from-inventory" in target_obj:
|
||||
# loop over the keys
|
||||
# loop over the keys, for regular board vs branches inventory
|
||||
for key in target_obj["items-from-inventory"]:
|
||||
to_add = []
|
||||
if key == "all":
|
||||
if key == "userspace":
|
||||
to_add.extend(get_userspace_inventory(target_obj["items-from-inventory"][key]))
|
||||
elif key == "all":
|
||||
to_add.extend(all_boards_all_branches)
|
||||
elif key == "not-eos":
|
||||
to_add.extend(not_eos_boards_all_branches)
|
||||
|
||||
23
lib/tools/info/userspace-inventory.py
Normal file
23
lib/tools/info/userspace-inventory.py
Normal file
@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
# Copyright (c) 2023 Ricardo Pardini <ricardo@pardini.net>
|
||||
# This file is a part of the Armbian Build Framework https://github.com/armbian/build/
|
||||
# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from common import armbian_utils
|
||||
|
||||
# Prepare logging
|
||||
armbian_utils.setup_logging()
|
||||
log: logging.Logger = logging.getLogger("userspace-inventory")
|
||||
|
||||
all = armbian_utils.armbian_get_all_userspace_inventory()
|
||||
log.info(f"Inventoried {len(all)} userspace combinations.")
|
||||
print(json.dumps(all, indent=4, sort_keys=True))
|
||||
Loading…
Reference in New Issue
Block a user