diff --git a/lib/functions/artifacts/artifacts-registry.sh b/lib/functions/artifacts/artifacts-registry.sh new file mode 100644 index 0000000000..6f909c90d5 --- /dev/null +++ b/lib/functions/artifacts/artifacts-registry.sh @@ -0,0 +1,10 @@ +function armbian_register_artifacts() { + + declare -g -A ARMBIAN_ARTIFACTS_TO_HANDLERS_DICT=( + #["firmware"]="firmware" + ["kernel"]="kernel" + ["u-boot"]="uboot" + ["uboot"]="uboot" + ) + +} diff --git a/lib/functions/artifacts/kernel.sh b/lib/functions/artifacts/kernel.sh new file mode 100644 index 0000000000..4735d4b010 --- /dev/null +++ b/lib/functions/artifacts/kernel.sh @@ -0,0 +1,150 @@ +function artifact_kernel_cli_adapter_pre_run() { + declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command. + + # "gimme root on a Linux machine" + cli_standard_relaunch_docker_or_sudo +} + +function artifact_kernel_cli_adapter_config_prep() { + declare KERNEL_ONLY="yes" # @TODO: this is a hack, for the board/family code's benefit... + use_board="yes" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive. +} + +function artifact_kernel_prepare_version() { + display_alert "artifact_kernel_XXXXXX" "artifact_kernel_XXXXXX" "warn" + artifact_version="undetermined" # outer scope + artifact_version_reason="undetermined" # outer scope + + # Prepare the version, "sans-repos": just the armbian/build repo contents are available. + # It is OK to reach out to the internet for a curl or ls-remote, but not for a git clone. + + # - Given KERNELSOURCE and KERNELBRANCH, get: + # - SHA1 of the commit (this is generic... and used for other pkgs) + # - The first 10 lines of the root Makefile at that commit (cached lookup, same SHA1=same Makefile) + # - This gives us the full version plus codename. + # - Make sure this is sane, ref KERNEL_MAJOR_MINOR. + # - Get the drivers patch hash (given LINUXFAMILY and the vX.Z.Y version) + # - Get the kernel patches hash. (could just hash the KERNELPATCHDIR non-disabled contents, or use Python patching proper?) + # - Get the kernel .config hash, composed of + # - KERNELCONFIG? .config hash + # - extensions mechanism, have an array of hashes that is then hashed together. + # - Hash of the relevant lib/ bash sources involved, say compilation-kernel*.sh etc + # All those produce a version string like: + # 6.1.8-<4-digit-SHA1>_<4_digit_drivers>-<4_digit_patches>-<4_digit_config>-<4_digit_libs> + # 6.2-rc5-a0b1-c2d3-e4f5-g6h7-i8j9 + + debug_var BRANCH + debug_var REVISION + debug_var KERNELSOURCE + debug_var KERNELBRANCH + debug_var LINUXFAMILY + debug_var BOARDFAMILY + debug_var KERNEL_MAJOR_MINOR + debug_var KERNELPATCHDIR + + declare short_hash_size=4 + + declare -A GIT_INFO=([GIT_SOURCE]="${KERNELSOURCE}" [GIT_REF]="${KERNELBRANCH}") + run_memoized GIT_INFO "git2info" memoized_git_ref_to_info "include_makefile_body" + debug_dict GIT_INFO + + declare short_sha1="${GIT_INFO[SHA1]:0:${short_hash_size}}" + + # get the drivers hash... + declare kernel_drivers_patch_hash + LOG_SECTION="kernel_drivers_create_patches_hash_only" do_with_logging do_with_hooks kernel_drivers_create_patches_hash_only + declare kernel_drivers_hash_short="${kernel_drivers_patch_hash:0:${short_hash_size}}" + + # get the kernel patches hash... + # @TODO: why not just delegate this to the python patching, with some "dry-run" / hash-only option? + declare patches_hash="undetermined" + declare hash_files="undetermined" + calculate_hash_for_all_files_in_dirs "${SRC}/patch/kernel/${KERNELPATCHDIR}" "${USERPATCHES_PATH}/kernel/${KERNELPATCHDIR}" + patches_hash="${hash_files}" + declare kernel_patches_hash_short="${patches_hash:0:${short_hash_size}}" + + # get the .config hash... also userpatches... + declare kernel_config_source_filename="" # which actual .config was used? + prepare_kernel_config_core_or_userpatches + declare hash_files="undetermined" + calculate_hash_for_files "${kernel_config_source_filename}" + config_hash="${hash_files}" + declare config_hash_short="${config_hash:0:${short_hash_size}}" + + # @TODO: get the extensions' .config modyfing hashes... + # @TODO: include the compiler version? host release? + + # get the hashes of the lib/ bash sources involved... + declare hash_files="undetermined" + calculate_hash_for_files "${SRC}"/lib/functions/compilation/kernel*.sh # maybe also this file, "${SRC}"/lib/functions/artifacts/kernel.sh + declare bash_hash="${hash_files}" + declare bash_hash_short="${bash_hash:0:${short_hash_size}}" + + # outer scope + artifact_version="${GIT_INFO[MAKEFILE_VERSION]}-S${short_sha1}-D${kernel_drivers_hash_short}-P${kernel_patches_hash_short}-C${config_hash_short}-B${bash_hash_short}" + # @TODO: validate it begins with a digit, and is at max X chars long. + + declare -a reasons=( + "version \"${GIT_INFO[MAKEFILE_FULL_VERSION]}\"" + "git revision \"${GIT_INFO[SHA1]}\"" + "codename \"${GIT_INFO[MAKEFILE_CODENAME]}\"" + "drivers hash \"${kernel_drivers_patch_hash}\"" + "patches hash \"${patches_hash}\"" + ".config hash \"${config_hash}\"" + "framework bash hash \"${bash_hash}\"" + ) + + artifact_version_reason="${reasons[*]}" # outer scope # @TODO better + + # map what "compile_kernel()" will produce - legacy deb names and versions + artifact_map_versions_legacy=( + ["linux-image-${BRANCH}-${LINUXFAMILY}"]="${REVISION}" + ["linux-dtb-${BRANCH}-${LINUXFAMILY}"]="${REVISION}" + ["linux-headers-${BRANCH}-${LINUXFAMILY}"]="${REVISION}" + ) + + # now, one for each file in the artifact... we've 3 packages produced, all the same version + artifact_map_versions=( + ["linux-image-${BRANCH}-${LINUXFAMILY}"]="${artifact_version}" + ["linux-dtb-${BRANCH}-${LINUXFAMILY}"]="${artifact_version}" + ["linux-headers-${BRANCH}-${LINUXFAMILY}"]="${artifact_version}" + ) + + return 0 +} + +function artifact_kernel_is_available_in_local_cache() { + display_alert "artifact_kernel_XXXXXX" "artifact_kernel_XXXXXX" "warn" + # Check if the exact DEB exists on disk (output/debs), nothing else. + # This is more about composing the .deb filename than checking if it exists. +} + +function artifact_kernel_is_available_in_remote_cache() { + display_alert "artifact_kernel_XXXXXX" "artifact_kernel_XXXXXX" "warn" + # Check if the DEB can be obtained remotely, eg: + # - in ghcr.io (via ORAS) + # - in an apt repo (via apt-get), eg, Armbian's repo. + # this is only about availability, not download. use HEAD requests / metadata-only pulls + # what about multiple possible OCI endpoints / URLs? try them all? +} + +function artifact_kernel_obtain_from_remote_cache() { + display_alert "artifact_kernel_XXXXXX" "artifact_kernel_XXXXXX" "warn" + # Having confirmed it is available remotely, go download it into the local cache. + # is_available_in_local_cache() must return =yes after this. + # could be a good idea to transfer some SHA256 id from "is_available" to "obtain" to avoid overhead? or just do it together? +} + +function artifact_kernel_build_from_sources() { + display_alert "artifact_kernel_XXXXXX" "artifact_kernel_XXXXXX" "warn" + # having failed all the cache obtaining, build it from sources. + compile_kernel + + capture_rename_legacy_debs_into_artifacts +} + +function artifact_kernel_deploy_to_remote_cache() { + display_alert "artifact_kernel_XXXXXX" "artifact_kernel_XXXXXX" "warn" + # having built a new artifact, deploy it to the remote cache. + # consider multiple targets, retries, etc. +} diff --git a/lib/functions/artifacts/u-boot.sh b/lib/functions/artifacts/u-boot.sh new file mode 100644 index 0000000000..f5557ad58a --- /dev/null +++ b/lib/functions/artifacts/u-boot.sh @@ -0,0 +1,130 @@ +function artifact_uboot_cli_adapter_pre_run() { + declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command. + + # "gimme root on a Linux machine" + cli_standard_relaunch_docker_or_sudo +} + +function artifact_uboot_cli_adapter_config_prep() { + declare KERNEL_ONLY="yes" # @TODO: this is a hack, for the board/family code's benefit... + use_board="yes" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive. +} + +function artifact_uboot_prepare_version() { + display_alert "artifact_uboot_XXXXXX" "artifact_uboot_XXXXXX" "warn" + artifact_version="undetermined" # outer scope + artifact_version_reason="undetermined" # outer scope + + # Prepare the version, "sans-repos": just the armbian/build repo contents are available. + # It is OK to reach out to the internet for a curl or ls-remote, but not for a git clone/fetch. + + # - Given BOOTSOURCE and BOOTBRANCH, get: + # - SHA1 of the commit (this is generic... and used for other pkgs) + # - The first 10 lines of the root Makefile at that commit (cached lookup, same SHA1=same Makefile) + # - This gives us the full version plus codename. + # - Get the u-boot patches hash. (could just hash the BOOTPATCHDIR non-disabled contents, or use Python patching proper?) + # - Hash of the relevant lib/ bash sources involved, say compilation/uboot*.sh etc + # All those produce a version string like: + # 2023.11-<4-digit-SHA1>_<4_digit_patches> + + debug_var BOOTSOURCE + debug_var BOOTBRANCH + debug_var BOOTPATCHDIR + debug_var BOARD + + declare short_hash_size=4 + + declare -A GIT_INFO=([GIT_SOURCE]="${BOOTSOURCE}" [GIT_REF]="${BOOTBRANCH}") + run_memoized GIT_INFO "git2info" memoized_git_ref_to_info "include_makefile_body" + debug_dict GIT_INFO + + declare short_sha1="${GIT_INFO[SHA1]:0:${short_hash_size}}" + + # get the uboot patches hash... + # @TODO: why not just delegate this to the python patching, with some "dry-run" / hash-only option? + # @TODO: this is even more grave in case of u-boot: v2022.10 has patches for many boards inside, gotta resolve. + declare patches_hash="undetermined" + declare hash_files="undetermined" + calculate_hash_for_all_files_in_dirs "${SRC}/patch/u-boot/${BOOTPATCHDIR}" "${USERPATCHES_PATH}/u-boot/${BOOTPATCHDIR}" + patches_hash="${hash_files}" + declare uboot_patches_hash_short="${patches_hash:0:${short_hash_size}}" + + # get the hashes of the lib/ bash sources involved... + declare hash_files="undetermined" + calculate_hash_for_files "${SRC}"/lib/functions/compilation/uboot*.sh # maybe also this file, "${SRC}"/lib/functions/artifacts/u-boot.sh + declare bash_hash="${hash_files}" + declare bash_hash_short="${bash_hash:0:${short_hash_size}}" + + # outer scope + artifact_version="${GIT_INFO[MAKEFILE_VERSION]}-S${short_sha1}-P${uboot_patches_hash_short}-B${bash_hash_short}" + # @TODO: validate it begins with a digit, and is at max X chars long. + + declare -a reasons=( + "version \"${GIT_INFO[MAKEFILE_FULL_VERSION]}\"" + "git revision \"${GIT_INFO[SHA1]}\"" + "patches hash \"${patches_hash}\"" + "framework bash hash \"${bash_hash}\"" + ) + + artifact_version_reason="${reasons[*]}" # outer scope # @TODO better + + # now, one for each file in the artifact... + artifact_map_versions=( + ["u-boot"]="${artifact_version}" + ) + + # map what "compile_uboot()" will produce - legacy deb names and versions + artifact_map_versions_legacy=( + ["linux-u-boot-${BRANCH}-${BOARD}"]="${REVISION}" + ) + + # now, one for each file in the artifact... single package, so just one entry + artifact_map_versions=( + ["linux-u-boot-${BRANCH}-${BOARD}"]="${artifact_version}" + ) + + return 0 +} + +function artifact_uboot_is_available_in_local_cache() { + display_alert "artifact_uboot_XXXXXX" "artifact_uboot_XXXXXX" "warn" + # Check if the exact DEB exists on disk (output/debs), nothing else. + # This is more about composing the .deb filename than checking if it exists. +} + +function artifact_uboot_is_available_in_remote_cache() { + display_alert "artifact_uboot_XXXXXX" "artifact_uboot_XXXXXX" "warn" + # Check if the DEB can be obtained remotely, eg: + # - in ghcr.io (via ORAS) + # - in an apt repo (via apt-get), eg, Armbian's repo. + # this is only about availability, not download. use HEAD requests / metadata-only pulls + # what about multiple possible OCI endpoints / URLs? try them all? +} + +function artifact_uboot_obtain_from_remote_cache() { + display_alert "artifact_uboot_XXXXXX" "artifact_uboot_XXXXXX" "warn" + # Having confirmed it is available remotely, go download it into the local cache. + # is_available_in_local_cache() must return =yes after this. + # could be a good idea to transfer some SHA256 id from "is_available" to "obtain" to avoid overhead? or just do it together? +} + +function artifact_uboot_build_from_sources() { + display_alert "artifact_uboot_XXXXXX" "artifact_uboot_XXXXXX" "warn" + # having failed all the cache obtaining, build it from sources. + + if [[ -n "${ATFSOURCE}" && "${ATFSOURCE}" != "none" ]]; then + LOG_SECTION="compile_atf" do_with_logging compile_atf + fi + + declare uboot_git_revision="not_determined_yet" + LOG_SECTION="uboot_prepare_git" do_with_logging_unless_user_terminal uboot_prepare_git + LOG_SECTION="compile_uboot" do_with_logging compile_uboot + + capture_rename_legacy_debs_into_artifacts # has its own logging section +} + +function artifact_uboot_deploy_to_remote_cache() { + display_alert "artifact_uboot_XXXXXX" "artifact_uboot_XXXXXX" "warn" + # having built a new artifact, deploy it to the remote cache. + # consider multiple targets, retries, etc. +} diff --git a/lib/functions/cli/cli-artifact.sh b/lib/functions/cli/cli-artifact.sh new file mode 100644 index 0000000000..fde1da34e8 --- /dev/null +++ b/lib/functions/cli/cli-artifact.sh @@ -0,0 +1,109 @@ +function cli_artifact_pre_run() { + initialize_artifact "${WHAT:-"kernel"}" + # Run the pre run adapter + artifact_cli_adapter_pre_run +} + +function cli_artifact_run() { + display_alert "artifact" "${chosen_artifact}" "warn" + display_alert "artifact" "${chosen_artifact} :: ${chosen_artifact_impl}()" "warn" + artifact_cli_adapter_config_prep # only if in cli. + + # only if in cli, if not just run it bare, since we'd be already inside do_with_default_build + do_with_default_build obtain_complete_artifact < /dev/null +} + +function create_artifact_functions() { + declare -a funcs=( + "cli_adapter_pre_run" "cli_adapter_config_prep" + "prepare_version" + "is_available_in_local_cache" "is_available_in_remote_cache" "obtain_from_remote_cache" + "deploy_to_remote_cache" + "build_from_sources" + ) + for func in "${funcs[@]}"; do + declare impl_func="artifact_${chosen_artifact_impl}_${func}" + if [[ $(type -t "${impl_func}") == function ]]; then + declare cmd + cmd="$( + cat <<- ARTIFACT_DEFINITION + function artifact_${func}() { + display_alert "Calling artifact function" "${impl_func}() \$*" "warn" + ${impl_func} "\$@" + } + ARTIFACT_DEFINITION + )" + eval "${cmd}" + else + exit_with_error "Missing artifact implementation function '${impl_func}'" + fi + done +} + +function initialize_artifact() { + declare -g chosen_artifact="${1}" + armbian_register_artifacts + declare -g chosen_artifact_impl="${ARMBIAN_ARTIFACTS_TO_HANDLERS_DICT["${chosen_artifact}"]}" + [[ "x${chosen_artifact_impl}x" == "xx" ]] && exit_with_error "Unknown artifact '${chosen_artifact}'" + display_alert "artifact" "${chosen_artifact} :: ${chosen_artifact_impl}()" "info" + create_artifact_functions +} + +function obtain_complete_artifact() { + declare -g artifact_version="undetermined" + declare -g artifact_version_reason="undetermined" + declare -A -g artifact_map_versions=() + declare -A -g artifact_map_versions_legacy=() + + # Check if REVISION is set, otherwise exit_with_error + [[ "x${REVISION}x" == "xx" ]] && exit_with_error "REVISION is not set" + + artifact_prepare_version + debug_var artifact_version + debug_var artifact_version_reason + debug_dict artifact_map_versions_legacy + debug_dict artifact_map_versions + + # @TODO the whole artifact upload/download dance + artifact_is_available_in_local_cache + artifact_is_available_in_remote_cache + artifact_obtain_from_remote_cache + + artifact_build_from_sources + + artifact_deploy_to_remote_cache +} + +# This is meant to be run after config, inside default build. +function build_artifact() { + initialize_artifact "${WHAT:-"kernel"}" + obtain_complete_artifact +} + +function capture_rename_legacy_debs_into_artifacts() { + LOG_SECTION="capture_rename_legacy_debs_into_artifacts" do_with_logging capture_rename_legacy_debs_into_artifacts_logged +} + +function capture_rename_legacy_debs_into_artifacts_logged() { + # So the deb-building code will consider the artifact_version in it's "Version: " field in the .debs. + # But it will produce .deb's with the legacy name. We gotta find and rename them. + # Loop over the artifact_map_versions, and rename the .debs. + debug_dict artifact_map_versions_legacy + debug_dict artifact_map_versions + + declare deb_name_base deb_name_full new_name_full legacy_version legacy_base_version + for deb_name_base in "${!artifact_map_versions[@]}"; do + legacy_base_version="${artifact_map_versions_legacy[${deb_name_base}]}" + if [[ -z "${legacy_base_version}" ]]; then + exit_with_error "Legacy base version not found for artifact '${deb_name_base}'" + fi + + display_alert "Legacy base version" "${legacy_base_version}" "info" + legacy_version="${legacy_base_version}_${ARCH}" # Arch-specific package; has ARCH at the end. + deb_name_full="${DEST}/debs/${deb_name_base}_${legacy_version}.deb" + new_name_full="${DEST}/debs/${deb_name_base}_${artifact_map_versions[${deb_name_base}]}_${ARCH}.deb" + display_alert "Full legacy deb name" "${deb_name_full}" "info" + display_alert "New artifact deb name" "${new_name_full}" "info" + run_host_command_logged mv -v "${deb_name_full}" "${new_name_full}" + done +} diff --git a/lib/functions/cli/commands.sh b/lib/functions/cli/commands.sh index e6b9c879ac..ee70d1dcd1 100644 --- a/lib/functions/cli/commands.sh +++ b/lib/functions/cli/commands.sh @@ -27,11 +27,15 @@ function armbian_register_commands() { # shortcuts, see vars set below. the use legacy single build, and try to control it via variables ["kernel"]="standard_build" + ["kernel-config"]="standard_build" ["u-boot"]="standard_build" ["uboot"]="standard_build" # external tooling, made easy. ["oras-upload"]="oras" # implemented in cli_oras_pre_run and cli_oras_run; up/down/info are the same, see vars below + + # all-around artifact wrapper + ["artifact"]="artifact" # implemented in cli_artifact_pre_run and cli_artifact_run ["undecided"]="undecided" # implemented in cli_undecided_pre_run and cli_undecided_run - relaunches either build or docker ) @@ -48,6 +52,7 @@ function armbian_register_commands() { ["config-dump"]="CONFIG_DEFS_ONLY='yes'" ["configdump"]="CONFIG_DEFS_ONLY='yes'" + ["kernel-config"]="KERNEL_ONLY='yes' JUST_KERNEL='yes' KERNEL_IGNORE_DEB='yes' KERNEL_CONFIGURE='yes'" ["kernel"]="KERNEL_ONLY='yes' JUST_KERNEL='yes' KERNEL_IGNORE_DEB='yes' KERNEL_CONFIGURE='no'" ["u-boot"]="KERNEL_ONLY='yes' JUST_UBOOT='yes' UBOOT_IGNORE_DEB='yes'" ["uboot"]="KERNEL_ONLY='yes' JUST_UBOOT='yes' UBOOT_IGNORE_DEB='yes'" diff --git a/lib/functions/compilation/distcc.sh b/lib/functions/compilation/distcc.sh index f3c2e1a0da..f10406de19 100644 --- a/lib/functions/compilation/distcc.sh +++ b/lib/functions/compilation/distcc.sh @@ -45,6 +45,7 @@ function prepare_distcc_compilation_config() { display_alert "DISTCC_TARGETS_SEGMENTS" "${DISTCC_TARGETS_SEGMENTS[*]}" "warn" else # If not using distcc, just add "$CTHREADS" to the DISTCC_MAKE_J_PARALLEL array. + [[ -z "${CTHREADS}" ]] && exit_with_error "CTHREADS is not set in prepare_distcc_compilation_config" DISTCC_MAKE_J_PARALLEL=("${CTHREADS}") fi diff --git a/lib/functions/compilation/kernel-config.sh b/lib/functions/compilation/kernel-config.sh index f5ad209a29..9ee57c4b64 100644 --- a/lib/functions/compilation/kernel-config.sh +++ b/lib/functions/compilation/kernel-config.sh @@ -1,3 +1,19 @@ +function prepare_kernel_config_core_or_userpatches() { + # LINUXCONFIG is set or exit_with_error + [[ -z "${LINUXCONFIG}" ]] && exit_with_error "LINUXCONFIG not set: '${LINUXCONFIG}'" + + if [[ -f $USERPATCHES_PATH/$LINUXCONFIG.config ]]; then + display_alert "Using kernel config provided by user" "userpatches/$LINUXCONFIG.config" "info" + kernel_config_source_filename="${USERPATCHES_PATH}/${LINUXCONFIG}.config" + elif [[ -f "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" ]]; then + display_alert "Using kernel config provided by user in config/kernel folder" "config/kernel/${LINUXCONFIG}.config" "info" + kernel_config_source_filename="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" + else + display_alert "Using kernel config file" "config/kernel/$LINUXCONFIG.config" "info" + kernel_config_source_filename="${SRC}/config/kernel/${LINUXCONFIG}.config" + fi +} + function kernel_config() { # check $kernel_work_dir is set and exists, or bail [[ -z "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir is not set" @@ -21,6 +37,7 @@ function kernel_config() { function kernel_config_initialize() { display_alert "Configuring kernel" "${LINUXCONFIG}" "info" + cd "${kernel_work_dir}" || exit_with_error "kernel_work_dir does not exist: ${kernel_work_dir}" # If a `.config` already exists (from previous build), store it, preserving date. # We will compare the result of the new configuration to it, and if the contents are the same, we'll restore the original date. @@ -34,28 +51,17 @@ function kernel_config_initialize() { # copy kernel config from configuration, userpatches if [[ "${KERNEL_KEEP_CONFIG}" == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then display_alert "Using previously-exported kernel config" "${DEST}/config/$LINUXCONFIG.config" "info" - run_host_command_logged cp -pv "${DEST}/config/${LINUXCONFIG}.config" .config + run_host_command_logged cp -pv "${DEST}/config/${LINUXCONFIG}.config" "${kernel_work_dir}/.config" else - # @TODO: rpardini: this is too contrived - if [[ -f $USERPATCHES_PATH/$LINUXCONFIG.config ]]; then - display_alert "Using kernel config provided by user" "userpatches/$LINUXCONFIG.config" "info" - run_host_command_logged cp -pv "${USERPATCHES_PATH}/${LINUXCONFIG}.config" .config - kernel_config_source_filename="${USERPATCHES_PATH}/${LINUXCONFIG}.config" - elif [[ -f "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" ]]; then - display_alert "Using kernel config provided by user in config/kernel folder" "config/kernel/${LINUXCONFIG}.config" "info" - run_host_command_logged cp -pv "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" .config - kernel_config_source_filename="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" - else - display_alert "Using kernel config file" "config/kernel/$LINUXCONFIG.config" "info" - run_host_command_logged cp -pv "${SRC}/config/kernel/${LINUXCONFIG}.config" .config - kernel_config_source_filename="${SRC}/config/kernel/${LINUXCONFIG}.config" - fi + prepare_kernel_config_core_or_userpatches + run_host_command_logged cp -pv "${kernel_config_source_filename}" "${kernel_work_dir}/.config" fi # Start by running olddefconfig -- always. # It "updates" the config, using defaults from Kbuild files in the source tree. # It is worthy noting that on the first run, it builds the tools, so the host-side compiler has to be working, # regardless of the cross-build toolchain. + cd "${kernel_work_dir}" || exit_with_error "kernel_work_dir does not exist: ${kernel_work_dir}" run_kernel_make olddefconfig # Run the core-armbian config modifications here, built-in extensions: diff --git a/lib/functions/compilation/kernel-debs.sh b/lib/functions/compilation/kernel-debs.sh index a8f0d8e318..eb5a6b2689 100644 --- a/lib/functions/compilation/kernel-debs.sh +++ b/lib/functions/compilation/kernel-debs.sh @@ -41,7 +41,18 @@ function prepare_kernel_packaging_debs() { # Some variables and settings used throughout the script declare kernel_version_family="${kernel_version}-${LINUXFAMILY}" - declare package_version="${REVISION}" + + # Package version. Affects users upgrading from repo! + declare package_version="${REVISION}" # default, "classic" Armbian non-version. + # If we're building an artifact, use the pre-determined artifact version. + if [[ "${artifact_version:-""}" != "" ]]; then + if [[ "${artifact_version}" == "undetermined" ]]; then + exit_with_error "Undetermined artifact version during kernel deb packaging. This is a bug, report it." + fi + display_alert "Using artifact version for kernel package version" "${artifact_version}" "info" + package_version="${artifact_version}" + fi + display_alert "Kernel .deb package version" "${package_version}" "info" # show incoming tree #display_alert "Kernel install dir" "incoming from KBUILD make" "debug" @@ -140,7 +151,9 @@ function create_kernel_deb() { #display_alert "Package dir" "for package ${package_name}" "debug" #run_host_command_logged tree -C -h -d --du "${package_directory}" - run_host_command_logged dpkg-deb ${DEB_COMPRESS:+-Z$DEB_COMPRESS} --build "${package_directory}" "${deb_output_dir}" # not KDEB compress, we're not under a Makefile + # @TODO: hmm, why doesn't this use fakeroot_dpkg_deb_build() ? + declare final_deb_filename="${deb_output_dir}/${package_name}_${REVISION}_${ARCH}.deb" # for compatibility with non-artifacts + run_host_command_logged dpkg-deb ${DEB_COMPRESS:+-Z$DEB_COMPRESS} --build "${package_directory}" "${final_deb_filename}" # not KDEB compress, we're not under a Makefile done_with_temp_dir "${cleanup_id}" # changes cwd to "${SRC}" and fires the cleanup function early } @@ -210,7 +223,7 @@ function kernel_package_callback_linux_image() { Maintainer: ${MAINTAINER} <${MAINTAINERMAIL}> Section: kernel Provides: linux-image, linux-image-armbian, armbian-$BRANCH - Description: Linux kernel, armbian version $kernel_version_family $BRANCH + Description: Armbian Linux $BRANCH kernel image ${artifact_version_reason:-"${kernel_version_family}"} This package contains the Linux kernel, modules and corresponding other files, kernel_version_family: $kernel_version_family. CONTROL_FILE @@ -263,7 +276,7 @@ function kernel_package_callback_linux_dtb() { Package: ${package_name} Architecture: ${ARCH} Provides: linux-dtb, linux-dtb-armbian, armbian-$BRANCH - Description: Armbian Linux DTB, version ${kernel_version_family} $BRANCH + Description: Armbian Linux $BRANCH DTBs ${artifact_version_reason:-"${kernel_version_family}"} This package contains device blobs from the Linux kernel, version ${kernel_version_family} CONTROL_FILE @@ -379,7 +392,7 @@ function kernel_package_callback_linux_headers() { Architecture: ${ARCH} Provides: linux-headers, linux-headers-armbian, armbian-$BRANCH Depends: make, gcc, libc6-dev, bison, flex, libssl-dev, libelf-dev - Description: Linux kernel headers for ${kernel_version_family} + Description: Armbian Linux $BRANCH headers ${artifact_version_reason:-"${kernel_version_family}"} This package provides kernel header files for ${kernel_version_family} . This is useful for DKMS and building of external modules. @@ -454,7 +467,7 @@ function kernel_package_callback_linux_headers_full_source() { Architecture: ${ARCH} Provides: linux-headers, linux-headers-armbian, armbian-$BRANCH Depends: make, gcc, libc6-dev, bison, flex, libssl-dev, libelf-dev - Description: Linux kernel headers for ${kernel_version_family} - based on full source + Description: Armbian Linux $BRANCH full-source headers ${artifact_version_reason:-"${kernel_version_family}"} This package provides kernel header files for ${kernel_version_family} . This is useful for DKMS and building of external modules. diff --git a/lib/functions/compilation/patch/drivers-harness.sh b/lib/functions/compilation/patch/drivers-harness.sh index ead9c6e46e..de540ebb15 100644 --- a/lib/functions/compilation/patch/drivers-harness.sh +++ b/lib/functions/compilation/patch/drivers-harness.sh @@ -1,12 +1,10 @@ -function calculate_hash_for_files() { - hash_files="$(sha256sum "${@}" | sha256sum | cut -d' ' -f1)" # hash of hashes - hash_files="${hash_files:0:16}" # shorten it to 16 characters - display_alert "Hash for files:" "$hash_files" "debug" +function kernel_drivers_create_patches_hash_only() { + hash_only="yes" kernel_drivers_create_patches "${@}" } function kernel_drivers_create_patches() { - declare kernel_work_dir="${1}" - declare kernel_git_revision="${2}" + kernel_drivers_patch_hash="undetermined" # outer scope + kernel_drivers_patch_file="undetermined" # outer scope declare hash_files # any changes in these files will trigger a cache miss; also any changes in misc .patch with "wireless" at start or "wifi" anywhere in the name calculate_hash_for_files "${SRC}/lib/functions/compilation/patch/drivers_network.sh" "${SRC}/lib/functions/compilation/patch/drivers-harness.sh" "${SRC}"/patch/misc/wireless*.patch "${SRC}"/patch/misc/*wifi*.patch @@ -22,8 +20,16 @@ function kernel_drivers_create_patches() { declare cache_target_file="${cache_dir_base}/${cache_key}.patch" # outer scope variables: - kernel_drivers_patch_file="${cache_target_file}" - kernel_drivers_patch_hash="${cache_key}" + kernel_drivers_patch_file="${cache_target_file}" # outer scope + kernel_drivers_patch_hash="${hash_files}" # outer scope + + if [[ "${hash_only:-"no"}" == "yes" ]]; then + display_alert "Hash-only kernel driver requested" "$kernel_drivers_patch_hash - returning" "warn" + return 0 + fi + + declare kernel_work_dir="${1}" + declare kernel_git_revision="${2}" # If the target file exists, we can skip the patch creation. if [[ -f "${cache_target_file}" ]]; then diff --git a/lib/functions/compilation/uboot.sh b/lib/functions/compilation/uboot.sh index 2dd85d38cf..5b75c642cf 100644 --- a/lib/functions/compilation/uboot.sh +++ b/lib/functions/compilation/uboot.sh @@ -18,7 +18,10 @@ function compile_uboot_target() { local uboot_work_dir="" uboot_work_dir="$(pwd)" - # outer scope variable: uboot_git_revision + # outer scope variable: uboot_git_revision, validate that it is set + if [[ -z "${uboot_git_revision}" ]]; then + exit_with_error "uboot_git_revision is not set" + fi display_alert "${uboot_prefix} Checking out to clean sources SHA1 ${uboot_git_revision}" "{$BOOTSOURCEDIR} for ${target_make}" git checkout -f -q "${uboot_git_revision}" @@ -319,10 +322,22 @@ function compile_uboot() { $(declare -f setup_write_uboot_platform || true) EOF + # Package version. Affects users upgrading from repo! + declare package_version="${REVISION}" # default, "classic" Armbian non-version. + # If we're building an artifact, use the pre-determined artifact version. + if [[ "${artifact_version:-""}" != "" ]]; then + if [[ "${artifact_version}" == "undetermined" ]]; then + exit_with_error "Undetermined artifact version during u-boot deb packaging. This is a bug, report it." + fi + display_alert "Using artifact version for u-boot package version" "${artifact_version}" "info" + package_version="${artifact_version}" + fi + display_alert "Das U-Boot .deb package version" "${package_version}" "info" + # set up control file cat <<- EOF > "$uboottempdir/${uboot_name}/DEBIAN/control" Package: linux-u-boot-${BOARD}-${BRANCH} - Version: $REVISION + Version: ${package_version} Architecture: $ARCH Maintainer: $MAINTAINER <$MAINTAINERMAIL> Installed-Size: 1 @@ -331,18 +346,20 @@ function compile_uboot() { Provides: armbian-u-boot Replaces: armbian-u-boot Conflicts: armbian-u-boot, u-boot-sunxi - Description: Uboot loader $version + Description: Das U-Boot for ${BOARD} ${artifact_version_reason:-"${version}"} EOF # copy config file to the package # useful for FEL boot with overlayfs_wrapper - [[ -f .config && -n $BOOTCONFIG ]] && cp .config "$uboottempdir/${uboot_name}/usr/lib/u-boot/${BOOTCONFIG}" 2>&1 + [[ -f .config && -n $BOOTCONFIG ]] && run_host_command_logged cp .config "$uboottempdir/${uboot_name}/usr/lib/u-boot/${BOOTCONFIG}" # copy license files from typical locations - [[ -f COPYING ]] && cp COPYING "$uboottempdir/${uboot_name}/usr/lib/u-boot/LICENSE" 2>&1 - [[ -f Licenses/README ]] && cp Licenses/README "$uboottempdir/${uboot_name}/usr/lib/u-boot/LICENSE" 2>&1 - [[ -n $atftempdir && -f $atftempdir/license.md ]] && cp "${atftempdir}/license.md" "$uboottempdir/${uboot_name}/usr/lib/u-boot/LICENSE.atf" 2>&1 + [[ -f COPYING ]] && run_host_command_logged cp COPYING "$uboottempdir/${uboot_name}/usr/lib/u-boot/LICENSE" + [[ -f Licenses/README ]] && run_host_command_logged cp Licenses/README "$uboottempdir/${uboot_name}/usr/lib/u-boot/LICENSE" + [[ -n $atftempdir && -f $atftempdir/license.md ]] && run_host_command_logged cp "${atftempdir}/license.md" "$uboottempdir/${uboot_name}/usr/lib/u-boot/LICENSE.atf" - display_alert "Building u-boot deb" "${uboot_name}.deb" + # Important: this forces the deb to have a specific name, and not be version-dependent... + # This is set to `uboot_name="${CHOSEN_UBOOT}_${REVISION}_${ARCH}"` in outer scope... + display_alert "Building u-boot deb" "(version: ${package_version}) ${uboot_name}.deb" fakeroot_dpkg_deb_build "$uboottempdir/${uboot_name}" "$uboottempdir/${uboot_name}.deb" rm -rf "$uboottempdir/${uboot_name}" [[ -n $atftempdir ]] && rm -rf "${atftempdir}" diff --git a/lib/functions/configuration/compilation-config.sh b/lib/functions/configuration/compilation-config.sh index a160d287b4..83742e6fb9 100644 --- a/lib/functions/configuration/compilation-config.sh +++ b/lib/functions/configuration/compilation-config.sh @@ -24,6 +24,8 @@ function prepare_compilation_vars() { Called early, before any compilation work starts. POST_DETERMINE_CTHREADS + # readonly, global + declare -g -r CTHREADS="${CTHREADS}" + return 0 } - diff --git a/lib/functions/general/git-ref2info.sh b/lib/functions/general/git-ref2info.sh new file mode 100644 index 0000000000..68306d1780 --- /dev/null +++ b/lib/functions/general/git-ref2info.sh @@ -0,0 +1,111 @@ +# This has... everything: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=linux-6.1.y +# This has... everything: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?h=v6.2-rc5 + +# get the sha1 of the commit on tag or branch +# git ls-remote --exit-code --symref git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git v6.2-rc5 +# git ls-remote --exit-code --symref git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git v6.2-rc5 + +# 93f875a8526a291005e7f38478079526c843cbec refs/heads/linux-6.1.y +# 4cc398054ac8efe0ff832c82c7caacbdd992312a refs/tags/v6.2-rc5 + +# https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/Makefile?h=linux-6.1.y +# plaintext: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/plain/Makefile?h=4cc398054ac8efe0ff832c82c7caacbdd992312a + +function memoized_git_ref_to_info() { + declare -n MEMO_DICT="${1}" # nameref + declare ref_type ref_name + git_parse_ref "${MEMO_DICT[GIT_REF]}" + MEMO_DICT+=(["REF_TYPE"]="${ref_type}") + MEMO_DICT+=(["REF_NAME"]="${ref_name}") + + # Get the SHA1 of the commit + declare sha1 + sha1="$(git ls-remote --exit-code "${MEMO_DICT[GIT_SOURCE]}" "${ref_name}" | cut -f1)" + MEMO_DICT+=(["SHA1"]="${sha1}") + + if [[ "${2}" == "include_makefile_body" ]]; then + + function obtain_makefile_body_from_git() { + declare git_source="${1}" + declare sha1="${2}" + makefile_body="undetermined" # outer scope + makefile_url="undetermined" # outer scope + makefile_version="undetermined" # outer scope + makefile_codename="undetermined" # outer scope + + declare url="undetermined" + case "${git_source}" in + + "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git") + url="https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/plain/Makefile?h=${sha1}" + ;; + + # @TODO: urgently add support for Google Mirror + # @TODO: china mirror etc. + # @TODO: mirrors might need to be resolved before/during/after this, refactor + + "https://github.com/"*) + # parse org/repo from https://github.com/org/repo + declare org_and_repo="" + org_and_repo="$(echo "${git_source}" | cut -d/ -f4-5)" + url="https://raw.githubusercontent.com/${org_and_repo}/${sha1}/Makefile" + ;; + + "https://source.codeaurora.org/external/imx/linux-imx") + # Random, bizarre stuff here, to keep compatibility with some old stuff + url="https://source.codeaurora.org/external/imx/linux-imx/plain/Makefile?h=${sha1}" + ;; + + *) + exit_with_error "Unknown git source '${git_source}'" + ;; + esac + + display_alert "Fetching Makefile via HTTP" "${url}" "warn" + makefile_url="${url}" + makefile_body="$(curl -sL "${url}")" + + parse_makefile_version "${makefile_body}" + + return 0 + } + + function parse_makefile_version() { + declare makefile_body="${1}" + makefile_version="undetermined" # outer scope + makefile_codename="undetermined" # outer scope + makefile_full_version="undetermined" # outer scope + + local ver=() + ver[0]=$(grep "^VERSION" <(echo "${makefile_body}") | head -1 | awk '{print $(NF)}' | grep -oE '^[[:digit:]]+' || true) + ver[1]=$(grep "^PATCHLEVEL" <(echo "${makefile_body}") | head -1 | awk '{print $(NF)}' | grep -oE '^[[:digit:]]+' || true) + ver[2]=$(grep "^SUBLEVEL" <(echo "${makefile_body}") | head -1 | awk '{print $(NF)}' | grep -oE '^[[:digit:]]+' || true) + ver[3]=$(grep "^EXTRAVERSION" <(echo "${makefile_body}") | head -1 | awk '{print $(NF)}' | grep -oE '^-rc[[:digit:]]+' || true) + makefile_version="${ver[0]:-0}${ver[1]:+.${ver[1]}}${ver[2]:+.${ver[2]}}${ver[3]}" + makefile_full_version="${makefile_version}" + if [[ "${ver[3]}" == "-rc"* ]]; then # contentious:, if an "-rc" EXTRAVERSION, don't include the SUBLEVEL + makefile_version="${ver[0]:-0}${ver[1]:+.${ver[1]}}${ver[3]}" + fi + + # grab the codename while we're at it + makefile_codename="$(grep "^NAME\ =\ " <(echo "${makefile_body}") | head -1 | cut -d '=' -f 2 | sed -e "s|'||g" | xargs echo -n || true)" + # remove any starting whitespace left + makefile_codename="${makefile_codename#"${makefile_codename%%[![:space:]]*}"}" + # remove any trailing whitespace left + makefile_codename="${makefile_codename%"${makefile_codename##*[![:space:]]}"}" + + return 0 + } + + display_alert "Fetching Makefile body" "${ref_name}" "warn" + declare makefile_body makefile_url + declare makefile_version makefile_codename makefile_full_version + obtain_makefile_body_from_git "${MEMO_DICT[GIT_SOURCE]}" "${sha1}" + MEMO_DICT+=(["MAKEFILE_URL"]="${makefile_url}") + #MEMO_DICT+=(["MAKEFILE_BODY"]="${makefile_body}") # large, don't store + MEMO_DICT+=(["MAKEFILE_VERSION"]="${makefile_version}") + MEMO_DICT+=(["MAKEFILE_FULL_VERSION"]="${makefile_full_version}") + MEMO_DICT+=(["MAKEFILE_CODENAME"]="${makefile_codename}") + fi + +} diff --git a/lib/functions/general/git.sh b/lib/functions/general/git.sh index 325d4775aa..8ee4762754 100644 --- a/lib/functions/general/git.sh +++ b/lib/functions/general/git.sh @@ -4,7 +4,7 @@ # sets: ref_type=branch|tag|commit, ref_name=xxx|yyyy|zzzz|HEAD function git_parse_ref() { declare ref="$1" - [[ -z $ref || ($ref != tag:* && $ref != branch:* && $ref != head && $ref != commit:*) ]] && exit_with_error "Error in configuration" + [[ -z $ref || ($ref != tag:* && $ref != branch:* && $ref != head && $ref != commit:*) ]] && exit_with_error "Error in configuration; git_ref '${ref}' is not valid" ref_type=${ref%%:*} # outer scope ref_name=${ref##*:} # outer scope if [[ $ref_type == head ]]; then diff --git a/lib/functions/general/hash-files.sh b/lib/functions/general/hash-files.sh new file mode 100644 index 0000000000..89b2834aaf --- /dev/null +++ b/lib/functions/general/hash-files.sh @@ -0,0 +1,50 @@ +function calculate_hash_for_all_files_in_dirs() { + declare -a dirs_to_hash=("$@") + declare -a files_to_hash=() + for dir in "${dirs_to_hash[@]}"; do + # skip if dir doesn't exist... + if [[ ! -d "${dir}" ]]; then + display_alert "calculate_hash_for_all_files_in_dirs" "skipping non-existent dir \"${dir}\"" "warn" + continue + fi + declare found_files="no" + # shellcheck disable=SC2044 # lets expand... # -L: follow symlinks + for file in $(find -L "${dir}" -type f); do + files_to_hash+=("${file}") + found_files="yes" + done + if [[ "${found_files}" == "no" ]]; then + display_alert "calculate_hash_for_all_files_in_dirs" "skipped empty dir \"${dir}\"" "warn" + fi + done + + #display_alert "calculate_hash_for_all_files_in_dirs" "files_to_hash_sorted: ${#files_to_hash_sorted[@]}" "warn" + #display_alert "calculate_hash_for_all_files_in_dirs" "files_to_hash_sorted: ${files_to_hash_sorted[*]}" "warn" + + calculate_hash_for_files "${files_to_hash[@]}" +} + +function calculate_hash_for_files() { + hash_files="undetermined" # outer scope + + # relativize the files to SRC + declare -a files_to_hash=("$@") + declare -a files_to_hash_relativized=() + for file in "${files_to_hash[@]}"; do + # remove the SRC/ from the file name + file="${file#${SRC}/}" + files_to_hash_relativized+=("${file}") + done + + # sort the array files_to_hash; use sort and readfile + declare -a files_to_hash_sorted + mapfile -t files_to_hash_sorted < <(for one in "${files_to_hash_relativized[@]}"; do echo "${one}"; done | LC_ALL=C sort -h) # "human" sorting + + display_alert "calculate_hash_for_files:" "files_to_hash_sorted: ${files_to_hash_sorted[*]}" "debug" + declare full_hash + full_hash="$(cd "${SRC}" && sha256sum "${files_to_hash_sorted[@]}")" + hash_files="$(echo "${full_hash}" | sha256sum | cut -d' ' -f1)" # hash of hashes + hash_files="${hash_files:0:16}" # shorten it to 16 characters + display_alert "Hash for files:" "$hash_files" "warn" + display_alert "Full hash input for files:" "\n${full_hash}" "warn" +} diff --git a/lib/functions/general/memoize-cached.sh b/lib/functions/general/memoize-cached.sh new file mode 100644 index 0000000000..463d86828c --- /dev/null +++ b/lib/functions/general/memoize-cached.sh @@ -0,0 +1,52 @@ +# This does many tricks. Beware. +# Call: +# run_memoized VAR_NAME cache_id memoized_function_name [function_args] +function run_memoized() { + declare var_n="${1}" + shift + declare cache_id="${1}" + shift + declare memoized_func="${1}" + shift + declare extra_args=("${@}") + + # shellcheck disable=SC2178 # nope, that's a nameref. + declare -n MEMO_DICT="${var_n}" # nameref + + #display_alert "memoize" "before" "info" + #debug_dict MEMO_DICT + + MEMO_DICT+=(["MEMO_TYPE"]="${cache_id}") + declare single_string_input="${cache_id}" + single_string_input="$(declare -p "${var_n}")" # this might use random order... + + MEMO_DICT+=(["MEMO_INPUT_HASH"]="$(echo "${var_n}-${single_string_input}--$(declare -f "${memoized_func}")" "${extra_args[@]}" | sha256sum | cut -f1 -d' ')") + + declare disk_cache_dir="${SRC}/cache/memoize/${MEMO_DICT[MEMO_TYPE]}" + mkdir -p "${disk_cache_dir}" + declare disk_cache_file="${disk_cache_dir}/${MEMO_DICT[MEMO_INPUT_HASH]}" + if [[ -f "${disk_cache_file}" ]]; then + # @TODO: check expiration; some stuff might want different expiration times, eg, branch vs tag vs commit + + display_alert "Using memoized ${var_n} from ${disk_cache_file}" "${MEMO_DICT[MEMO_INPUT]}" "info" + cat "${disk_cache_file}" + # shellcheck disable=SC1090 # yep, I'm sourcing the cache here. produced below. + source "${disk_cache_file}" + + #display_alert "after cache hit" "before" "info" + #debug_dict MEMO_DICT + return 0 + fi + + display_alert "Memoizing ${var_n} to ${disk_cache_file}" "${MEMO_DICT[MEMO_INPUT]}" "info" + # if cache miss, run the memoized_func... + ${memoized_func} "${var_n}" "${extra_args[@]}" + + # ... and save the output to the cache; twist declare -p's output due to the nameref + declare -p "${var_n}" | sed -e 's|^declare -A ||' > "${disk_cache_file}" + + #display_alert "after cache miss" "before" "info" + #debug_dict MEMO_DICT + + return 0 +} diff --git a/lib/functions/logging/debug-dump.sh b/lib/functions/logging/debug-dump.sh new file mode 100644 index 0000000000..76f1c73d37 --- /dev/null +++ b/lib/functions/logging/debug-dump.sh @@ -0,0 +1,13 @@ +function debug_dict() { + local dict_name="$1" + declare -n dict="${dict_name}" + for key in "${!dict[@]}"; do + debug_var "${dict_name}[${key}]" + done +} + +function debug_var() { + local varname="$1" + local -a var_val_array=("${!varname}") + display_alert "${gray_color:-}# ${yellow_color:-}${varname}${normal_color:-}=${bright_yellow_color:-}${var_val_array[*]@Q}${ansi_reset_color:-}" "" "info" +} diff --git a/lib/functions/logging/logging.sh b/lib/functions/logging/logging.sh index fba575b393..86ca0eae03 100644 --- a/lib/functions/logging/logging.sh +++ b/lib/functions/logging/logging.sh @@ -18,6 +18,7 @@ function logging_init() { declare -g bright_red_color="\e[1;31m" red_color="\e[0;31m" declare -g bright_blue_color="\e[1;34m" blue_color="\e[0;34m" declare -g bright_magenta_color="\e[1;35m" magenta_color="\e[0;35m" + declare -g bright_yellow_color="\e[1;33m" yellow_color="\e[0;33m" declare -g ansi_reset_color="\e[0m" declare -g -i logging_section_counter=0 # -i: integer declare -g tool_color="${gray_color}" # default to gray... (should be ok on terminals) diff --git a/lib/functions/main/build-packages.sh b/lib/functions/main/build-packages.sh index 5c9807705c..59935c9e37 100644 --- a/lib/functions/main/build-packages.sh +++ b/lib/functions/main/build-packages.sh @@ -38,9 +38,6 @@ function main_default_build_packages() { done fi - # Prepare ccache, cthreads, etc for the build - LOG_SECTION="prepare_compilation_vars" do_with_logging prepare_compilation_vars - if [[ "${do_build_uboot}" == "yes" ]]; then # Don't build u-boot at all if the BOOTCONFIG is 'none'. if [[ "${BOOTCONFIG}" != "none" ]]; then diff --git a/lib/functions/main/start-end.sh b/lib/functions/main/start-end.sh index 8534904cf5..23e8f40903 100644 --- a/lib/functions/main/start-end.sh +++ b/lib/functions/main/start-end.sh @@ -26,7 +26,7 @@ function main_default_start_build() { declare -g start_timestamp # global timestamp; read below by main_default_end_build() start_timestamp=$(date +%s) - ### Write config summary + ### Write config summary # @TODO: or not? this is a bit useless LOG_SECTION="config_summary" do_with_logging write_config_summary_output_file # Check and install dependencies, directory structure and settings @@ -40,6 +40,9 @@ function main_default_start_build() { ln -s "/usr/bin/python2" "${BIN_WORK_DIR}/python" export PATH="${BIN_WORK_DIR}:${PATH}" + # Prepare ccache, cthreads, etc for the build + LOG_SECTION="prepare_compilation_vars" do_with_logging prepare_compilation_vars + return 0 } diff --git a/lib/library-functions.sh b/lib/library-functions.sh index 5807ba9862..2d20b0ea84 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -1,6 +1,33 @@ #!/usr/bin/env bash # This file is/was autogenerated by lib/tools/gen-library.sh; don't modify manually +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/artifacts/artifacts-registry.sh +# shellcheck source=lib/functions/artifacts/artifacts-registry.sh +source "${SRC}"/lib/functions/artifacts/artifacts-registry.sh + +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/artifacts/kernel.sh +# shellcheck source=lib/functions/artifacts/kernel.sh +source "${SRC}"/lib/functions/artifacts/kernel.sh + +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/artifacts/u-boot.sh +# shellcheck source=lib/functions/artifacts/u-boot.sh +source "${SRC}"/lib/functions/artifacts/u-boot.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled @@ -28,6 +55,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/bsp/utils-bsp.sh source "${SRC}"/lib/functions/bsp/utils-bsp.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/cli/cli-artifact.sh +# shellcheck source=lib/functions/cli/cli-artifact.sh +source "${SRC}"/lib/functions/cli/cli-artifact.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled @@ -496,6 +532,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/general/extensions.sh source "${SRC}"/lib/functions/general/extensions.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/general/git-ref2info.sh +# shellcheck source=lib/functions/general/git-ref2info.sh +source "${SRC}"/lib/functions/general/git-ref2info.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled @@ -514,6 +559,24 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/general/github-actions.sh source "${SRC}"/lib/functions/general/github-actions.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/general/hash-files.sh +# shellcheck source=lib/functions/general/hash-files.sh +source "${SRC}"/lib/functions/general/hash-files.sh + +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/general/memoize-cached.sh +# shellcheck source=lib/functions/general/memoize-cached.sh +source "${SRC}"/lib/functions/general/memoize-cached.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled @@ -721,6 +784,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/logging/capture.sh source "${SRC}"/lib/functions/logging/capture.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/logging/debug-dump.sh +# shellcheck source=lib/functions/logging/debug-dump.sh +source "${SRC}"/lib/functions/logging/debug-dump.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled