From 091ae1d27f7238f1b59fb68f0e9b4c0367f5fcf3 Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Sun, 22 Jan 2023 16:09:38 +0100 Subject: [PATCH] armbian-next: `rootfs`: adapt to new reality after huge refactor - make `tmpfs_estimated_size` and `available_physical_memory_mib` readonly and global and only ever set once - remove subshell from `get_rootfs_cache_list()`, new version `get_rootfs_cache_list_into_array_variable()` for sanity - warns if `ROOT_FS_CREATE_ONLY=yes` is set during cached build path - add a `wait_for_disk_sync()` after restoring rootfs cache - fix logging sections for `build_rootfs_only()` and `prepare_rootfs_build_params_and_trap()` in `build_rootfs_and_image()` - remove the logging section from `get_or_create_rootfs_cache_chroot_sdcard()` call in `build_rootfs_only()` (which is now itself logged) - @TODO: `deploy_qemu_binary_to_chroot()` is leaking a qemu bin in rootfs cache! --- lib/functions/cli/cli-rootfs.sh | 1 - lib/functions/main/rootfs-image.sh | 6 +-- lib/functions/rootfs/create-cache.sh | 65 +++++++++++++++++++-------- lib/functions/rootfs/rootfs-create.sh | 2 +- lib/functions/rootfs/trap-rootfs.sh | 4 ++ 5 files changed, 54 insertions(+), 24 deletions(-) diff --git a/lib/functions/cli/cli-rootfs.sh b/lib/functions/cli/cli-rootfs.sh index 93d4097255..312c6be9e3 100644 --- a/lib/functions/cli/cli-rootfs.sh +++ b/lib/functions/cli/cli-rootfs.sh @@ -15,7 +15,6 @@ function cli_rootfs_run() { # This is run inside do_with_default_build(), above. function cli_rootfs_only_in_default_build() { - declare -i tmpfs_estimated_size # in MiB; set by prepare_rootfs_build_params_and_trap() LOG_SECTION="prepare_rootfs_build_params_and_trap" do_with_logging prepare_rootfs_build_params_and_trap LOG_SECTION="calculate_rootfs_cache_id" do_with_logging calculate_rootfs_cache_id diff --git a/lib/functions/main/rootfs-image.sh b/lib/functions/main/rootfs-image.sh index ed70c67513..18011eed65 100644 --- a/lib/functions/main/rootfs-image.sh +++ b/lib/functions/main/rootfs-image.sh @@ -4,11 +4,10 @@ function build_rootfs_and_image() { display_alert "Checking for rootfs cache" "$(echo "${BRANCH} ${BOARD} ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" | tr -s " ")" "info" [[ $ROOTFS_TYPE != ext4 ]] && display_alert "Assuming ${BOARD} ${BRANCH} kernel supports ${ROOTFS_TYPE}" "" "wrn" - declare -i tmpfs_estimated_size # in MiB; set by prepare_rootfs_build_params_and_trap() - prepare_rootfs_build_params_and_trap + LOG_SECTION="prepare_rootfs_build_params_and_trap" do_with_logging prepare_rootfs_build_params_and_trap # get a basic rootfs, either from cache or from scratch - build_rootfs_only + LOG_SECTION="build_rootfs_only" do_with_logging build_rootfs_only # only occurrence of this # stage: with a basic rootfs available, we mount the chroot and work on it mount_chroot "${SDCARD}" @@ -86,4 +85,3 @@ function list_installed_packages() { display_alert "Recording list of installed packages" "asset log" "debug" LOG_ASSET="installed_packages.txt" do_with_log_asset chroot_sdcard dpkg --get-selections "| grep -v deinstall | awk '{print \$1}' | cut -f1 -d':'" } - diff --git a/lib/functions/rootfs/create-cache.sh b/lib/functions/rootfs/create-cache.sh index 4925cb864a..f118d628e9 100644 --- a/lib/functions/rootfs/create-cache.sh +++ b/lib/functions/rootfs/create-cache.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash +# This is already run under logging, don't use do_with_logging under here. function build_rootfs_only() { # validate that tmpfs_estimated_size is set and higher than zero, or exit_with_error [[ -z ${tmpfs_estimated_size} ]] && exit_with_error "tmpfs_estimated_size is not set" [[ ${tmpfs_estimated_size} -le 0 ]] && exit_with_error "tmpfs_estimated_size is not higher than zero" # stage: prepare basic rootfs: unpack cache or create from scratch - LOG_SECTION="get_or_create_rootfs_cache_chroot_sdcard" do_with_logging get_or_create_rootfs_cache_chroot_sdcard + get_or_create_rootfs_cache_chroot_sdcard # only occurrence of this # obtain the size, in MiB, of "${SDCARD}" at this point. declare -i rootfs_size_mib @@ -23,14 +24,19 @@ function calculate_rootfs_cache_id() { # Validate that AGGREGATED_ROOTFS_HASH is set [[ -z "${AGGREGATED_ROOTFS_HASH}" ]] && exit_with_error "AGGREGATED_ROOTFS_HASH is not set at calculate_rootfs_cache_id()" - declare -g packages_hash="${AGGREGATED_ROOTFS_HASH}" # Produced by aggregation.py - currently only AGGREGATED_PACKAGES_DEBOOTSTRAP and AGGREGATED_PACKAGES_ROOTFS - declare -g -r packages_hash=${packages_hash:0:16} # it's an md5, which is 32 hex digits; used to be 8; make readonly + # If the vars are already set and not empty, exit_with_error + [[ "x${packages_hash}x" != "xx" ]] && exit_with_error "packages_hash is already set" + [[ "x${cache_type}x" != "xx" ]] && exit_with_error "cache_type is already set" - declare -g cache_type="cli" + declare -g -r packages_hash="${AGGREGATED_ROOTFS_HASH:0:16}" # Produced by aggregation.py - currently only AGGREGATED_PACKAGES_DEBOOTSTRAP and AGGREGATED_PACKAGES_ROOTFS + + declare cache_type="cli" [[ ${BUILD_DESKTOP} == yes ]] && cache_type="xfce-desktop" [[ -n ${DESKTOP_ENVIRONMENT} ]] && cache_type="${DESKTOP_ENVIRONMENT}" [[ ${BUILD_MINIMAL} == yes ]] && cache_type="minimal" declare -g -r cache_type="${cache_type}" + + display_alert "calculate_rootfs_cache_id: done with packages-hash" "${packages_hash}" "warn" } # this gets from cache or produces a basic new rootfs, ready, but not mounted, at "$SDCARD" @@ -68,19 +74,26 @@ function get_or_create_rootfs_cache_chroot_sdcard() { display_alert "ROOTFSCACHE_VERSION found online or preset" "${ROOTFSCACHE_VERSION}" "warn" calculate_rootfs_cache_id # this sets packages_hash and cache_type + # seek last cache, proceed to previous otherwise build it - local cache_list - readarray -t cache_list <<< "$(get_rootfs_cache_list "$cache_type" "$packages_hash" | sort -r)" + local -a cache_list=() + get_rootfs_cache_list_into_array_variable # sets cache_list + + # Show the number of items in the cache_list array + display_alert "Found possible rootfs caches: " "${#cache_list[@]}" "warn" display_alert "ROOTFSCACHE_VERSION after getting cache list" "${ROOTFSCACHE_VERSION}" "warn" - declare possible_cached_version # @TODO: does this kill the above var? + declare possible_cached_version for possible_cached_version in "${cache_list[@]}"; do ROOTFSCACHE_VERSION="${possible_cached_version}" # global var local cache_name="${ARCH}-${RELEASE}-${cache_type}-${packages_hash}-${ROOTFSCACHE_VERSION}.tar.zst" local cache_fname="${SRC}/cache/rootfs/${cache_name}" - [[ "$ROOT_FS_CREATE_ONLY" == yes ]] && break + if [[ "$ROOT_FS_CREATE_ONLY" == yes ]]; then + display_alert "Using deprecated" "ROOT_FS_CREATE_ONLY=yes during search for existing cache" "warn" + break + fi display_alert "Checking cache" "$cache_name" "info" @@ -104,15 +117,18 @@ function get_or_create_rootfs_cache_chroot_sdcard() { # if aria2 file exists, download didn't succeeded, so skip it # @TODO this could be named IGNORE_EXISTING_ROOTFS_CACHE=yes if [[ "${ROOT_FS_CREATE_ONLY}" != "yes" && -f "${cache_fname}" && ! -f "${cache_fname}.aria2" ]]; then - [[ ! -d "${SDCARD:?}" ]] || exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: ${SDCARD} is not a directory" + # validate sanity + [[ "x${SDCARD}x" == "xx" ]] && exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: SDCARD: ${SDCARD} is not set" local date_diff=$((($(date +%s) - $(stat -c %Y "${cache_fname}")) / 86400)) display_alert "Extracting $cache_name" "$date_diff days old" "info" pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "extract_rootfs") $cache_name" "$cache_fname" | zstdmt -dc | tar xp --xattrs -C "${SDCARD}"/ - # @TODO: this never runs, since 'set -e' ("errexit") is in effect, and https://github.com/koalaman/shellcheck/wiki/SC2181 # [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Restart." + #echo >&2 # newline to stderr after using pv? + wait_for_disk_sync "after restoring rootfs cache" + run_host_command_logged rm -v "${SDCARD}"/etc/resolv.conf run_host_command_logged echo "nameserver ${NAMESERVER}" ">" "${SDCARD}"/etc/resolv.conf @@ -155,15 +171,18 @@ function create_new_rootfs_cache() { return 0 # protect against possible future short-circuiting above this } -# get_rootfs_cache_list -# -# return a list of versions of all avaiable cache from remote and local. -function get_rootfs_cache_list() { - local cache_type=$1 - local packages_hash=$2 +# return a list of versions of all available cache from remote and local into outer scoe "cache_list" variable +function get_rootfs_cache_list_into_array_variable() { + # If global vars are empty, exit_with_error + [[ "x${ARCH}x" == "xx" ]] && exit_with_error "ARCH is not set" + [[ "x${RELEASE}x" == "xx" ]] && exit_with_error "RELEASE is not set" + [[ "x${packages_hash}x" == "xx" ]] && exit_with_error "packages_hash is not set" + [[ "x${cache_type}x" == "xx" ]] && exit_with_error "cache_type is not set" # this uses `jq` hostdep - { + + declare -a local_cache_list=() # outer scope variable + readarray -t local_cache_list <<< "$({ # Don't even try remote if we're told to skip. if [[ "${SKIP_ARMBIAN_REPO}" != "yes" ]]; then curl --silent --fail -L "https://api.github.com/repos/armbian/cache/releases?per_page=3" | jq -r '.[].tag_name' || @@ -174,5 +193,15 @@ function get_rootfs_cache_list() { sed -e 's#^.*/##' | sed -e 's#\..*$##' | awk -F'-' '{print $5}' - } | sort | uniq + } | sort | uniq | sort -r)" + + # Show the contents + display_alert "Available cache versions number" "${#local_cache_list[*]}" "warn" + # Loop each and show + for cache_version in "${local_cache_list[@]}"; do + display_alert "One available cache version" "${cache_version}" "warn" + done + + # return the list to outer scope + cache_list=("${local_cache_list[@]}") } diff --git a/lib/functions/rootfs/rootfs-create.sh b/lib/functions/rootfs/rootfs-create.sh index cadf91cfe9..641182cd15 100644 --- a/lib/functions/rootfs/rootfs-create.sh +++ b/lib/functions/rootfs/rootfs-create.sh @@ -83,7 +83,7 @@ function create_new_rootfs_cache_via_debootstrap() { local_apt_deb_cache_prepare "after debootstrap" # just for size reference in logs - deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() + deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() @TODO: which is too late for a cache display_alert "Installing base system" "Stage 2/2" "info" export if_error_detail_message="Debootstrap second stage failed ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" diff --git a/lib/functions/rootfs/trap-rootfs.sh b/lib/functions/rootfs/trap-rootfs.sh index 98ead6e95b..27ac902916 100644 --- a/lib/functions/rootfs/trap-rootfs.sh +++ b/lib/functions/rootfs/trap-rootfs.sh @@ -35,6 +35,10 @@ function prepare_rootfs_build_params_and_trap() { display_alert "Not using tmpfs for rootfs" "RAM available: ${available_physical_memory_mib}MiB < ${tmpfs_estimated_size}MiB estimated" "info" fi + # make global and readonly, for sanity + declare -g -r -i tmpfs_estimated_size="${tmpfs_estimated_size}" + declare -g -r -i available_physical_memory_mib="${available_physical_memory_mib}" + if [[ $use_tmpfs == yes ]]; then declare -g -r ROOTFS_IS_UNDER_TMPFS=yes mount -t tmpfs tmpfs "${SDCARD}" # do not specify size; we've calculated above that it should fit, and Linux will try its best if it doesn't.