From 00bc1b4a1e405791814350cc036efa4bf5fa368b Mon Sep 17 00:00:00 2001 From: Igor Date: Tue, 24 Jan 2023 13:02:59 +0100 Subject: [PATCH] armbian-next: rootfs: introduce `ROOTFS_COMPRESSION_RATIO`, default 5 for general use, default 15 for `rootfs` CLI; fix pv | tar bug; remove GPG signing - Remove PGP signing from the script as its done in GHA - Also remove generating .current control file as its obsolete in this design Co-authored-by: Ricardo Pardini --- lib/functions/cli/cli-rootfs.sh | 2 ++ lib/functions/cli/commands.sh | 1 - lib/functions/rootfs/rootfs-create.sh | 32 ++++++++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/functions/cli/cli-rootfs.sh b/lib/functions/cli/cli-rootfs.sh index 55b6473f1e..f3921900a4 100644 --- a/lib/functions/cli/cli-rootfs.sh +++ b/lib/functions/cli/cli-rootfs.sh @@ -6,6 +6,8 @@ function cli_rootfs_pre_run() { } function cli_rootfs_run() { + declare -g ROOTFS_COMPRESSION_RATIO="${ROOTFS_COMPRESSION_RATIO:-"15"}" # default to Compress stronger when we make rootfs cache + declare -a vars_cant_be_set=("LINUXFAMILY" "BOARDFAMILY") # loop through all vars and check if they are set and bomb out for var in "${vars_cant_be_set[@]}"; do diff --git a/lib/functions/cli/commands.sh b/lib/functions/cli/commands.sh index 10ee06aae8..b37675b9ff 100644 --- a/lib/functions/cli/commands.sh +++ b/lib/functions/cli/commands.sh @@ -23,7 +23,6 @@ function armbian_register_commands() { ["distccd"]="distccd" # implemented in cli_distccd_pre_run and cli_distccd_run ["rootfs"]="rootfs" # implemented in cli_rootfs_pre_run and cli_rootfs_run - ["rootfs-cache"]="rootfs" # idem, alias # shortcuts, see vars set below. the use legacy single build, and try to control it via variables ["kernel"]="standard_build" diff --git a/lib/functions/rootfs/rootfs-create.sh b/lib/functions/rootfs/rootfs-create.sh index 641182cd15..df9d681986 100644 --- a/lib/functions/rootfs/rootfs-create.sh +++ b/lib/functions/rootfs/rootfs-create.sh @@ -1,14 +1,31 @@ function create_new_rootfs_cache_tarball() { + # validate cache_fname is set + [[ -n "${cache_fname}" ]] || exit_with_error "create_new_rootfs_cache_tarball: cache_fname is not set" + # validate SDCARD is set + [[ -n "${SDCARD}" ]] || exit_with_error "create_new_rootfs_cache_tarball: SDCARD is not set" + # validate cache_name is set + [[ -n "${cache_name}" ]] || exit_with_error "create_new_rootfs_cache_tarball: cache_name is not set" + # create list of installed packages for debug purposes - this captures it's own stdout. # @TODO: sanity check, compare this with the source of the hash coming from aggregation chroot_sdcard "dpkg -l | grep ^ii | awk '{ print \$2\",\"\$3 }'" > "${cache_fname}.list" echo "${AGGREGATED_ROOTFS_HASH_TEXT}" > "${cache_fname}.hash_text" - display_alert "zstd tarball of rootfs" "${RELEASE}:: ${cache_name}" "debug" + declare compression_ratio_rootfs="${ROOTFS_COMPRESSION_RATIO:-"5"}" + + display_alert "zstd tarball of rootfs" "${RELEASE}:: ${cache_name} :: compression ${compression_ratio_rootfs}" "info" tar cp --xattrs --directory="$SDCARD"/ --exclude='./dev/*' --exclude='./proc/*' --exclude='./run/*' \ --exclude='./tmp/*' --exclude='./sys/*' --exclude='./home/*' --exclude='./root/*' . | pv -p -b -r -s "$(du -sb "$SDCARD"/ | cut -f1)" -N "$(logging_echo_prefix_for_pv "store_rootfs") $cache_name" | - zstdmt -5 -c > "${cache_fname}" + zstdmt "-${compression_ratio_rootfs}" -c > "${cache_fname}" + + declare -a pv_tar_zstdmt_pipe_status=("${PIPESTATUS[@]}") # capture and the pipe_status array from PIPESTATUS + declare one_pipe_status + for one_pipe_status in "${pv_tar_zstdmt_pipe_status[@]}"; do + if [[ "$one_pipe_status" != "0" ]]; then + exit_with_error "create_new_rootfs_cache_tarball: compress: ${cache_fname} failed (${pv_tar_zstdmt_pipe_status[*]}) - out of disk space?" + fi + done wait_for_disk_sync "after zstd tarball rootfs" @@ -16,17 +33,6 @@ function create_new_rootfs_cache_tarball() { local cache_size cache_size=$(du -sh "${cache_fname}" | cut -f1) - # sign rootfs cache archive that it can be used for web cache once. Internal purposes - if [[ -n "${GPG_PASS}" && "${SUDO_USER}" ]]; then - display_alert "Using, does nothing" "GPG_PASS and SUDO_USER" "warn" - # @TODO: rpardini: igor is this still needed? I see the GHA scripts does its own signing? - #[[ -n ${SUDO_USER} ]] && sudo chown -R "${SUDO_USER}:${SUDO_USER}" "${DEST}"/images/ - #echo "${GPG_PASS}" | sudo -H -u "${SUDO_USER}" bash -c "gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes ${cache_fname}" || exit 1 - fi - - # needed for backend to keep current only @TODO: say that again? what backend? - echo "$cache_fname" > "${cache_fname}.current" - display_alert "rootfs cache created" "${cache_fname} [${cache_size}]" "info" }