From fef862f774fcb4578409f5be2c7114ccd1043e96 Mon Sep 17 00:00:00 2001 From: tabris Date: Wed, 22 Oct 2025 20:33:20 -0400 Subject: [PATCH] build framework mmdebstrap - LEGACY_DEBOOTSTRAP allow old debootstrap to be used instead. --- lib/functions/artifacts/artifact-rootfs.sh | 3 +- lib/functions/rootfs/distro-specific.sh | 2 - lib/functions/rootfs/rootfs-create.sh | 139 +++++++++++++++------ 3 files changed, 105 insertions(+), 39 deletions(-) diff --git a/lib/functions/artifacts/artifact-rootfs.sh b/lib/functions/artifacts/artifact-rootfs.sh index 5548723c09..50ffc48b4a 100644 --- a/lib/functions/artifacts/artifact-rootfs.sh +++ b/lib/functions/artifacts/artifact-rootfs.sh @@ -15,9 +15,10 @@ function artifact_rootfs_config_dump() { artifact_input_variables[DESKTOP_ENVIRONMENT]="${DESKTOP_ENVIRONMENT:-"no_DESKTOP_ENVIRONMENT_set"}" artifact_input_variables[DESKTOP_ENVIRONMENT_CONFIG_NAME]="${DESKTOP_ENVIRONMENT_CONFIG_NAME:-"no_DESKTOP_ENVIRONMENT_CONFIG_NAME_set"}" artifact_input_variables[DESKTOP_APPGROUPS_SELECTED]="${DESKTOP_APPGROUPS_SELECTED:-"no_DESKTOP_APPGROUPS_SELECTED_set"}" + artifact_input_variables[LEGACY_DEBOOTSTRAP]="${LEGACY_DEBOOTSTRAP:-"no"}" # Hash of the packages added/removed by extensions declare pkgs_hash="undetermined" - pkgs_hash="$(echo "${REMOVE_PACKAGES[*]} ${EXTRA_PACKAGES_ROOTFS[*]} ${PACKAGE_LIST_BOARD_REMOVE} ${PACKAGE_LIST_FAMILY_REMOVE}" | sha256sum | cut -d' ' -f1)" + pkgs_hash="$(echo "${REMOVE_PACKAGES[*]} ${EXTRA_PACKAGES_ROOTFS[*]} ${PACKAGE_LIST_BOARD_REMOVE} ${PACKAGE_LIST_FAMILY_REMOVE} ${LEGACY_DEBOOTSTRAP}" | sha256sum | cut -d' ' -f1)" artifact_input_variables[EXTRA_PKG_ADD_REMOVE_HASH]="${pkgs_hash}" } diff --git a/lib/functions/rootfs/distro-specific.sh b/lib/functions/rootfs/distro-specific.sh index b2c4cb5121..e68bebccd8 100644 --- a/lib/functions/rootfs/distro-specific.sh +++ b/lib/functions/rootfs/distro-specific.sh @@ -91,7 +91,6 @@ function fetch_distro_keyring() { debian) if [ -e "${CACHEDIR}/debian-archive-keyring.gpg" ]; then display_alert "fetch_distro_keyring($release)" "cache found, skipping" "info" - /bin/true; else BASEURI='https://deb.debian.org/debian/pool/main/d/debian-archive-keyring/' #FIXME: write something to retrieve newest @@ -119,7 +118,6 @@ function fetch_distro_keyring() { ubuntu) if [ -e "${CACHEDIR}/ubuntu-archive-keyring.gpg" ]; then display_alert "fetch_distro_keyring($release)" "cache found, skipping" "info" - /bin/true; else BASEURI='https://archive.ubuntu.com/ubuntu/pool/main/u/ubuntu-keyring/' #FIXME: write something to retrieve newest diff --git a/lib/functions/rootfs/rootfs-create.sh b/lib/functions/rootfs/rootfs-create.sh index 9934cff2c0..9606c969de 100644 --- a/lib/functions/rootfs/rootfs-create.sh +++ b/lib/functions/rootfs/rootfs-create.sh @@ -46,11 +46,28 @@ function create_new_rootfs_cache_tarball() { } # create_new_rootfs_cache_via_debootstrap populates a root FS into -# SDCARD using mmdebstrap, configures locales and apt sources, installs -# additional packages (and optionally desktop packages), performs chroot -# setup and cleanup (policy diverts, qemu-binfmt handling, resolvconf, -# machine-id masking), and prepares the rootfs for packaging & caching. +# SDCARD using mmdebstrap [or debootstrap if LEGACY_DEBOOTSTRAP], +# configures locales and apt sources, installs additional packages (and +# optionally desktop packages), performs chroot setup and cleanup +# (policy diverts, qemu-binfmt handling, resolvconf, machine-id +# masking), and prepares the rootfs for packaging & caching. function create_new_rootfs_cache_via_debootstrap() { + LOG_NAME="" + case "${LEGACY_DEBOOTSTRAP,,}" in + "yes") + LOG_NAME="debootstrap" + ;; + "no") + LOG_NAME="mmdebstrap" + ;; + "") + LOG_NAME="mmdebstrap" + LEGACY_DEBOOTSTRAP="no" + ;; + *) + exit_with_error "invalid value for LEGACY_DEBOOTSTRAP: ${LEGACY_DEBOOTSTRAP}" + ;; + esac [[ ! -d "${SDCARD:?}" ]] && exit_with_error "create_new_rootfs_cache_via_debootstrap: ${SDCARD} is not a directory" # this is different between debootstrap and regular apt-get; here we use acng as a prefix to the real repo @@ -76,37 +93,64 @@ function create_new_rootfs_cache_via_debootstrap() { # @TODO: one day: https://gitlab.mister-muffin.de/josch/mmdebstrap/src/branch/main/mmdebstrap - # Obtain the latest debootstrap (which is just a shell script) from mister-muffin or Ubuntu's git + # Obtain the latest debootstrap/mmdebstrap (which is just a shell script) from mister-muffin or Ubuntu's git declare debootstrap_bin="" debootstrap_version="" debootstrap_wanted_dir="" debootstrap_default_script="" - display_alert "Preparing debootstrap" "for ${DISTRIBUTION}'s ${RELEASE}" "info" + display_alert "Preparing ${LOG_NAME}" "for ${DISTRIBUTION}'s ${RELEASE}" "info" case "${DISTRIBUTION}" in Ubuntu) - export GIT_FIXED_WORKDIR="mmdebstrap-ubuntu-devel" - #FIXME: branch should be a variable eventually - fetch_from_repo "https://git.launchpad.net/ubuntu/+source/mmdebstrap" "${GIT_FIXED_WORKDIR}" "branch:ubuntu/noble" - debootstrap_wanted_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" - debootstrap_default_script="gutsy" - debootstrap_version="$(sed 's/.*(\(.*\)).*/\1/; q' "${debootstrap_wanted_dir}/debian/changelog")" + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "yes" ]]; then + export GIT_FIXED_WORKDIR="debootstrap-ubuntu-devel" + fetch_from_repo "https://git.launchpad.net/ubuntu/+source/debootstrap" "debootstrap-ubuntu-devel" "tag:import/1.0.118ubuntu1.13" + debootstrap_wanted_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" + debootstrap_default_script="gutsy" + debootstrap_version="$(sed 's/.*(\(.*\)).*/\1/; q' "${debootstrap_wanted_dir}/debian/changelog")" + debootstrap_bin="${debootstrap_wanted_dir}/debootstrap" + else + export GIT_FIXED_WORKDIR="mmdebstrap-ubuntu-devel" + #FIXME: branch should be a variable eventually + fetch_from_repo "https://git.launchpad.net/ubuntu/+source/mmdebstrap" "${GIT_FIXED_WORKDIR}" "branch:ubuntu/noble" + debootstrap_wanted_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" + debootstrap_version="$(sed 's/.*(\(.*\)).*/\1/; q' "${debootstrap_wanted_dir}/debian/changelog")" + debootstrap_bin="${debootstrap_wanted_dir}/mmdebstrap" + fi ;; Debian) - export GIT_FIXED_WORKDIR="mmdebstrap-debian-devel" - #FIXME: branch should be a variable eventually - fetch_from_repo "https://gitlab.mister-muffin.de/josch/mmdebstrap" "${GIT_FIXED_WORKDIR}" "branch:main" - debootstrap_wanted_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" - debootstrap_default_script="sid" - debootstrap_version="$(sed 's/^## \[\([^]]*\)\].*/\1/; q' "${debootstrap_wanted_dir}/CHANGELOG.md")" + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "yes" ]]; then + export GIT_FIXED_WORKDIR="debootstrap-debian-devel" + fetch_from_repo "https://salsa.debian.org/installer-team/debootstrap.git" "debootstrap-debian-devel" "branch:master" + debootstrap_wanted_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" + debootstrap_default_script="sid" + debootstrap_version="$(sed 's/.*(\(.*\)).*/\1/; q' "${debootstrap_wanted_dir}/debian/changelog")" + debootstrap_bin="${debootstrap_wanted_dir}/debootstrap" + else + export GIT_FIXED_WORKDIR="mmdebstrap-debian-devel" + #FIXME: branch should be a variable eventually + fetch_from_repo "https://gitlab.mister-muffin.de/josch/mmdebstrap" "${GIT_FIXED_WORKDIR}" "branch:main" + debootstrap_wanted_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" + debootstrap_default_script="sid" + debootstrap_version="$(sed 's/^## \[\([^]]*\)\].*/\1/; q' "${debootstrap_wanted_dir}/CHANGELOG.md")" + debootstrap_bin="${debootstrap_wanted_dir}/mmdebstrap" + fi ;; *) - exit_with_error "Unknown distribution for debootstrap" "${DISTRIBUTION}" + exit_with_error "Unknown distribution for ${LOG_NAME}" "${DISTRIBUTION}" ;; esac - debootstrap_bin="${debootstrap_wanted_dir}/mmdebstrap" run_host_command_logged chmod a+x "${debootstrap_bin}" - display_alert "Debootstrap version" "'${debootstrap_version}' for ${debootstrap_bin}" "info" + display_alert "${LOG_NAME} version" "'${debootstrap_version}' for ${debootstrap_bin}" "info" - display_alert "Installing base system with ${#AGGREGATED_PACKAGES_DEBOOTSTRAP[@]} packages" "Stage 1/1" "info" + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "yes" ]]; then + # check if the debootstrap has the scripts/${RELEASE} script present, otherwise symlink it to debootstrap_default_script + if [[ ! -f "${debootstrap_wanted_dir}/scripts/${RELEASE}" ]]; then + display_alert "Symlinking" "debootstrap scripts/${RELEASE} to scripts/${debootstrap_default_script}" "info" + run_host_command_logged ln -sv "${debootstrap_wanted_dir}/scripts/${debootstrap_default_script}" "${debootstrap_wanted_dir}/scripts/${RELEASE}" + fi + display_alert "Installing base system with ${#AGGREGATED_PACKAGES_DEBOOTSTRAP[@]} packages" "Stage 1/2" "info" + else + display_alert "Installing base system with ${#AGGREGATED_PACKAGES_DEBOOTSTRAP[@]} packages" "Stage 1/1" "info" + fi cd "${SDCARD}" || exit_with_error "cray-cray about SDCARD" "${SDCARD}" # this will prevent error sh: 0: getcwd() failed declare -ga debootstrap_arguments=( @@ -114,37 +158,60 @@ function create_new_rootfs_cache_via_debootstrap() { "--arch=${ARCH}" # the arch "'--include=${AGGREGATED_PACKAGES_DEBOOTSTRAP_COMMA}'" # from aggregation.py "'--components=${AGGREGATED_DEBOOTSTRAP_COMPONENTS_COMMA}'" # from aggregation.py - "'--skip=check/empty'" # skips check if the rootfs dir is empty at start ) - fetch_distro_keyring "$RELEASE" + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "no" ]]; then + debootstrap_arguments+=("'--skip=check/empty'") # skips check if the rootfs dir is empty at start + fetch_distro_keyring "$RELEASE" + fi # This is necessary to debootstrap from a non-official repo [[ $ARCH == loong64 ]] && debootstrap_arguments+=("--keyring=/usr/share/keyrings/debian-ports-archive-keyring.gpg") # Small detour for local apt caching option. - local_apt_deb_cache_prepare "before debootstrap" # sets LOCAL_APT_CACHE_INFO + local_apt_deb_cache_prepare "before ${LOG_NAME}" # sets LOCAL_APT_CACHE_INFO if [[ "${LOCAL_APT_CACHE_INFO[USE]}" == "yes" ]]; then - debootstrap_arguments+=("--setup-hook='mkdir -p ${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]} \"\$1\"/var/cache/apt/archives/'") - debootstrap_arguments+=("--setup-hook='sync-in ${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]} /var/cache/apt/archives/'") - debootstrap_arguments+=("--customize-hook='sync-out /var/cache/apt/archives/ ${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]}'") + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "no" ]]; then + debootstrap_arguments+=("--setup-hook='mkdir -p ${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]} \"\$1\"/var/cache/apt/archives/'") + debootstrap_arguments+=("--setup-hook='sync-in ${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]} /var/cache/apt/archives/'") + debootstrap_arguments+=("--customize-hook='sync-out /var/cache/apt/archives/ ${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]}'") + else + debootstrap_arguments+=("--cache-dir=${LOCAL_APT_CACHE_INFO[HOST_DEBOOTSTRAP_CACHE_DIR]}") # cache .deb's used + debootstrap_arguments+=("--foreign") # prefixed before release name + fi fi debootstrap_arguments+=("${RELEASE}" "${SDCARD}/" "${debootstrap_apt_mirror}") # release, path and mirror; always last, positional arguments. - run_host_command_logged "${debootstrap_bin}" "${debootstrap_arguments[@]}" || { - exit_with_error "mmdebstrap failed" "${debootstrap_bin} ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" - } + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "no" ]]; then + run_host_command_logged "${debootstrap_bin}" "${debootstrap_arguments[@]}" || { + exit_with_error "${LOG_NAME} failed" "${debootstrap_bin} ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" + } + else + # Set DEBOOTSTRAP_DIR only for this invocation + export DEBOOTSTRAP_DIR="${debootstrap_wanted_dir}" + run_host_command_logged "${debootstrap_bin}" "${debootstrap_arguments[@]}" || { + exit_with_error "Debootstrap first stage failed" "${debootstrap_bin} ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" + } + export -n DEBOOTSTRAP_DIR + [[ ! -f ${SDCARD}/debootstrap/debootstrap ]] && exit_with_error "Debootstrap first stage did not produce marker file" + fi - skip_target_check="yes" local_apt_deb_cache_prepare "for mmdebstrap" # just for size reference in logs + skip_target_check="yes" local_apt_deb_cache_prepare "for ${LOG_NAME}" # just for size reference in logs deploy_qemu_binary_to_chroot "${SDCARD}" "rootfs" # undeployed near the end of this function - [[ ! -f "${SDCARD}/bin/bash" ]] && exit_with_error "mmdebstrap did not produce /bin/bash" + if [[ "${LEGACY_DEBOOTSTRAP,,}" == "yes" ]]; then + display_alert "Installing base system" "Stage 2/2" "info" + declare -g -a if_error_find_files_sdcard=("debootstrap.log") # if command fails, go look for this file and show it's contents during error processing + declare -g if_error_detail_message="Debootstrap second stage failed ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" + chroot_sdcard LC_ALL=C LANG=C /debootstrap/debootstrap --second-stage + fi + [[ ! -f "${SDCARD}/bin/bash" ]] && exit_with_error "${LOG_NAME} did not produce /bin/bash" # Done with mmdebstrap. Clean-up its litterbox. - display_alert "Cleaning up after mmdebstrap" "mmdebstrap cleanup" "info" + display_alert "Cleaning up after ${LOG_NAME}" "${LOG_NAME} cleanup" "info" run_host_command_logged rm -rf "${SDCARD}/var/cache/apt" "${SDCARD}/var/lib/apt/lists" - local_apt_deb_cache_prepare "after mmdebstrap cleanup" # just for size reference in logs + local_apt_deb_cache_prepare "after ${LOG_NAME} cleanup" # just for size reference in logs mount_chroot "${SDCARD}" # we mount the chroot here... it's un-mounted below when all is done, or by cleanup handler '' @TODO @@ -285,7 +352,7 @@ function create_new_rootfs_cache_via_debootstrap() { undeploy_qemu_binary_from_chroot "${SDCARD}" "rootfs" # stage: make rootfs cache archive - display_alert "Ending mmdebstrap process and preparing cache" "$RELEASE" "info" + display_alert "Ending ${LOG_NAME} process and preparing cache" "$RELEASE" "info" wait_for_disk_sync "before tar rootfs" # we're done with using the chroot which we mounted above.