From 6755e9190a970893b6ceecbe211aed6b4f6cf102 Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Sat, 28 Dec 2024 15:15:45 +0100 Subject: [PATCH] don't ship qemu binary in rootfs cache + armhf image/rootfs building on non-armhf-capable-arm64 hosts (Apple M1+) - refactor `prepare_host_binfmt_qemu()` out of `prepare_host_noninteractive()` and into `rootfs/qemu-static.sh` - further split into more functions and return early to avoid deep nesting - implement force import and load of qemu-arm for non-armhf capable arm64 hosts (incl magic numbers) - enhance `deploy_qemu_binary_to_chroot()` & `undeploy_qemu_binary_from_chroot()`; - add 2nd param "caller" for better logging/tracking - does sanity-check and preserve existing binary if it exists - explicitly deploy/undeploy for the 3 cases: - image: moved undeploy from `post_debootstrap_tweaks()` into image build proper for consistency - rootfs: was leaving trash behind (since post_debootstrap_tweaks never ran for rootfs), now properly undeploys - initrd: was already fine, just added caller info - added `arch-test` host dependency - ensure `arch-test ${ARCH}` works during prepare-host - > tl,dr: "can build 32-bit armv7 armhf using Apple silicon; can use rootfs cache cross-arch reliably" --- lib/functions/host/prepare-host.sh | 77 +--------- lib/functions/image/initrd.sh | 4 +- lib/functions/main/rootfs-image.sh | 6 + lib/functions/rootfs/post-tweaks.sh | 6 +- lib/functions/rootfs/qemu-static.sh | 197 +++++++++++++++++++++++--- lib/functions/rootfs/rootfs-create.sh | 5 +- 6 files changed, 192 insertions(+), 103 deletions(-) diff --git a/lib/functions/host/prepare-host.sh b/lib/functions/host/prepare-host.sh index 1232bdbc1c..e8246cafbc 100644 --- a/lib/functions/host/prepare-host.sh +++ b/lib/functions/host/prepare-host.sh @@ -117,80 +117,7 @@ function prepare_host_noninteractive() { download_external_toolchains # Mostly deprecated, since SKIP_EXTERNAL_TOOLCHAINS=yes is the default fi - # NEEDS_BINFMT=yes is set by default build and rootfs artifact build. - # if we're building an image, not only packages/artifacts... - # ... and the host arch does not match the target arch ... - # ... we then require binfmt_misc to be enabled. - # "enable arm binary format so that the cross-architecture chroot environment will work" - if [[ "${NEEDS_BINFMT:-"no"}" == "yes" ]]; then - - if [[ "${SHOW_DEBUG}" == "yes" ]]; then - display_alert "Debugging binfmt - early" "/proc/sys/fs/binfmt_misc/" "debug" - run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true - fi - - if dpkg-architecture -e "${ARCH}"; then - display_alert "Native arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "cachehit" - else - local failed_binfmt_modprobe=0 - - display_alert "Cross arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "debug" - - # Check if binfmt_misc is loaded; if not, try to load it, but don't fail: it might be built in. - if grep -q "^binfmt_misc" /proc/modules; then - display_alert "binfmt_misc is already loaded" "binfmt_misc already loaded" "debug" - else - display_alert "binfmt_misc is not loaded" "trying to load binfmt_misc" "debug" - - # try to modprobe. if it fails, emit a warning later, but not here. - # this is for the in-container case, where the host already has the module, but won't let the container know about it. - modprobe -q binfmt_misc || failed_binfmt_modprobe=1 - fi - - # Now, /proc/sys/fs/binfmt_misc/ has to be mounted. Mount, or fail with a message - if mountpoint -q /proc/sys/fs/binfmt_misc/; then - display_alert "binfmt_misc is already mounted" "binfmt_misc already mounted" "debug" - else - display_alert "binfmt_misc is not mounted" "trying to mount binfmt_misc" "debug" - mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ || { - if [[ $failed_binfmt_modprobe == 1 ]]; then - display_alert "Failed to load binfmt_misc module" "modprobe binfmt_misc failed" "wrn" - fi - display_alert "Check your HOST kernel" "CONFIG_BINFMT_MISC=m is required in host kernel" "warn" - display_alert "Failed to mount" "binfmt_misc /proc/sys/fs/binfmt_misc/" "err" - exit_with_error "Failed to mount binfmt_misc" - } - display_alert "binfmt_misc mounted" "binfmt_misc mounted" "debug" - fi - - declare host_arch - host_arch="$(arch)" - local -a wanted_arches=("arm" "aarch64" "x86_64" "riscv64") - display_alert "Preparing binfmts for arch" "binfmts: host '${host_arch}', wanted arches '${wanted_arches[*]}'" "debug" - declare wanted_arch - for wanted_arch in "${wanted_arches[@]}"; do - if [[ "${host_arch}" != "${wanted_arch}" ]]; then - if [[ ! -e "/proc/sys/fs/binfmt_misc/qemu-${wanted_arch}" ]]; then - display_alert "Updating binfmts" "update-binfmts --enable qemu-${wanted_arch}" "debug" - if [[ "${host_arch}" == "aarch64" && "${wanted_arch}" == "arm" ]]; then - display_alert "Trying to update binfmts - aarch64 (sometimes) does 32-bit sans emulation" "update-binfmts --enable qemu-${wanted_arch}" "debug" - run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading. - else - run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" || display_alert "Failed to update binfmts" "update-binfmts --enable qemu-${wanted_arch}" "err" # log & continue on failure - fi - fi - fi - done - - # @TODO: we could create a tiny loop here to test if the binfmt_misc is working, but this is before deps are installed. - fi - - if [[ "${SHOW_DEBUG}" == "yes" ]]; then - display_alert "Debugging binfmt - late" "/proc/sys/fs/binfmt_misc/" "debug" - run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true - fi - - fi + prepare_host_binfmt_qemu # in qemu-static.sh as most binfmt/qemu logic is there now # @TODO: rpardini: this does not belong here, instead with the other templates, pre-configuration. [[ ! -f "${USERPATCHES_PATH}"/customize-image.sh ]] && run_host_command_logged cp -pv "${SRC}"/config/templates/customize-image.sh.template "${USERPATCHES_PATH}"/customize-image.sh @@ -267,7 +194,7 @@ function adaptative_prepare_host_dependencies() { ncurses-base ncurses-term # for `make menuconfig` ntpdate patchutils pkg-config pv - qemu-user-static + "qemu-user-static" "arch-test" rsync swig # swig is needed for some u-boot's. example: "bananapi.conf" u-boot-tools diff --git a/lib/functions/image/initrd.sh b/lib/functions/image/initrd.sh index dcd3278905..476ae84173 100644 --- a/lib/functions/image/initrd.sh +++ b/lib/functions/image/initrd.sh @@ -75,7 +75,7 @@ update_initramfs() { display_alert "initrd cache hash" "${initrd_hash}" "debug" display_alert "Mounting chroot for update-initramfs" "update-initramfs" "debug" - deploy_qemu_binary_to_chroot "${chroot_target}" + deploy_qemu_binary_to_chroot "${chroot_target}" "initrd"# is undeployed at the end of this function mount_chroot "$chroot_target/" @@ -125,7 +125,7 @@ update_initramfs() { display_alert "Unmounting chroot" "update-initramfs" "debug" umount_chroot "${chroot_target}/" - undeploy_qemu_binary_from_chroot "${chroot_target}" + undeploy_qemu_binary_from_chroot "${chroot_target}" "initrd" # deployed at the start of this function # no need to remove ${initrd_cache_current_manifest_filepath} manually, since it's under ${WORKDIR} return 0 # avoid future short-circuit problems diff --git a/lib/functions/main/rootfs-image.sh b/lib/functions/main/rootfs-image.sh index 2ee3cd93fa..89f1bd2b6e 100644 --- a/lib/functions/main/rootfs-image.sh +++ b/lib/functions/main/rootfs-image.sh @@ -13,6 +13,9 @@ function build_rootfs_and_image() { # get a basic rootfs, either from cache or from scratch get_or_create_rootfs_cache_chroot_sdcard # only occurrence of this; has its own logging sections + # deploy the qemu binary, no matter where the rootfs came from (built or cached) + LOG_SECTION="deploy_qemu_binary_to_chroot_image" do_with_logging deploy_qemu_binary_to_chroot "${SDCARD}" "image" # undeployed at end of this function + # stage: with a basic rootfs available, we mount the chroot and work on it LOG_SECTION="mount_chroot_sdcard" do_with_logging mount_chroot "${SDCARD}" @@ -66,6 +69,9 @@ function build_rootfs_and_image() { LOG_SECTION="post_debootstrap_tweaks" do_with_logging post_debootstrap_tweaks + # undeploy the qemu binary from the image; we don't want to ship the host's qemu in the target image + LOG_SECTION="undeploy_qemu_binary_from_chroot_image" do_with_logging undeploy_qemu_binary_from_chroot "${SDCARD}" "image" + # clean up / prepare for making the image LOG_SECTION="umount_chroot_sdcard" do_with_logging umount_chroot "${SDCARD}" diff --git a/lib/functions/rootfs/post-tweaks.sh b/lib/functions/rootfs/post-tweaks.sh index f714a30841..1675b9d861 100644 --- a/lib/functions/rootfs/post-tweaks.sh +++ b/lib/functions/rootfs/post-tweaks.sh @@ -25,12 +25,10 @@ function post_debootstrap_tweaks() { chroot_sdcard dpkg-divert --quiet --local --rename --remove /sbin/start-stop-daemon run_host_command_logged rm -fv "${SDCARD}"/usr/sbin/policy-rc.d - # remove the qemu static binary - undeploy_qemu_binary_from_chroot "${SDCARD}" - call_extension_method "post_post_debootstrap_tweaks" "config_post_debootstrap_tweaks" <<- 'POST_POST_DEBOOTSTRAP_TWEAKS' *run after removing diversions and qemu with chroot unmounted* Last chance to touch the `${SDCARD}` filesystem before it is copied to the final media. - It is too late to run any chrooted commands, since the supporting filesystems are already unmounted. POST_POST_DEBOOTSTRAP_TWEAKS + + return 0 } diff --git a/lib/functions/rootfs/qemu-static.sh b/lib/functions/rootfs/qemu-static.sh index a2a8babab8..d4b8f789ee 100644 --- a/lib/functions/rootfs/qemu-static.sh +++ b/lib/functions/rootfs/qemu-static.sh @@ -8,30 +8,185 @@ # https://github.com/armbian/build/ function deploy_qemu_binary_to_chroot() { - local chroot_target="${1}" + declare chroot_target="${1}" caller="${2}" + display_alert "deploy_qemu_binary_to_chroot" "deploy_qemu_binary_to_chroot '${chroot_target}' during '${caller}'" "debug" - # @TODO: rpardini: Only deploy the binary if we're actually building a different architecture? otherwise unneeded. - - if [[ ! -f "${chroot_target}/usr/bin/${QEMU_BINARY}" ]]; then - display_alert "Deploying qemu-user-static binary to chroot" "${QEMU_BINARY}" "debug" - run_host_command_logged cp -pv "/usr/bin/${QEMU_BINARY}" "${chroot_target}/usr/bin/" - else - display_alert "qemu-user-static binary already deployed, skipping" "${QEMU_BINARY}" "debug" - fi -} - -function undeploy_qemu_binary_from_chroot() { - local chroot_target="${1}" - - # Hack: Check for magic "/usr/bin/qemu-s390x-static" marker; if that exists, it means "qemu-user-static" was installed - # in the chroot, and we shouldn't remove the binary, otherwise it's gonna be missing in the final image. - if [[ -f "${chroot_target}/usr/bin/qemu-s390x-static" ]]; then - display_alert "Not removing qemu binary, qemu-user-static package is installed in the chroot" "${QEMU_BINARY}" "debug" + # Only deploy the binary if we're actually building a non-native architecture. + if dpkg-architecture -e "${ARCH}"; then + display_alert "Native build" "not deploying qemu binary during ${caller}" "debug" return 0 fi - if [[ -f "${chroot_target}/usr/bin/${QEMU_BINARY}" ]]; then - display_alert "Removing qemu-user-static binary from chroot" "${QEMU_BINARY}" "debug" - run_host_command_logged rm -fv "${chroot_target}/usr/bin/${QEMU_BINARY}" + declare src_host="/usr/bin/${QEMU_BINARY}" + declare dst_target="${chroot_target}/usr/bin/${QEMU_BINARY}" + declare dst_target_bkp="${dst_target}.armbian.orig" + + # Assume we're getting a clean base to work with. Namely, we count on the rootfs cache to _not_ have left a dangling binary. + # If the dst_target already exists, it means the target actually has the qemu-static package installed. + # In that case, we preserve the original binary; it will be restored by the undeploy. + if [[ -f "${dst_target}" ]]; then + display_alert "Preserving existing qemu binary" "${QEMU_BINARY} during ${caller}" "info" + run_host_command_logged mv -v "${dst_target}" "${dst_target_bkp}" + fi + + display_alert "Deploying qemu-user-static binary to chroot" "${QEMU_BINARY} during ${caller}" "info" + run_host_command_logged cp -pv "${src_host}" "${dst_target}" + + return 0 +} + +function undeploy_qemu_binary_from_chroot() { + declare chroot_target="${1}" caller="${2}" + display_alert "undeploy_qemu_binary_from_chroot" "undeploy_qemu_binary_from_chroot '${chroot_target}' during '${caller}'" "debug" + + # Only deploy the binary if we're actually building a non-native architecture. + if dpkg-architecture -e "${ARCH}"; then + display_alert "Native build" "not deploying qemu binary during ${caller}" "debug" + return 0 + fi + + declare dst_target="${chroot_target}/usr/bin/${QEMU_BINARY}" + declare dst_target_bkp="${dst_target}.armbian.orig" + + # Check the binary we deployed is there. If not, panic, as we've lost control. + if [[ ! -f "${dst_target}" ]]; then + exit_with_error "Missing qemu binary during undeploy_qemu_binary_from_chroot from ${caller}" + fi + + # Remove the binary we deployed, and restore the original if we had to preserve it. + display_alert "Removing qemu-user-static binary from chroot" "${QEMU_BINARY} during ${caller}" "info" + run_host_command_logged rm -fv "${dst_target}" + + if [[ -f "${dst_target_bkp}" ]]; then + display_alert "Restoring original qemu binary" "${QEMU_BINARY} during ${caller}" "info" + run_host_command_logged mv -v "${dst_target_bkp}" "${dst_target}" + fi + + return 0 +} + +# "enable arm binary format so that the cross-architecture chroot environment will work" - classic comment from 2013 +# this is called from prepare-host.sh::prepare_host_noninteractive() unconditionally. +function prepare_host_binfmt_qemu() { + # NEEDS_BINFMT=yes is set by "default build" (image build) and rootfs artifact build, which is what requires binfmt_misc to be working. + if [[ "${NEEDS_BINFMT:-"no"}" != "yes" ]]; then + return 0 + fi + + if [[ "${SHOW_DEBUG}" == "yes" ]]; then + display_alert "Debugging binfmt - early" "/proc/sys/fs/binfmt_misc/" "debug" + run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true + fi + + if dpkg-architecture -e "${ARCH}"; then + display_alert "Native arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "cachehit" + else + prepare_host_binfmt_qemu_cross + fi + + if [[ "${SHOW_DEBUG}" == "yes" ]]; then + display_alert "Debugging binfmt - late" "/proc/sys/fs/binfmt_misc/" "debug" + run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true + fi + + # Actually test, using `arch-test`, that we can run binaries for the wanted architecture. + display_alert "checking" "arch-test for '${ARCH}'" "info" + run_host_command_logged arch-test "${ARCH}" + + # Everything should be either setup or previously correct if we get here. + return 0 +} + +# The actual binfmt manipulations when cross-build is confirmed above. +function prepare_host_binfmt_qemu_cross() { + local failed_binfmt_modprobe=0 + + display_alert "Cross arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "info" + + # Check if binfmt_misc is loaded; if not, try to load it, but don't fail: it might be built in. + if grep -q "^binfmt_misc" /proc/modules; then + display_alert "binfmt_misc is already loaded" "binfmt_misc already loaded" "debug" + else + display_alert "binfmt_misc is not loaded" "trying to load binfmt_misc" "debug" + + # try to modprobe. if it fails, emit a warning later, but not here. + # this is for the in-container case, where the host already has the module, but won't let the container know about it. + modprobe -q binfmt_misc || failed_binfmt_modprobe=1 + fi + + # Now, /proc/sys/fs/binfmt_misc/ has to be mounted. Mount, or fail with a message + if mountpoint -q /proc/sys/fs/binfmt_misc/; then + display_alert "binfmt_misc is already mounted" "binfmt_misc already mounted" "debug" + else + display_alert "binfmt_misc is not mounted" "trying to mount binfmt_misc" "debug" + mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ || { + if [[ $failed_binfmt_modprobe == 1 ]]; then + display_alert "Failed to load binfmt_misc module" "modprobe binfmt_misc failed" "wrn" + fi + display_alert "Check your HOST kernel" "CONFIG_BINFMT_MISC=m is required in host kernel" "warn" + display_alert "Failed to mount" "binfmt_misc /proc/sys/fs/binfmt_misc/" "err" + exit_with_error "Failed to mount binfmt_misc" + } + display_alert "binfmt_misc mounted" "binfmt_misc mounted" "debug" + fi + + declare host_arch + host_arch="$(arch)" + declare -a wanted_arches=("arm" "aarch64" "x86_64" "riscv64") + declare -A arch_aliases=(["aarch64"]="arm64" ["x86_64"]="amd64") + display_alert "Preparing binfmts for arch" "binfmts: host '${host_arch}', wanted arches '${wanted_arches[*]}'" "debug" + declare wanted_arch arch_alias + for wanted_arch in "${wanted_arches[@]}"; do + arch_alias="${arch_aliases["${wanted_arch}"]:-"${wanted_arch}"}" + display_alert "Preparing binfmts for arch" "wanted arch '${wanted_arch}' alias '${arch_alias}'" "debug" + + if [[ "${host_arch}" == "${wanted_arch}" ]]; then + continue + fi + + if [[ ! -e "/proc/sys/fs/binfmt_misc/qemu-${wanted_arch}" || ! -e "/usr/share/binfmts/qemu-${wanted_arch}" ]]; then + display_alert "Updating binfmts" "update-binfmts --enable qemu-${wanted_arch}" "debug" + + # special case: some arm64 machines cant' really run armhf binaries natively (Apple Silicon); check if that is the case and forcibly import and enable qemu-arm for them. + if [[ "${host_arch}" == "aarch64" && "${wanted_arch}" == "arm" ]]; then + prepare_host_binfmt_qemu_cross_arm64_host_armhf_target + else + run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" || display_alert "Failed to update binfmts" "update-binfmts --enable qemu-${wanted_arch}" "err" # log & continue on failure + fi + fi + done +} + +function prepare_host_binfmt_qemu_cross_arm64_host_armhf_target() { + display_alert "Trying to update binfmts - aarch64 mostly does 32-bit sans emulation, but Apple said no" "update-binfmts --enable qemu-${wanted_arch}" "debug" + run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading. + + if [[ "${SHOW_DEBUG}" == "yes" ]]; then + display_alert "Debugging arch-test" "full output" "debug" + run_host_command_logged arch-test "||" true + fi + + # to check, we use arch-test; if will return 0 if _either_ the host can natively run armhf, or if qemu-arm is correctly working. + if arch-test arm; then + display_alert "Host can run armhf natively or emulation is correctly setup already" "no need to enable qemu-arm" "debug" + else + display_alert "arm64 host can't run armhf natively" "importing enabling qemu-arm" "debug" + cat <<-BINFMT_ARM_MAGIC >/usr/share/binfmts/qemu-arm + package qemu-user-static + interpreter /usr/bin/qemu-arm-static + magic \x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00 + offset 0 + mask \xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff + credentials yes + fix_binary no + preserve yes + BINFMT_ARM_MAGIC + run_host_command_logged update-binfmts --import "qemu-${wanted_arch}" + run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" + + # Test again using arch-test. + display_alert "Checking if arm 32-bit emulation on arm64 works after enabling" "qemu-arm emulation" "info" + run_host_command_logged arch-test arm + display_alert "arm 32-bit emulation on arm64" "has been correctly setup" "cachehit" fi } diff --git a/lib/functions/rootfs/rootfs-create.sh b/lib/functions/rootfs/rootfs-create.sh index c7960213b5..5f18fe24b8 100644 --- a/lib/functions/rootfs/rootfs-create.sh +++ b/lib/functions/rootfs/rootfs-create.sh @@ -121,7 +121,7 @@ function create_new_rootfs_cache_via_debootstrap() { skip_target_check="yes" local_apt_deb_cache_prepare "after debootstrap first stage" # just for size reference in logs; skip the target check: debootstrap uses it for second stage. - deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() @TODO: which is too late for a cache + deploy_qemu_binary_to_chroot "${SDCARD}" "rootfs" # undeployed near the end of this function display_alert "Installing base system" "Stage 2/2" "info" declare -g -a if_error_find_files_sdcard=("debootstrap.log") # if command fails, go look for this file and show it's contents during error processing @@ -270,6 +270,9 @@ function create_new_rootfs_cache_via_debootstrap() { # `armbian-first-run` will do the same thing later chroot_sdcard systemctl mask systemd-firstboot.service + # undeploy the qemu binary; we don't want to ship the host's qemu binary in the rootfs cache. + undeploy_qemu_binary_from_chroot "${SDCARD}" "rootfs" + # stage: make rootfs cache archive display_alert "Ending debootstrap process and preparing cache" "$RELEASE" "info" wait_for_disk_sync "before tar rootfs"