From 84c0bf0f7d43d45f88f99a3391331639d3f0dcfa Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Wed, 30 Nov 2022 13:29:58 +0100 Subject: [PATCH] armbian-next: kernel: use `git worktree` & kernel.org clone bundles for kernel working copies - now we've a single `.git` for all kernels with all branches; a "master" cache - back to full stable git bundle usage from kernel.org - bye bye, "gitballs" - split kernel-related git stuff into `kernel-git.sh` - introduce `ARMBIAN_MOUNTPOINTS_DEPRECATED` @TODO actually use for cleaning non-Docker - add `axel` hostdep for multi-conn download of bundle, which comes from a CDN. - deprecate old mountpoints for gitballs and linux-kernel - move cleaning of old sources/kernel to mountpoints code --- lib/functions/compilation/kernel-git.sh | 105 ++++++++++++++++++++++++ lib/functions/compilation/kernel.sh | 72 ++-------------- lib/functions/general/git.sh | 75 ++++++----------- lib/functions/host/mountpoints.sh | 47 ++++++----- lib/functions/host/prepare-host.sh | 4 +- lib/functions/main/config-prepare.sh | 4 +- lib/library-functions.sh | 9 ++ 7 files changed, 179 insertions(+), 137 deletions(-) create mode 100644 lib/functions/compilation/kernel-git.sh diff --git a/lib/functions/compilation/kernel-git.sh b/lib/functions/compilation/kernel-git.sh new file mode 100644 index 0000000000..2ddab3fd81 --- /dev/null +++ b/lib/functions/compilation/kernel-git.sh @@ -0,0 +1,105 @@ +# This is run under do_with_retries. +function download_git_kernel_bundle() { + # See https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/ + + # rpardini: I'm a bit undecided about using the stable bundle (5gb) or the Linus bundle (3gb) + # the stable seems much better at the end of the day. + declare bundle_type="${bundle_type:-"stable"}" linux_clone_bundle_url="" + case "${bundle_type}" in + stable) + display_alert "Using Kernel bundle" "${bundle_type}" "debug" + linux_clone_bundle_url="https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/stable/linux/clone.bundle" + ;; + linus) + display_alert "Using Kernel bundle" "${bundle_type}" "debug" + linux_clone_bundle_url="https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/torvalds/linux/clone.bundle" + ;; + esac + + declare git_bundles_dir="${SRC}/cache/git-bundles/kernel" + run_host_command_logged mkdir -pv "${git_bundles_dir}" + + # defines outer scope value + linux_kernel_clone_bundle_file="${git_bundles_dir}/linux-${bundle_type}.bundle" + declare linux_kernel_clone_bundle_file_tmp="${linux_kernel_clone_bundle_file}.tmp" + + # if the file already exists, do nothing + if [[ -f "${linux_kernel_clone_bundle_file}" ]]; then + display_alert "Kernel bundle already exists" "${bundle_type}" "cachehit" + return 0 + fi + + # download into the tmp_file until it works, then rename to final; use axel. + do_with_retries 5 kernel_download_bundle_with_axel + + # move into place, only if everything worked, retried or not. + run_host_command_logged mv -v "${linux_kernel_clone_bundle_file_tmp}" "${linux_kernel_clone_bundle_file}" + + return 0 +} + +function kernel_download_bundle_with_axel() { + display_alert "Downloading Kernel bundle" "${bundle_type}; this might take a long time" "info" + declare -a verbose_params=() + if_user_on_terminal_and_not_logging_add verbose_params "--verbose" "--alternate" + if_user_not_on_terminal_or_is_logging_add verbose_params "--quiet" + run_host_command_logged axel "${verbose_params[@]}" "--output=${linux_kernel_clone_bundle_file_tmp}" \ + "${linux_clone_bundle_url}" +} + +function kernel_prepare_bare_repo_from_bundle() { + kernel_git_bare_tree="${SRC}/cache/git-bare/kernel" # sets the outer scope variable + declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/.git/armbian-bare-tree-done" + + if [[ ! -d "${kernel_git_bare_tree}" || ! -f "${kernel_git_bare_tree_done_marker}" ]]; then + if [[ -d "${kernel_git_bare_tree}" ]]; then + display_alert "Removing old kernel bare tree" "${kernel_git_bare_tree}" "info" + rm -rf "${kernel_git_bare_tree}" + fi + + # now, make sure we've the bundle downloaded correctly... + # this defines linux_kernel_clone_bundle_file + declare linux_kernel_clone_bundle_file + download_git_kernel_bundle + + # fetch it, completely, into the bare tree; use clone, not fetch. + display_alert "Cloning from bundle into bare tree" "this might take a very long time" "info" + declare -a verbose_params=() && if_user_on_terminal_and_not_logging_add verbose_params "--verbose" "--progress" + run_host_command_logged git clone "${verbose_params[@]}" --tags --no-checkout \ + "${linux_kernel_clone_bundle_file}" "${kernel_git_bare_tree}" + + # write the marker file + touch "${kernel_git_bare_tree_done_marker}" + fi + + return 0 +} + +function kernel_prepare_git_pre_fetch() { + local remote_name="kernel-stable-${KERNEL_MAJOR_MINOR}" + local remote_url="${MAINLINE_KERNEL_SOURCE}" + local remote_tags_to_fetch="v${KERNEL_MAJOR_MINOR}*" + + # shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context. + if [[ "${do_add_origin}" == "yes" ]]; then + display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git" + regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source + + # Fetch the tags. This allows working -rcX versions of still-unreleased minor versions. + improved_git_fetch "${remote_name}" "'refs/tags/${remote_tags_to_fetch}:refs/tags/${remote_tags_to_fetch}'" || true # Fetch the remote branch tags + display_alert "After mainline stable tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull + fi +} + +function kernel_prepare_git() { + [[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then? + [[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree" + + display_alert "Downloading sources" "kernel" "git" + + GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \ + GIT_PRE_FETCH_HOOK=kernel_prepare_git_pre_fetch_tags \ + GIT_BARE_REPO_FOR_WORKTREE="${kernel_git_bare_tree}" \ + GIT_BARE_REPO_INITIAL_BRANCH="master" \ + fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes" +} diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index dcccbfb556..7169d6763d 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -27,7 +27,7 @@ function run_kernel_make_internal() { common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}") common_make_params_quoted=( - # @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir. + # @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir; this has high impact in headers packaging though. "${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled. @@ -76,8 +76,13 @@ function compile_kernel() { `${kernel_work_dir}` is set, but not yet populated with kernel sources. FETCH_SOURCES_FOR_KERNEL_DRIVER + # Prepare the git bare repo for the kernel. + declare kernel_git_bare_tree + LOG_SECTION="kernel_prepare_bare_repo_from_bundle" do_with_logging_unless_user_terminal do_with_hooks \ + kernel_prepare_bare_repo_from_bundle # this sets kernel_git_bare_tree + declare checked_out_revision_mtime="" checked_out_revision_ts="" # set by fetch_from_repo - LOG_SECTION="kernel_prepare_git" do_with_logging do_with_hooks kernel_prepare_git + LOG_SECTION="kernel_prepare_git" do_with_logging_unless_user_terminal do_with_hooks kernel_prepare_git # Capture date variables set by fetch_from_repo; it's the date of the last kernel revision declare kernel_base_revision_date @@ -122,69 +127,6 @@ function compile_kernel() { return 0 } -function kernel_init_git_from_http_gitball() { - local kernel_git_dir="${1}" - - local gitball_dir="${SRC}/cache/gitballs/kernel" - if [[ ! -d "${gitball_dir}" ]]; then - display_alert "Creating kernel git gitball cache dir" "${gitball_dir}" "info" - run_host_command_logged mkdir -pv "${gitball_dir}" - [[ -d "${SRC}/cache/gitbundles" ]] && run_host_command_logged rm -rf "${SRC}/cache/gitbundles" # remove old gitbundles dir. we won't be using those anymore; @TODO: remove this line in the future - fi - - local gitball_file="${gitball_dir}/linux-${KERNEL_MAJOR_MINOR}.git.tar" - local gitball_url="https://github.com/armbian/gitutils/releases/download/latest/linux-${KERNEL_MAJOR_MINOR}.git.tar" - - if [[ ! -f "${gitball_file}" ]]; then # Download the gitball file if it does not exist. - display_alert "Downloading Git cold gitball via HTTP" "${gitball_url}" "info" # This gonna take a while. And waste bandwidth - run_host_command_logged wget --continue --progress=dot:giga --output-document="${gitball_file}.tmp" "${gitball_url}" - run_host_command_logged mv -v "${gitball_file}.tmp" "${gitball_file}" - else - display_alert "Cold gitball file exists, using it" "${gitball_file}" "git" - fi - - # Extract the gitball file to the git directory. This will create '.git' - run_host_command_logged tar -xvf "${gitball_file}" -C "${kernel_git_dir}" - - # Sanity check - regular_git branch -a --list --color -} - -function kernel_prepare_git_pre_fetch() { - local remote_name="kernel-stable-${KERNEL_MAJOR_MINOR}" - local remote_url="${MAINLINE_KERNEL_SOURCE}" - local remote_tags_to_fetch="v${KERNEL_MAJOR_MINOR}*" - - # shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context. - if [[ "${do_add_origin}" == "yes" ]]; then - display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git" - regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source - - # Fetch the tags. This allows working -rcX versions of still-unreleased minor versions. - improved_git_fetch "${remote_name}" "'refs/tags/${remote_tags_to_fetch}:refs/tags/${remote_tags_to_fetch}'" || true # Fetch the remote branch tags - display_alert "After mainline stable tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull - fi -} - -function kernel_prepare_git() { - [[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then? - - # LINUXSOURCEDIR has changed a lot, cleanup old incarnations if they exist - if [[ -d "${SRC}/cache/sources/kernel" ]]; then - display_alert "Cleaning up old kernel sources directory" "might take a while" "debug" - run_host_command_logged rm -rf "${SRC}/cache/sources/kernel" - fi - - [[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree" - - display_alert "Downloading sources" "kernel" "git" - - GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \ - GIT_INIT_REPO_HOOK=kernel_init_git_from_http_gitball \ - GIT_PRE_FETCH_HOOK=kernel_prepare_git_pre_fetch_tags \ - fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes" -} - function kernel_maybe_clean() { if [[ $CLEAN_LEVEL == *make-kernel* ]]; then display_alert "Cleaning Kernel tree - CLEAN_LEVEL contains 'make-kernel'" "$LINUXSOURCEDIR" "info" diff --git a/lib/functions/general/git.sh b/lib/functions/general/git.sh index 878c1b7252..d381406d0d 100644 --- a/lib/functions/general/git.sh +++ b/lib/functions/general/git.sh @@ -90,53 +90,34 @@ fetch_from_repo() { git_work_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}" fi - mkdir -p "${git_work_dir}" || exit_with_error "No path or no write permission" "${git_work_dir}" - - cd "${git_work_dir}" || exit - display_alert "Git working dir" "${git_work_dir}" "git" - git_ensure_safe_directory "${git_work_dir}" - local expected_origin_url actual_origin_url - expected_origin_url="$(echo -n "${url}" | sed 's/^.*@//' | sed 's/^.*\/\///')" - local do_add_origin="no" - - # Make sure the origin matches what is expected, if the repo already exists. - if [[ -d ".git" && "$(git rev-parse --git-dir)" == ".git" ]]; then - actual_origin_url="$(git config remote.origin.url | sed 's/^.*@//' | sed 's/^.*\/\///')" - if [[ "a${actual_origin_url}a" == "aa" ]]; then - display_alert "Repo's origin is empty, adding it" "${url}" "warn" - offline=false # Force online, we'll need to fetch. - do_add_origin="yes" # Empty origin, we'll add it. - elif [[ "${expected_origin_url}" != "${actual_origin_url}" ]]; then - display_alert "Remote git URL does not match" "${git_work_dir} expected: '${expected_origin_url}' actual: '${actual_origin_url}'" "warn" - if [[ "${ALLOW_GIT_ORIGIN_CHANGE}" != "yes" ]]; then - exit_with_error "Would have deleted ${git_work_dir} to change origin URL, but ALLOW_GIT_ORIGIN_CHANGE is not set to 'yes'" - fi - display_alert "Remote git URL does not match, deleting working copy" "${git_work_dir} expected: '${expected_origin_url}' actual: '${actual_origin_url}'" "warn" - cd "${SRC}" || exit 3 # free up cwd - run_host_command_logged rm -rf "${git_work_dir}" # delete the dir - mkdir -p "${git_work_dir}" || exit_with_error "No path or no write permission" "${git_work_dir}" # recreate - cd "${git_work_dir}" || exit #reset cwd - fi - fi - - if [[ ! -d ".git" || "$(git rev-parse --git-dir)" != ".git" ]]; then - # Dir is not a git working copy. Make it so; - # If callback is defined, call it. Give it the dir as param. The rest it will read from environment. - # If not callback defined, do an init, and schedule a fetch. - - if [[ $(type -t ${GIT_INIT_REPO_HOOK} || true) == function ]]; then - display_alert "Delegating to ${GIT_INIT_REPO_HOOK}()" "git init: $dir $ref_name" "debug" - ${GIT_INIT_REPO_HOOK} "${git_work_dir}" + # Support using worktrees; needs GIT_BARE_REPO_FOR_WORKTREE set + if [[ "x${GIT_BARE_REPO_FOR_WORKTREE}x" != "xx" ]]; then + # If it is already a worktree... + if [[ -f "${git_work_dir}/.git" ]]; then + display_alert "Using existing worktree" "${git_work_dir}" "git" else + if [[ -d "${git_work_dir}" ]]; then + display_alert "Removing previously half-checked-out tree" "${git_work_dir}" "warn" + cd "${SRC}" || exit_with_error "Could not cd to ${SRC}" + rm -rf "${git_work_dir}" + fi + display_alert "Creating new worktree" "${git_work_dir}" "git" + run_host_command_logged git -C "${GIT_BARE_REPO_FOR_WORKTREE}" worktree add "${git_work_dir}" "${GIT_BARE_REPO_INITIAL_BRANCH}" --no-checkout --force + cd "${git_work_dir}" || exit + fi + cd "${git_work_dir}" || exit + else + mkdir -p "${git_work_dir}" || exit_with_error "No path or no write permission" "${git_work_dir}" + cd "${git_work_dir}" || exit + if [[ ! -d ".git" || "$(git rev-parse --git-dir)" != ".git" ]]; then + # Dir is not a git working copy. Make it so; display_alert "Initializing empty git local copy" "git init: $dir $ref_name" regular_git init -q --initial-branch="armbian_unused_initial_branch" . + offline=false # Force online, we'll need to fetch. fi - - offline=false # Force online, we'll need to fetch. - do_add_origin="yes" # Just created the repo, it needs an origin later. fi local changed=false @@ -186,18 +167,14 @@ fetch_from_repo() { ${GIT_PRE_FETCH_HOOK} "${git_work_dir}" "${url}" "$ref_type" "$ref_name" fi - if [[ "${do_add_origin}" == "yes" ]]; then - regular_git remote add origin "${url}" - fi - # remote was updated, fetch and check out updates, but not tags; tags pull their respective commits too, making it a huge fetch. - display_alert "Fetching updates from origin" "$dir $ref_name" + display_alert "Fetching updates from remote repository" "$dir $ref_name" case $ref_type in - branch | commit) improved_git_fetch --no-tags origin "${ref_name}" ;; - tag) improved_git_fetch --no-tags origin tags/"${ref_name}" ;; - head) improved_git_fetch --no-tags origin HEAD ;; + branch | commit) improved_git_fetch --no-tags "${url}" "${ref_name}" ;; + tag) improved_git_fetch --no-tags "${url}" tags/"${ref_name}" ;; + head) improved_git_fetch --no-tags "${url}" HEAD ;; esac - display_alert "Origin fetch completed, working copy size" "$(du -h -s | awk '{print $1}')" "git" + display_alert "Remote repository fetch completed, working copy size" "$(du -h -s | awk '{print $1}')" "git" checkout_from="FETCH_HEAD" fi diff --git a/lib/functions/host/mountpoints.sh b/lib/functions/host/mountpoints.sh index c783a46c49..b374dbca6a 100644 --- a/lib/functions/host/mountpoints.sh +++ b/lib/functions/host/mountpoints.sh @@ -5,33 +5,42 @@ function prepare_armbian_mountpoints_description_dict() { declare -g -a ARMBIAN_MOUNTPOINTS_ARRAY=( ".tmp" "output" "output/images" "output/debs" "output/logs" - "cache" "cache/gitballs" "cache/toolchain" + "cache" + "cache/git-bare" + "cache/git-bundles" + "cache/toolchain" "cache/aptcache" - "cache/rootfs" "cache/initrd" - "cache/sources" "cache/sources/linux-kernel" + "cache/rootfs" + "cache/initrd" + "cache/sources" + "cache/sources/linux-kernel-worktree" "cache/ccache" ) declare -A -g ARMBIAN_MOUNTPOINTS_DESC_DICT=( - [".tmp"]="docker_kind_linux=anonymous docker_kind_darwin=anonymous" # tmpfs, discard, anonymous; whatever you wanna call it. It just needs to be 100% local to the container, and there's very little value in being able to look at it from the host. - ["output"]="docker_kind_linux=bind docker_kind_darwin=bind" # catch-all output. specific subdirs are mounted below. it's a bind mount by default on both Linux and Darwin. - ["output/images"]="docker_kind_linux=bind docker_kind_darwin=bind" # 99% of users want this as the result of their build, no matter if it's slow or not. bind on both. - ["output/debs"]="docker_kind_linux=bind docker_kind_darwin=bind" # generated output .deb files. most people are interested in this, to update kernels or dtbs after initial build. bind on both Linux and Darwin. - ["output/logs"]="docker_kind_linux=bind docker_kind_darwin=bind" # log files produced. 100% of users want this. Bind on both Linux and Darwin. Is used to integrate launcher and actual-build logs, so must exist and work otherwise confusion ensues. - ["cache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # catch-all cache, could be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/gitballs"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # tarballs of git repos, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/toolchain"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # toolchain cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/aptcache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # .deb apt cache, replaces apt-cacher-ng. Can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/rootfs"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # rootfs .tar.zst cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/initrd"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # initrd.img cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/sources"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # operating directory. many things are cloned in here, and some are even built inside. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/sources/linux-kernel"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for kernel builds. huge. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. - ["cache/ccache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # ccache object store. limited to 5gb by default. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. + [".tmp"]="docker_kind_linux=anonymous docker_kind_darwin=anonymous" # tmpfs, discard, anonymous; whatever you wanna call it. It just needs to be 100% local to the container, and there's very little value in being able to look at it from the host. + ["output"]="docker_kind_linux=bind docker_kind_darwin=bind" # catch-all output. specific subdirs are mounted below. it's a bind mount by default on both Linux and Darwin. + ["output/images"]="docker_kind_linux=bind docker_kind_darwin=bind" # 99% of users want this as the result of their build, no matter if it's slow or not. bind on both. + ["output/debs"]="docker_kind_linux=bind docker_kind_darwin=bind" # generated output .deb files. most people are interested in this, to update kernels or dtbs after initial build. bind on both Linux and Darwin. + ["output/logs"]="docker_kind_linux=bind docker_kind_darwin=bind" # log files produced. 100% of users want this. Bind on both Linux and Darwin. Is used to integrate launcher and actual-build logs, so must exist and work otherwise confusion ensues. + ["cache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # catch-all cache, could be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/git-bare"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # Git bare repos (kernel/u-boot). On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/git-bundles"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # Downloads of git bundles, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/toolchain"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # toolchain cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/aptcache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # .deb apt cache, replaces apt-cacher-ng. Can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/rootfs"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # rootfs .tar.zst cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/initrd"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # initrd.img cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/sources"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # operating directory. many things are cloned in here, and some are even built inside. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/sources/linux-kernel-worktree"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for kernel builds. huge. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/ccache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # ccache object store. limited to 5gb by default. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. ) # These, if found, will be removed on `dockerpurge` and other cleanups. # They "used to be" used by the build system, but no longer. declare -g -a ARMBIAN_MOUNTPOINTS_DEPRECATED=( + "cache/gitballs" + "cache/sources/kernel" + "cache/sources/linux-kernel" ) } @@ -63,10 +72,10 @@ function loop_over_armbian_mountpoints() { display_alert "Unsupported host OS" "${DOCKER_ARMBIAN_HOST_OS_UNAME} - cant map mountpoint to Docker volume" "warn" ;; esac - + # volume_id is the mountpoint with all slashes replaced with dashes local volume_id="${mountpoint//\//-}" - + # shellcheck disable=SC2086 eval "$values docker_kind=$docker_kind $func '$mountpoint'" done diff --git a/lib/functions/host/prepare-host.sh b/lib/functions/host/prepare-host.sh index fecff48257..32cfe88008 100644 --- a/lib/functions/host/prepare-host.sh +++ b/lib/functions/host/prepare-host.sh @@ -233,7 +233,7 @@ function early_prepare_host_dependencies() { pkg-config pv qemu-user-static rsync swig u-boot-tools udev uuid-dev whiptail zlib1g-dev busybox fdisk - + # distcc, experimental, optional; see cli-distcc.sh and kernel.sh distcc @@ -250,7 +250,7 @@ function early_prepare_host_dependencies() { file ccze colorized-logs tree expect # logging utilities; expect is needed for 'unbuffer' command unzip zip p7zip-full pigz pixz pbzip2 lzop zstd # compressors et al parted gdisk # partition tools - aria2 curl wget # downloaders et al + aria2 curl wget axel # downloaders et al parallel # do things in parallel # toolchains. NEW: using metapackages, allow us to have same list of all arches; brings both C and C++ compilers crossbuild-essential-armhf crossbuild-essential-armel # for ARM 32-bit, both HF and EL are needed in some cases. diff --git a/lib/functions/main/config-prepare.sh b/lib/functions/main/config-prepare.sh index 502a346a42..10f481bf6c 100644 --- a/lib/functions/main/config-prepare.sh +++ b/lib/functions/main/config-prepare.sh @@ -157,11 +157,11 @@ function prepare_and_config_main_build_single() { fi # Default LINUXSOURCEDIR: - export LINUXSOURCEDIR="linux-kernel/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}/${ARCH}" + export LINUXSOURCEDIR="linux-kernel-worktree/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}__${ARCH}" # Allow adding to it with KERNEL_EXTRA_DIR if [[ "${KERNEL_EXTRA_DIR}" != "" ]]; then - export LINUXSOURCEDIR="linux-kernel/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}_${KERNEL_EXTRA_DIR}/${ARCH}" + export LINUXSOURCEDIR="${LINUXSOURCEDIR}__${KERNEL_EXTRA_DIR}" display_alert "Using kernel extra dir: '${KERNEL_EXTRA_DIR}'" "LINUXSOURCEDIR: ${LINUXSOURCEDIR}" "debug" fi else diff --git a/lib/library-functions.sh b/lib/library-functions.sh index ec6af1f77c..82ac927baa 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -172,6 +172,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/compilation/kernel-debs.sh source "${SRC}"/lib/functions/compilation/kernel-debs.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/compilation/kernel-git.sh +# shellcheck source=lib/functions/compilation/kernel-git.sh +source "${SRC}"/lib/functions/compilation/kernel-git.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled