armbian-next: kernel: use git worktree & kernel.org clone bundles for kernel working copies
- now we've a single `.git` for all kernels with all branches; a "master" cache - back to full stable git bundle usage from kernel.org - bye bye, "gitballs" - split kernel-related git stuff into `kernel-git.sh` - introduce `ARMBIAN_MOUNTPOINTS_DEPRECATED` @TODO actually use for cleaning non-Docker - add `axel` hostdep for multi-conn download of bundle, which comes from a CDN. - deprecate old mountpoints for gitballs and linux-kernel - move cleaning of old sources/kernel to mountpoints code
This commit is contained in:
Ricardo Pardini
parent
a51448a1da
commit
84c0bf0f7d
105
lib/functions/compilation/kernel-git.sh
Normal file
105
lib/functions/compilation/kernel-git.sh
Normal file
@ -0,0 +1,105 @@
|
||||
# This is run under do_with_retries.
|
||||
function download_git_kernel_bundle() {
|
||||
# See https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/
|
||||
|
||||
# rpardini: I'm a bit undecided about using the stable bundle (5gb) or the Linus bundle (3gb)
|
||||
# the stable seems much better at the end of the day.
|
||||
declare bundle_type="${bundle_type:-"stable"}" linux_clone_bundle_url=""
|
||||
case "${bundle_type}" in
|
||||
stable)
|
||||
display_alert "Using Kernel bundle" "${bundle_type}" "debug"
|
||||
linux_clone_bundle_url="https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/stable/linux/clone.bundle"
|
||||
;;
|
||||
linus)
|
||||
display_alert "Using Kernel bundle" "${bundle_type}" "debug"
|
||||
linux_clone_bundle_url="https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/torvalds/linux/clone.bundle"
|
||||
;;
|
||||
esac
|
||||
|
||||
declare git_bundles_dir="${SRC}/cache/git-bundles/kernel"
|
||||
run_host_command_logged mkdir -pv "${git_bundles_dir}"
|
||||
|
||||
# defines outer scope value
|
||||
linux_kernel_clone_bundle_file="${git_bundles_dir}/linux-${bundle_type}.bundle"
|
||||
declare linux_kernel_clone_bundle_file_tmp="${linux_kernel_clone_bundle_file}.tmp"
|
||||
|
||||
# if the file already exists, do nothing
|
||||
if [[ -f "${linux_kernel_clone_bundle_file}" ]]; then
|
||||
display_alert "Kernel bundle already exists" "${bundle_type}" "cachehit"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# download into the tmp_file until it works, then rename to final; use axel.
|
||||
do_with_retries 5 kernel_download_bundle_with_axel
|
||||
|
||||
# move into place, only if everything worked, retried or not.
|
||||
run_host_command_logged mv -v "${linux_kernel_clone_bundle_file_tmp}" "${linux_kernel_clone_bundle_file}"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function kernel_download_bundle_with_axel() {
|
||||
display_alert "Downloading Kernel bundle" "${bundle_type}; this might take a long time" "info"
|
||||
declare -a verbose_params=()
|
||||
if_user_on_terminal_and_not_logging_add verbose_params "--verbose" "--alternate"
|
||||
if_user_not_on_terminal_or_is_logging_add verbose_params "--quiet"
|
||||
run_host_command_logged axel "${verbose_params[@]}" "--output=${linux_kernel_clone_bundle_file_tmp}" \
|
||||
"${linux_clone_bundle_url}"
|
||||
}
|
||||
|
||||
function kernel_prepare_bare_repo_from_bundle() {
|
||||
kernel_git_bare_tree="${SRC}/cache/git-bare/kernel" # sets the outer scope variable
|
||||
declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/.git/armbian-bare-tree-done"
|
||||
|
||||
if [[ ! -d "${kernel_git_bare_tree}" || ! -f "${kernel_git_bare_tree_done_marker}" ]]; then
|
||||
if [[ -d "${kernel_git_bare_tree}" ]]; then
|
||||
display_alert "Removing old kernel bare tree" "${kernel_git_bare_tree}" "info"
|
||||
rm -rf "${kernel_git_bare_tree}"
|
||||
fi
|
||||
|
||||
# now, make sure we've the bundle downloaded correctly...
|
||||
# this defines linux_kernel_clone_bundle_file
|
||||
declare linux_kernel_clone_bundle_file
|
||||
download_git_kernel_bundle
|
||||
|
||||
# fetch it, completely, into the bare tree; use clone, not fetch.
|
||||
display_alert "Cloning from bundle into bare tree" "this might take a very long time" "info"
|
||||
declare -a verbose_params=() && if_user_on_terminal_and_not_logging_add verbose_params "--verbose" "--progress"
|
||||
run_host_command_logged git clone "${verbose_params[@]}" --tags --no-checkout \
|
||||
"${linux_kernel_clone_bundle_file}" "${kernel_git_bare_tree}"
|
||||
|
||||
# write the marker file
|
||||
touch "${kernel_git_bare_tree_done_marker}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function kernel_prepare_git_pre_fetch() {
|
||||
local remote_name="kernel-stable-${KERNEL_MAJOR_MINOR}"
|
||||
local remote_url="${MAINLINE_KERNEL_SOURCE}"
|
||||
local remote_tags_to_fetch="v${KERNEL_MAJOR_MINOR}*"
|
||||
|
||||
# shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context.
|
||||
if [[ "${do_add_origin}" == "yes" ]]; then
|
||||
display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git"
|
||||
regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source
|
||||
|
||||
# Fetch the tags. This allows working -rcX versions of still-unreleased minor versions.
|
||||
improved_git_fetch "${remote_name}" "'refs/tags/${remote_tags_to_fetch}:refs/tags/${remote_tags_to_fetch}'" || true # Fetch the remote branch tags
|
||||
display_alert "After mainline stable tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull
|
||||
fi
|
||||
}
|
||||
|
||||
function kernel_prepare_git() {
|
||||
[[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then?
|
||||
[[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree"
|
||||
|
||||
display_alert "Downloading sources" "kernel" "git"
|
||||
|
||||
GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \
|
||||
GIT_PRE_FETCH_HOOK=kernel_prepare_git_pre_fetch_tags \
|
||||
GIT_BARE_REPO_FOR_WORKTREE="${kernel_git_bare_tree}" \
|
||||
GIT_BARE_REPO_INITIAL_BRANCH="master" \
|
||||
fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes"
|
||||
}
|
||||
@ -27,7 +27,7 @@ function run_kernel_make_internal() {
|
||||
common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}")
|
||||
|
||||
common_make_params_quoted=(
|
||||
# @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir.
|
||||
# @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir; this has high impact in headers packaging though.
|
||||
|
||||
"${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled.
|
||||
|
||||
@ -76,8 +76,13 @@ function compile_kernel() {
|
||||
`${kernel_work_dir}` is set, but not yet populated with kernel sources.
|
||||
FETCH_SOURCES_FOR_KERNEL_DRIVER
|
||||
|
||||
# Prepare the git bare repo for the kernel.
|
||||
declare kernel_git_bare_tree
|
||||
LOG_SECTION="kernel_prepare_bare_repo_from_bundle" do_with_logging_unless_user_terminal do_with_hooks \
|
||||
kernel_prepare_bare_repo_from_bundle # this sets kernel_git_bare_tree
|
||||
|
||||
declare checked_out_revision_mtime="" checked_out_revision_ts="" # set by fetch_from_repo
|
||||
LOG_SECTION="kernel_prepare_git" do_with_logging do_with_hooks kernel_prepare_git
|
||||
LOG_SECTION="kernel_prepare_git" do_with_logging_unless_user_terminal do_with_hooks kernel_prepare_git
|
||||
|
||||
# Capture date variables set by fetch_from_repo; it's the date of the last kernel revision
|
||||
declare kernel_base_revision_date
|
||||
@ -122,69 +127,6 @@ function compile_kernel() {
|
||||
return 0
|
||||
}
|
||||
|
||||
function kernel_init_git_from_http_gitball() {
|
||||
local kernel_git_dir="${1}"
|
||||
|
||||
local gitball_dir="${SRC}/cache/gitballs/kernel"
|
||||
if [[ ! -d "${gitball_dir}" ]]; then
|
||||
display_alert "Creating kernel git gitball cache dir" "${gitball_dir}" "info"
|
||||
run_host_command_logged mkdir -pv "${gitball_dir}"
|
||||
[[ -d "${SRC}/cache/gitbundles" ]] && run_host_command_logged rm -rf "${SRC}/cache/gitbundles" # remove old gitbundles dir. we won't be using those anymore; @TODO: remove this line in the future
|
||||
fi
|
||||
|
||||
local gitball_file="${gitball_dir}/linux-${KERNEL_MAJOR_MINOR}.git.tar"
|
||||
local gitball_url="https://github.com/armbian/gitutils/releases/download/latest/linux-${KERNEL_MAJOR_MINOR}.git.tar"
|
||||
|
||||
if [[ ! -f "${gitball_file}" ]]; then # Download the gitball file if it does not exist.
|
||||
display_alert "Downloading Git cold gitball via HTTP" "${gitball_url}" "info" # This gonna take a while. And waste bandwidth
|
||||
run_host_command_logged wget --continue --progress=dot:giga --output-document="${gitball_file}.tmp" "${gitball_url}"
|
||||
run_host_command_logged mv -v "${gitball_file}.tmp" "${gitball_file}"
|
||||
else
|
||||
display_alert "Cold gitball file exists, using it" "${gitball_file}" "git"
|
||||
fi
|
||||
|
||||
# Extract the gitball file to the git directory. This will create '.git'
|
||||
run_host_command_logged tar -xvf "${gitball_file}" -C "${kernel_git_dir}"
|
||||
|
||||
# Sanity check
|
||||
regular_git branch -a --list --color
|
||||
}
|
||||
|
||||
function kernel_prepare_git_pre_fetch() {
|
||||
local remote_name="kernel-stable-${KERNEL_MAJOR_MINOR}"
|
||||
local remote_url="${MAINLINE_KERNEL_SOURCE}"
|
||||
local remote_tags_to_fetch="v${KERNEL_MAJOR_MINOR}*"
|
||||
|
||||
# shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context.
|
||||
if [[ "${do_add_origin}" == "yes" ]]; then
|
||||
display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git"
|
||||
regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source
|
||||
|
||||
# Fetch the tags. This allows working -rcX versions of still-unreleased minor versions.
|
||||
improved_git_fetch "${remote_name}" "'refs/tags/${remote_tags_to_fetch}:refs/tags/${remote_tags_to_fetch}'" || true # Fetch the remote branch tags
|
||||
display_alert "After mainline stable tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull
|
||||
fi
|
||||
}
|
||||
|
||||
function kernel_prepare_git() {
|
||||
[[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then?
|
||||
|
||||
# LINUXSOURCEDIR has changed a lot, cleanup old incarnations if they exist
|
||||
if [[ -d "${SRC}/cache/sources/kernel" ]]; then
|
||||
display_alert "Cleaning up old kernel sources directory" "might take a while" "debug"
|
||||
run_host_command_logged rm -rf "${SRC}/cache/sources/kernel"
|
||||
fi
|
||||
|
||||
[[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree"
|
||||
|
||||
display_alert "Downloading sources" "kernel" "git"
|
||||
|
||||
GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \
|
||||
GIT_INIT_REPO_HOOK=kernel_init_git_from_http_gitball \
|
||||
GIT_PRE_FETCH_HOOK=kernel_prepare_git_pre_fetch_tags \
|
||||
fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes"
|
||||
}
|
||||
|
||||
function kernel_maybe_clean() {
|
||||
if [[ $CLEAN_LEVEL == *make-kernel* ]]; then
|
||||
display_alert "Cleaning Kernel tree - CLEAN_LEVEL contains 'make-kernel'" "$LINUXSOURCEDIR" "info"
|
||||
|
||||
@ -90,53 +90,34 @@ fetch_from_repo() {
|
||||
git_work_dir="${SRC}/cache/sources/${GIT_FIXED_WORKDIR}"
|
||||
fi
|
||||
|
||||
mkdir -p "${git_work_dir}" || exit_with_error "No path or no write permission" "${git_work_dir}"
|
||||
|
||||
cd "${git_work_dir}" || exit
|
||||
|
||||
display_alert "Git working dir" "${git_work_dir}" "git"
|
||||
|
||||
git_ensure_safe_directory "${git_work_dir}"
|
||||
|
||||
local expected_origin_url actual_origin_url
|
||||
expected_origin_url="$(echo -n "${url}" | sed 's/^.*@//' | sed 's/^.*\/\///')"
|
||||
local do_add_origin="no"
|
||||
|
||||
# Make sure the origin matches what is expected, if the repo already exists.
|
||||
if [[ -d ".git" && "$(git rev-parse --git-dir)" == ".git" ]]; then
|
||||
actual_origin_url="$(git config remote.origin.url | sed 's/^.*@//' | sed 's/^.*\/\///')"
|
||||
if [[ "a${actual_origin_url}a" == "aa" ]]; then
|
||||
display_alert "Repo's origin is empty, adding it" "${url}" "warn"
|
||||
offline=false # Force online, we'll need to fetch.
|
||||
do_add_origin="yes" # Empty origin, we'll add it.
|
||||
elif [[ "${expected_origin_url}" != "${actual_origin_url}" ]]; then
|
||||
display_alert "Remote git URL does not match" "${git_work_dir} expected: '${expected_origin_url}' actual: '${actual_origin_url}'" "warn"
|
||||
if [[ "${ALLOW_GIT_ORIGIN_CHANGE}" != "yes" ]]; then
|
||||
exit_with_error "Would have deleted ${git_work_dir} to change origin URL, but ALLOW_GIT_ORIGIN_CHANGE is not set to 'yes'"
|
||||
fi
|
||||
display_alert "Remote git URL does not match, deleting working copy" "${git_work_dir} expected: '${expected_origin_url}' actual: '${actual_origin_url}'" "warn"
|
||||
cd "${SRC}" || exit 3 # free up cwd
|
||||
run_host_command_logged rm -rf "${git_work_dir}" # delete the dir
|
||||
mkdir -p "${git_work_dir}" || exit_with_error "No path or no write permission" "${git_work_dir}" # recreate
|
||||
cd "${git_work_dir}" || exit #reset cwd
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -d ".git" || "$(git rev-parse --git-dir)" != ".git" ]]; then
|
||||
# Dir is not a git working copy. Make it so;
|
||||
# If callback is defined, call it. Give it the dir as param. The rest it will read from environment.
|
||||
# If not callback defined, do an init, and schedule a fetch.
|
||||
|
||||
if [[ $(type -t ${GIT_INIT_REPO_HOOK} || true) == function ]]; then
|
||||
display_alert "Delegating to ${GIT_INIT_REPO_HOOK}()" "git init: $dir $ref_name" "debug"
|
||||
${GIT_INIT_REPO_HOOK} "${git_work_dir}"
|
||||
# Support using worktrees; needs GIT_BARE_REPO_FOR_WORKTREE set
|
||||
if [[ "x${GIT_BARE_REPO_FOR_WORKTREE}x" != "xx" ]]; then
|
||||
# If it is already a worktree...
|
||||
if [[ -f "${git_work_dir}/.git" ]]; then
|
||||
display_alert "Using existing worktree" "${git_work_dir}" "git"
|
||||
else
|
||||
if [[ -d "${git_work_dir}" ]]; then
|
||||
display_alert "Removing previously half-checked-out tree" "${git_work_dir}" "warn"
|
||||
cd "${SRC}" || exit_with_error "Could not cd to ${SRC}"
|
||||
rm -rf "${git_work_dir}"
|
||||
fi
|
||||
display_alert "Creating new worktree" "${git_work_dir}" "git"
|
||||
run_host_command_logged git -C "${GIT_BARE_REPO_FOR_WORKTREE}" worktree add "${git_work_dir}" "${GIT_BARE_REPO_INITIAL_BRANCH}" --no-checkout --force
|
||||
cd "${git_work_dir}" || exit
|
||||
fi
|
||||
cd "${git_work_dir}" || exit
|
||||
else
|
||||
mkdir -p "${git_work_dir}" || exit_with_error "No path or no write permission" "${git_work_dir}"
|
||||
cd "${git_work_dir}" || exit
|
||||
if [[ ! -d ".git" || "$(git rev-parse --git-dir)" != ".git" ]]; then
|
||||
# Dir is not a git working copy. Make it so;
|
||||
display_alert "Initializing empty git local copy" "git init: $dir $ref_name"
|
||||
regular_git init -q --initial-branch="armbian_unused_initial_branch" .
|
||||
offline=false # Force online, we'll need to fetch.
|
||||
fi
|
||||
|
||||
offline=false # Force online, we'll need to fetch.
|
||||
do_add_origin="yes" # Just created the repo, it needs an origin later.
|
||||
fi
|
||||
|
||||
local changed=false
|
||||
@ -186,18 +167,14 @@ fetch_from_repo() {
|
||||
${GIT_PRE_FETCH_HOOK} "${git_work_dir}" "${url}" "$ref_type" "$ref_name"
|
||||
fi
|
||||
|
||||
if [[ "${do_add_origin}" == "yes" ]]; then
|
||||
regular_git remote add origin "${url}"
|
||||
fi
|
||||
|
||||
# remote was updated, fetch and check out updates, but not tags; tags pull their respective commits too, making it a huge fetch.
|
||||
display_alert "Fetching updates from origin" "$dir $ref_name"
|
||||
display_alert "Fetching updates from remote repository" "$dir $ref_name"
|
||||
case $ref_type in
|
||||
branch | commit) improved_git_fetch --no-tags origin "${ref_name}" ;;
|
||||
tag) improved_git_fetch --no-tags origin tags/"${ref_name}" ;;
|
||||
head) improved_git_fetch --no-tags origin HEAD ;;
|
||||
branch | commit) improved_git_fetch --no-tags "${url}" "${ref_name}" ;;
|
||||
tag) improved_git_fetch --no-tags "${url}" tags/"${ref_name}" ;;
|
||||
head) improved_git_fetch --no-tags "${url}" HEAD ;;
|
||||
esac
|
||||
display_alert "Origin fetch completed, working copy size" "$(du -h -s | awk '{print $1}')" "git"
|
||||
display_alert "Remote repository fetch completed, working copy size" "$(du -h -s | awk '{print $1}')" "git"
|
||||
checkout_from="FETCH_HEAD"
|
||||
fi
|
||||
|
||||
|
||||
@ -5,33 +5,42 @@ function prepare_armbian_mountpoints_description_dict() {
|
||||
declare -g -a ARMBIAN_MOUNTPOINTS_ARRAY=(
|
||||
".tmp"
|
||||
"output" "output/images" "output/debs" "output/logs"
|
||||
"cache" "cache/gitballs" "cache/toolchain"
|
||||
"cache"
|
||||
"cache/git-bare"
|
||||
"cache/git-bundles"
|
||||
"cache/toolchain"
|
||||
"cache/aptcache"
|
||||
"cache/rootfs" "cache/initrd"
|
||||
"cache/sources" "cache/sources/linux-kernel"
|
||||
"cache/rootfs"
|
||||
"cache/initrd"
|
||||
"cache/sources"
|
||||
"cache/sources/linux-kernel-worktree"
|
||||
"cache/ccache"
|
||||
)
|
||||
|
||||
declare -A -g ARMBIAN_MOUNTPOINTS_DESC_DICT=(
|
||||
[".tmp"]="docker_kind_linux=anonymous docker_kind_darwin=anonymous" # tmpfs, discard, anonymous; whatever you wanna call it. It just needs to be 100% local to the container, and there's very little value in being able to look at it from the host.
|
||||
["output"]="docker_kind_linux=bind docker_kind_darwin=bind" # catch-all output. specific subdirs are mounted below. it's a bind mount by default on both Linux and Darwin.
|
||||
["output/images"]="docker_kind_linux=bind docker_kind_darwin=bind" # 99% of users want this as the result of their build, no matter if it's slow or not. bind on both.
|
||||
["output/debs"]="docker_kind_linux=bind docker_kind_darwin=bind" # generated output .deb files. most people are interested in this, to update kernels or dtbs after initial build. bind on both Linux and Darwin.
|
||||
["output/logs"]="docker_kind_linux=bind docker_kind_darwin=bind" # log files produced. 100% of users want this. Bind on both Linux and Darwin. Is used to integrate launcher and actual-build logs, so must exist and work otherwise confusion ensues.
|
||||
["cache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # catch-all cache, could be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/gitballs"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # tarballs of git repos, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/toolchain"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # toolchain cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/aptcache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # .deb apt cache, replaces apt-cacher-ng. Can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/rootfs"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # rootfs .tar.zst cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/initrd"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # initrd.img cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/sources"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # operating directory. many things are cloned in here, and some are even built inside. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/sources/linux-kernel"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for kernel builds. huge. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/ccache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # ccache object store. limited to 5gb by default. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
[".tmp"]="docker_kind_linux=anonymous docker_kind_darwin=anonymous" # tmpfs, discard, anonymous; whatever you wanna call it. It just needs to be 100% local to the container, and there's very little value in being able to look at it from the host.
|
||||
["output"]="docker_kind_linux=bind docker_kind_darwin=bind" # catch-all output. specific subdirs are mounted below. it's a bind mount by default on both Linux and Darwin.
|
||||
["output/images"]="docker_kind_linux=bind docker_kind_darwin=bind" # 99% of users want this as the result of their build, no matter if it's slow or not. bind on both.
|
||||
["output/debs"]="docker_kind_linux=bind docker_kind_darwin=bind" # generated output .deb files. most people are interested in this, to update kernels or dtbs after initial build. bind on both Linux and Darwin.
|
||||
["output/logs"]="docker_kind_linux=bind docker_kind_darwin=bind" # log files produced. 100% of users want this. Bind on both Linux and Darwin. Is used to integrate launcher and actual-build logs, so must exist and work otherwise confusion ensues.
|
||||
["cache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # catch-all cache, could be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/git-bare"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # Git bare repos (kernel/u-boot). On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/git-bundles"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # Downloads of git bundles, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/toolchain"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # toolchain cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/aptcache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # .deb apt cache, replaces apt-cacher-ng. Can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/rootfs"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # rootfs .tar.zst cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/initrd"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # initrd.img cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/sources"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # operating directory. many things are cloned in here, and some are even built inside. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/sources/linux-kernel-worktree"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for kernel builds. huge. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
["cache/ccache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # ccache object store. limited to 5gb by default. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default.
|
||||
)
|
||||
|
||||
# These, if found, will be removed on `dockerpurge` and other cleanups.
|
||||
# They "used to be" used by the build system, but no longer.
|
||||
declare -g -a ARMBIAN_MOUNTPOINTS_DEPRECATED=(
|
||||
"cache/gitballs"
|
||||
"cache/sources/kernel"
|
||||
"cache/sources/linux-kernel"
|
||||
)
|
||||
}
|
||||
|
||||
@ -63,10 +72,10 @@ function loop_over_armbian_mountpoints() {
|
||||
display_alert "Unsupported host OS" "${DOCKER_ARMBIAN_HOST_OS_UNAME} - cant map mountpoint to Docker volume" "warn"
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
# volume_id is the mountpoint with all slashes replaced with dashes
|
||||
local volume_id="${mountpoint//\//-}"
|
||||
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
eval "$values docker_kind=$docker_kind $func '$mountpoint'"
|
||||
done
|
||||
|
||||
@ -233,7 +233,7 @@ function early_prepare_host_dependencies() {
|
||||
pkg-config pv qemu-user-static rsync swig
|
||||
u-boot-tools udev uuid-dev whiptail
|
||||
zlib1g-dev busybox fdisk
|
||||
|
||||
|
||||
# distcc, experimental, optional; see cli-distcc.sh and kernel.sh
|
||||
distcc
|
||||
|
||||
@ -250,7 +250,7 @@ function early_prepare_host_dependencies() {
|
||||
file ccze colorized-logs tree expect # logging utilities; expect is needed for 'unbuffer' command
|
||||
unzip zip p7zip-full pigz pixz pbzip2 lzop zstd # compressors et al
|
||||
parted gdisk # partition tools
|
||||
aria2 curl wget # downloaders et al
|
||||
aria2 curl wget axel # downloaders et al
|
||||
parallel # do things in parallel
|
||||
# toolchains. NEW: using metapackages, allow us to have same list of all arches; brings both C and C++ compilers
|
||||
crossbuild-essential-armhf crossbuild-essential-armel # for ARM 32-bit, both HF and EL are needed in some cases.
|
||||
|
||||
@ -157,11 +157,11 @@ function prepare_and_config_main_build_single() {
|
||||
fi
|
||||
|
||||
# Default LINUXSOURCEDIR:
|
||||
export LINUXSOURCEDIR="linux-kernel/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}/${ARCH}"
|
||||
export LINUXSOURCEDIR="linux-kernel-worktree/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}__${ARCH}"
|
||||
|
||||
# Allow adding to it with KERNEL_EXTRA_DIR
|
||||
if [[ "${KERNEL_EXTRA_DIR}" != "" ]]; then
|
||||
export LINUXSOURCEDIR="linux-kernel/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}_${KERNEL_EXTRA_DIR}/${ARCH}"
|
||||
export LINUXSOURCEDIR="${LINUXSOURCEDIR}__${KERNEL_EXTRA_DIR}"
|
||||
display_alert "Using kernel extra dir: '${KERNEL_EXTRA_DIR}'" "LINUXSOURCEDIR: ${LINUXSOURCEDIR}" "debug"
|
||||
fi
|
||||
else
|
||||
|
||||
@ -172,6 +172,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true
|
||||
# shellcheck source=lib/functions/compilation/kernel-debs.sh
|
||||
source "${SRC}"/lib/functions/compilation/kernel-debs.sh
|
||||
|
||||
# no errors tolerated. invoked before each sourced file to make sure.
|
||||
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
|
||||
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
|
||||
set -o errtrace # trace ERR through - enabled
|
||||
set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled
|
||||
### lib/functions/compilation/kernel-git.sh
|
||||
# shellcheck source=lib/functions/compilation/kernel-git.sh
|
||||
source "${SRC}"/lib/functions/compilation/kernel-git.sh
|
||||
|
||||
# no errors tolerated. invoked before each sourced file to make sure.
|
||||
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
|
||||
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
|
||||
|
||||
Loading…
Reference in New Issue
Block a user