From 95e0b9d4843924fe03191addd0c5a44e6845630d Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Wed, 22 Jun 2022 23:57:26 +0200 Subject: [PATCH] armbian-next: remove warm/cold bundles from `fetch_from_repo`, replace with 2 invocation-specific hooks and `gitballs` (.tar of .git) - drop experiment 'caches/gitbundles' and its code; all lives now in separate repo - change LINUXSOURCEDIR **again**, cleanup the old one --- lib/functions/compilation/kernel.sh | 77 ++++++-- lib/functions/configuration/main-config.sh | 2 +- lib/functions/general/git.sh | 196 ++------------------- lib/functions/main/config-prepare.sh | 2 +- 4 files changed, 79 insertions(+), 198 deletions(-) diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index 09a24859e7..aa1b5ded96 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -95,26 +95,67 @@ function compile_kernel() { return 0 } -function kernel_prepare_git() { - if [[ -n $KERNELSOURCE ]]; then - [[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree" +function kernel_init_git_from_http_gitball() { + local kernel_git_dir="${1}" - display_alert "Downloading sources" "kernel" "git" - - # Does not work well with rpi for example: GIT_WARM_REMOTE_SHALLOW_AT_TAG="v${KERNEL_MAJOR_MINOR}" \ - # GIT_WARM_REMOTE_SHALLOW_AT_TAG sets GIT_WARM_REMOTE_SHALLOW_AT_DATE for you, as long as it is included by GIT_WARM_REMOTE_FETCH_TAGS - # GIT_WARM_REMOTE_SHALLOW_AT_DATE is the only one really used for making shallow - - GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \ - GIT_WARM_REMOTE_NAME="kernel-stable-${KERNEL_MAJOR_MINOR}" \ - GIT_WARM_REMOTE_URL="${MAINLINE_KERNEL_SOURCE}" \ - GIT_WARM_REMOTE_BRANCH="linux-${KERNEL_MAJOR_MINOR}.y" \ - GIT_WARM_REMOTE_FETCH_TAGS="v${KERNEL_MAJOR_MINOR}*" \ - GIT_WARM_REMOTE_SHALLOW_AT_TAG="${KERNEL_MAJOR_SHALLOW_TAG}" \ - GIT_WARM_REMOTE_BUNDLE="kernel-stable-${KERNEL_MAJOR_MINOR}" \ - GIT_COLD_BUNDLE_URL="${MAINLINE_KERNEL_COLD_BUNDLE_URL}" \ - fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes" + local gitball_dir="${SRC}/cache/gitballs/kernel" + if [[ ! -d "${gitball_dir}" ]]; then + display_alert "Creating kernel git gitball cache dir" "${gitball_dir}" "info" + run_host_command_logged mkdir -pv "${gitball_dir}" + [[ -d "${SRC}/cache/gitbundles" ]] && run_host_command_logged rm -rf "${SRC}/cache/gitbundles" # remove old gitbundles dir. we won't be using those anymore; @TODO: remove this line in the future fi + + local gitball_file="${gitball_dir}/linux-${KERNEL_MAJOR_MINOR}.git.tar" + local gitball_url="https://github.com/armbian/gitutils/releases/download/latest/linux-${KERNEL_MAJOR_MINOR}.git.tar" + + if [[ ! -f "${gitball_file}" ]]; then # Download the gitball file if it does not exist. + display_alert "Downloading Git cold gitball via HTTP" "${gitball_url}" "info" # This gonna take a while. And waste bandwidth + run_host_command_logged wget --continue --progress=dot:giga --output-document="${gitball_file}.tmp" "${gitball_url}" + run_host_command_logged mv -v "${gitball_file}.tmp" "${gitball_file}" + else + display_alert "Cold gitball file exists, using it" "${gitball_file}" "git" + fi + + # Extract the gitball file to the git directory. This will create '.git' + run_host_command_logged tar -xvf "${gitball_file}" -C "${kernel_git_dir}" + + # Sanity check + regular_git branch -a --list --color +} + +function kernel_prepare_git_pre_fetch() { + local remote_name="kernel-stable-${KERNEL_MAJOR_MINOR}" + local remote_url="${MAINLINE_KERNEL_SOURCE}" + local remote_tags_to_fetch="v${KERNEL_MAJOR_MINOR}*" + + # shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context. + if [[ "${do_add_origin}" == "yes" ]]; then + display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git" + regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source + + # Fetch the tags. This allows working -rcX versions of still-unreleased minor versions. + improved_git_fetch "${remote_name}" "'refs/tags/${remote_tags_to_fetch}:refs/tags/${remote_tags_to_fetch}'" || true # Fetch the remote branch tags + display_alert "After mainline stable tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull + fi +} + +function kernel_prepare_git() { + [[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then? + + # LINUXSOURCEDIR has changed a lot, cleanup old incarnations if they exist + if [[ -d "${SRC}/cache/sources/kernel" ]]; then + display_alert "Cleaning up old kernel sources directory" "might take a while" "debug" + run_host_command_logged rm -rf "${SRC}/cache/sources/kernel" + fi + + [[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree" + + display_alert "Downloading sources" "kernel" "git" + + GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \ + GIT_INIT_REPO_HOOK=kernel_init_git_from_http_gitball \ + GIT_PRE_FETCH_HOOK=kernel_prepare_git_pre_fetch_tags \ + fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes" } function kernel_maybe_clean() { diff --git a/lib/functions/configuration/main-config.sh b/lib/functions/configuration/main-config.sh index e30cfb002f..9f615fe369 100644 --- a/lib/functions/configuration/main-config.sh +++ b/lib/functions/configuration/main-config.sh @@ -120,7 +120,7 @@ function do_main_configuration() { MAINLINE_FIRMWARE_SOURCE='https://mirrors.bfsu.edu.cn/git/linux-firmware.git' ;; *) - MAINLINE_KERNEL_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git' + MAINLINE_KERNEL_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git' # "linux-stable" was renamed to "linux" MAINLINE_FIRMWARE_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git' ;; esac diff --git a/lib/functions/general/git.sh b/lib/functions/general/git.sh index 36191dc9d0..b2c3a6d58a 100644 --- a/lib/functions/general/git.sh +++ b/lib/functions/general/git.sh @@ -109,17 +109,23 @@ fetch_from_repo() { fi fi - local do_warmup_remote="no" do_cold_bundle="no" do_add_origin="no" + local do_add_origin="no" if [[ "$(git rev-parse --git-dir)" != ".git" ]]; then - # Dir is not a git working copy. Make it so. - display_alert "Creating local copy" "$dir $ref_name" - regular_git init -q --initial-branch="armbian_unused_initial_branch" . + # Dir is not a git working copy. Make it so; + # If callback is defined, call it. Give it the dir as param. The rest it will read from environment. + # If not callback defined, do an init, and schedule a fetch. + + if [[ $(type -t ${GIT_INIT_REPO_HOOK} || true) == function ]]; then + display_alert "Delegating to ${GIT_INIT_REPO_HOOK}()" "git init: $dir $ref_name" "debug" + ${GIT_INIT_REPO_HOOK} "${git_work_dir}" + else + display_alert "Initializing empty git local copy" "git init: $dir $ref_name" + regular_git init -q --initial-branch="armbian_unused_initial_branch" . + fi + offline=false # Force online, we'll need to fetch. do_add_origin="yes" # Just created the repo, it needs an origin later. - do_warmup_remote="yes" # Just created the repo, mark it as ready to receive the warm remote if exists. - do_cold_bundle="yes" # Just created the repo, mark it as ready to receive a cold bundle if that is available. - # @TODO: possibly hang a cleanup handler here: if this fails, ${git_work_dir} should be removed. fi local changed=false @@ -163,7 +169,11 @@ fetch_from_repo() { local checkout_from="HEAD" # Probably best to use the local revision? if [[ "${changed}" == "true" ]]; then - git_handle_cold_and_warm_bundle_remotes # Delegate to function to find or create cache if appropriate. + + if [[ $(type -t ${GIT_PRE_FETCH_HOOK} || true) == function ]]; then + display_alert "Delegating to ${GIT_PRE_FETCH_HOOK}()" "before git fetch" "debug" + ${GIT_PRE_FETCH_HOOK} "${git_work_dir}" "${url}" "$ref_type" "$ref_name" + fi if [[ "${do_add_origin}" == "yes" ]]; then regular_git remote add origin "${url}" @@ -193,11 +203,6 @@ fetch_from_repo() { #fasthash_debug "before git clean of $dir $ref_name" regular_git clean -q -d -f # Files that are not tracked by git and were added when the patch was applied must be removed. - # set the checkout date on all the versioned files. - # @TODO: this is contentious. disable for now. patches will still use the mininum date set by checked_out_revision_mtime above - #git ls-tree -r -z --name-only "${checkout_from}" | xargs -0 -- touch -m -t "${checked_out_revision_mtime:0:12}.${checked_out_revision_mtime:12}" - #fasthash_debug "after setting checkout time for $dir $ref_name" #yeah - if [[ -f .gitmodules ]]; then if [[ "${GIT_SKIP_SUBMODULES}" == "yes" ]]; then display_alert "Skipping submodules" "GIT_SKIP_SUBMODULES=yes" "debug" @@ -224,168 +229,3 @@ fetch_from_repo() { #fasthash_debug "at the end of fetch_from_repo $dir $ref_name" } -function git_fetch_from_bundle_file() { - local bundle_file="${1}" remote_name="${2}" shallow_file="${3}" - regular_git bundle verify "${bundle_file}" # Make sure bundle is valid. - regular_git remote add "${remote_name}" "${bundle_file}" # Add the remote pointing to the cold bundle file - if [[ -f "${shallow_file}" ]]; then - display_alert "Bundle is shallow" "${shallow_file}" "git" - cp -p "${shallow_file}" ".git/shallow" - fi - improved_git_fetch --tags "${remote_name}" # Fetch it! (including tags!) - display_alert "Bundle fetch '${remote_name}' completed, working copy size" "$(du -h -s | awk '{print $1}')" "git" -} - -function download_git_bundle_from_http() { - local bundle_file="${1}" bundle_url="${2}" - if [[ ! -f "${git_cold_bundle_cache_file}" ]]; then # Download the bundle file if it does not exist. - display_alert "Downloading Git cold bundle via HTTP" "${bundle_url}" "info" # This gonna take a while. And waste bandwidth - run_host_command_logged wget --continue --progress=dot:giga --output-document="${bundle_file}" "${bundle_url}" - else - display_alert "Cold bundle file exists, using it" "${bundle_file}" "git" - fi -} - -function git_remove_cold_and_warm_bundle_remotes() { - # Remove the cold bundle remote, otherwise it holds references that impede the shallow to actually work. - if [[ ${has_git_cold_remote} -gt 0 ]]; then - regular_git remote remove "${git_cold_bundle_remote_name}" - has_git_cold_remote=0 - fi - - # Remove the warmup remote, otherwise it holds references forever. - if [[ ${has_git_warm_remote} -gt 0 ]]; then - regular_git remote remove "${GIT_WARM_REMOTE_NAME}" - has_git_warm_remote=0 - fi -} - -function git_handle_cold_and_warm_bundle_remotes() { - - local has_git_cold_remote=0 - local has_git_warm_remote=0 - local git_warm_remote_bundle_file git_warm_remote_bundle_cache_dir git_warm_remote_bundle_file_shallowfile - local git_warm_remote_bundle_extra_fn="" - - # First check the warm remote bundle cache. If that exists, use that, and skip the cold bundle. - if [[ "${do_warmup_remote}" == "yes" ]]; then - if [[ "${GIT_WARM_REMOTE_NAME}" != "" ]] && [[ "${GIT_WARM_REMOTE_BUNDLE}" != "" ]]; then - # Add extras to filename, for shallow by tag or revision - if [[ "${GIT_WARM_REMOTE_SHALLOW_REVISION}" != "" ]]; then - git_warm_remote_bundle_extra_fn="-shallow-rev-${GIT_WARM_REMOTE_SHALLOW_REVISION}" - elif [[ "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" != "" ]]; then - git_warm_remote_bundle_extra_fn="-shallow-tag-${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" - fi - git_warm_remote_bundle_cache_dir="${SRC}/cache/gitbundles/warm" # calculate the id, dir and name of local file and remote - git_warm_remote_bundle_file="${git_warm_remote_bundle_cache_dir}/${GIT_WARM_REMOTE_BUNDLE}${git_warm_remote_bundle_extra_fn}.gitbundle" # final filename of bundle - git_warm_remote_bundle_file_shallowfile="${git_warm_remote_bundle_file}.shallow" # it can be there's a shallow revision - if [[ -f "${git_warm_remote_bundle_file}" ]]; then - display_alert "Fetching from warm git bundle, wait" "${GIT_WARM_REMOTE_BUNDLE}" "info" # This is gonna take a long while... - git_fetch_from_bundle_file "${git_warm_remote_bundle_file}" "${GIT_WARM_REMOTE_NAME}" "${git_warm_remote_bundle_file_shallowfile}" - do_cold_bundle="no" # Skip the cold bundle, below. - do_warmup_remote="no" # Skip the warm bundle creation, below, too. - has_git_warm_remote=1 # mark warm remote as added. - else - display_alert "Could not find warm bundle file" "${git_warm_remote_bundle_file}" "git" - fi - fi - fi - - if [[ "${do_cold_bundle}" == "yes" ]]; then - # If there's a cold bundle URL specified: - # - if there's already a cold_bundle_xxx remote, move on. - # - grab the bundle via http/https first, and fetch from that, into "cold_bundle_xxx" remote. - # - do nothing else with this, it'll be used internally by git to avoid a huge fetch later. - # - but, after this, the wanted branch will be fetched. signal has_git_cold_remote=1 for later. - if [[ "${GIT_COLD_BUNDLE_URL}" != "" ]]; then - local git_cold_bundle_id git_cold_bundle_cache_dir git_cold_bundle_cache_file git_cold_bundle_remote_name - git_cold_bundle_cache_dir="${SRC}/cache/gitbundles/cold" # calculate the id, dir and name of local file and remote - git_cold_bundle_id="$(echo -n "${GIT_COLD_BUNDLE_URL}" | md5sum | awk '{print $1}')" # md5 of the URL. - git_cold_bundle_cache_file="${git_cold_bundle_cache_dir}/${git_cold_bundle_id}.gitbundle" # final filename of bundle - git_cold_bundle_remote_name="cold_bundle_${git_cold_bundle_id}" # name of the remote that will point to bundle - mkdir -p "${git_cold_bundle_cache_dir}" # make sure directory exists before downloading - download_git_bundle_from_http "${git_cold_bundle_cache_file}" "${GIT_COLD_BUNDLE_URL}" - display_alert "Fetching from cold git bundle, wait" "${git_cold_bundle_id}" "info" # This is gonna take a long while... - git_fetch_from_bundle_file "${git_cold_bundle_cache_file}" "${git_cold_bundle_remote_name}" - has_git_cold_remote=1 # marker for pruning logic below - fi - fi - - # If there's a warmup remote specified. - # - if there's a cached warmup bundle file, add it as remote and fetch from it, and move on. - # - add the warmup as remote, fetch from it; export it as a cached bundle for next time. - if [[ "${do_warmup_remote}" == "yes" ]]; then - if [[ "${GIT_WARM_REMOTE_NAME}" != "" ]] && [[ "${GIT_WARM_REMOTE_URL}" != "" ]] && [[ "${GIT_WARM_REMOTE_BRANCH}" != "" ]]; then - - display_alert "Using Warmup Remote before origin fetch" "${GIT_WARM_REMOTE_NAME} - ${GIT_WARM_REMOTE_BRANCH}" "git" - regular_git remote add "${GIT_WARM_REMOTE_NAME}" "${GIT_WARM_REMOTE_URL}" # Add the remote to the warmup source - has_git_warm_remote=1 # mark as done. Will export the bundle! - - improved_git_fetch --no-tags "${GIT_WARM_REMOTE_NAME}" "${GIT_WARM_REMOTE_BRANCH}" # Fetch the remote branch, but no tags - display_alert "After warm bundle, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull - - # Checkout that to a branch. We wanna have a local reference to what has been fetched. - # @TODO: could be a param instead of FETCH_HEAD; would drop commits after that rev - local git_warm_branch_name="warm__${GIT_WARM_REMOTE_BRANCH}" - regular_git branch "${git_warm_branch_name}" FETCH_HEAD || true - - improved_git_fetch "${GIT_WARM_REMOTE_NAME}" "'refs/tags/${GIT_WARM_REMOTE_FETCH_TAGS}:refs/tags/${GIT_WARM_REMOTE_FETCH_TAGS}'" || true # Fetch the remote branch, but no tags - display_alert "After warm bundle tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull - - # Lookup the tag (at the warm remote directly) to find the rev to shallow to. - if [[ "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" != "" ]]; then - display_alert "GIT_WARM_REMOTE_SHALLOW_AT_TAG" "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" "git" - GIT_WARM_REMOTE_SHALLOW_AT_DATE="$(git tag --list --format="%(creatordate)" "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}")" - display_alert "GIT_WARM_REMOTE_SHALLOW_AT_TAG ${GIT_WARM_REMOTE_SHALLOW_AT_TAG} resulted in GIT_WARM_REMOTE_SHALLOW_AT_DATE" "Date: ${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" "git" - fi - - # At this stage, we might wanna make the local copy shallow and re-pack it. - if [[ "${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" != "" ]]; then - display_alert "Making working copy shallow" "before date ${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" "info" - - # 'git clone' is the only consistent, usable thing we can do to do this. - # it does require a temporary dir, though. use one. - - local temp_git_dir="${git_work_dir}.making.shallow.temp" - rm -rf "${temp_git_dir}" - - regular_git clone --no-checkout --progress --verbose \ - --single-branch --branch="${git_warm_branch_name}" \ - --tags --shallow-since="${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" \ - "file://${git_work_dir}" "${temp_git_dir}" - - display_alert "After shallow clone, temp_git_dir" "$(du -h -s "${temp_git_dir}" | awk '{print $1}')" "git" # Show size after shallow - - # Get rid of original, replace with new. Move cwd so no warnings are produced. - cd "${SRC}" || exit_with_error "Failed to move cwd away so we can remove" "${git_work_dir}" - rm -rf "${git_work_dir}" - mv -v "${temp_git_dir}" "${git_work_dir}" - cd "${git_work_dir}" || exit_with_error "Failed to get new dir after clone" "${git_work_dir}" - - # dir switched, no more the original remotes. but origin is leftover, remove it - regular_git remote remove origin || true - has_git_cold_remote=0 - has_git_warm_remote=0 - - display_alert "After shallow, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after shallow - fi - - # Now git working copy has a precious state we might wanna preserve (export the bundle). - if [[ "${GIT_WARM_REMOTE_BUNDLE}" != "" ]]; then - mkdir -p "${git_warm_remote_bundle_cache_dir}" - display_alert "Exporting warm remote bundle" "${git_warm_remote_bundle_file}" "info" - regular_git bundle create "${git_warm_remote_bundle_file}" --all - - rm -f "${git_warm_remote_bundle_file_shallowfile}" # not shallow at first... - if [[ -f ".git/shallow" ]]; then - display_alert "Exported bundle is shallow" "Will copy to ${git_warm_remote_bundle_file_shallowfile}" "git" - cp -p ".git/shallow" "${git_warm_remote_bundle_file_shallowfile}" - fi - - fi - fi - fi - - # Make sure to remove the cold and warm bundle remote, otherwise it holds references for no good reason. - git_remove_cold_and_warm_bundle_remotes -} diff --git a/lib/functions/main/config-prepare.sh b/lib/functions/main/config-prepare.sh index 6ad4331549..d2f23bd1a5 100644 --- a/lib/functions/main/config-prepare.sh +++ b/lib/functions/main/config-prepare.sh @@ -155,7 +155,7 @@ function prepare_and_config_main_build_single() { exit_with_error "Kernel series unsupported" "'${KERNEL_MAJOR_MINOR}' is unsupported, or bad config" fi - export LINUXSOURCEDIR="kernel/${ARCH}__${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}" + export LINUXSOURCEDIR="linux-kernel/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}/${ARCH}" else export KERNEL_HAS_WORKING_HEADERS="yes" # I assume non-Armbian kernels have working headers, eg: Debian/Ubuntu generic do. export ARMBIAN_WILL_BUILD_KERNEL=no