* add hook to allow customizing before kernel make env creation * Hook runs in docker_cli_prepare_launch() just before DOCKER_EXTRA_ARGS is processed, allowing extensions to add Docker arguments with a more descriptive hook name than add_host_dependencies. * Extension: ccache-remote Enables ccache with remote Redis storage for sharing compilation cache across build hosts. Features: - Auto-discovery via Avahi/mDNS (ccache.local hostname) - Explicit Redis server configuration via CCACHE_REMOTE_STORAGE - Build statistics display at end of build (hit/miss/error rates) - Support for both Docker and native builds - Hooks for kernel and u-boot compilation environments Documentation includes server setup instructions with security warnings, client mDNS configuration, and cache sharing requirements. * uboot: fix ccache environment and add extension hook U-Boot build uses `env -i` which clears all environment variables. CCACHE_DIR and CCACHE_TEMPDIR were not explicitly passed to make, unlike kernel build (kernel-make.sh). This caused ccache to use default directory instead of configured Armbian one, breaking cache statistics and shared cache functionality. Changes: - Add CCACHE_DIR and CCACHE_TEMPDIR to uboot_make_envs - Add uboot_make_config hook for extensions (similar to kernel_make_config), allowing modification of environment variables before compilation * add long list of allowed ccache-related env vars * set permissions to ccache files RW for everyone if cache not private * ccache: add ccache_post_compilation hook for extensions * ccache-remote: use ccache_post_compilation hook instead of cleanup handler Show remote ccache stats after each compilation (kernel, uboot) via hook, instead of once at the end via cleanup handler. Stats now shown even on build failure. * ccache: show stats with safe arithmetic * ccache/uboot: improve code comments per review feedback - uboot.sh: clarify ARMBIAN=foe workaround for dual-compiler scenario - ccache-remote.sh: document that CCACHE_REDIS_CONNECT_TIMEOUT must be set before extension loads * ccache-remote: mask storage URLs in logs Mask CCACHE_REMOTE_STORAGE when emitting Docker env debug logs. * ccache-remote: extract ccache_inject_envs() helper to deduplicate passthrough loops Extract ccache_inject_envs() helper to deduplicate identical passthrough loops in kernel and uboot make config hooks. ccache-remote: rename functions to follow project naming conventions Rename get_redis_stats and mask_storage_url to ccache_get_redis_stats and ccache_mask_storage_url to follow project naming conventions. ccache-remote: mask credentials in debug log output for passthrough loops Mask CCACHE_REMOTE_STORAGE value through ccache_mask_storage_url() before logging in both Docker env and make env passthrough loops to avoid leaking credentials into build logs. * ccache-remote: add HTTP/WebDAV backend and DNS discovery * ccache-remote: move extension script into directory layout * ccache-remote: add server setup docs and config files * ccache-remote: validate Redis credentials in URLs * ccache-remote: document Redis auth options and safe passwords Add separate insecure config example for trusted networks. Recommend URL-safe hex passwords and update setup docs. * ccache-remote: improve Docker loopback handling and IPv6 host parsing
135 lines
6.8 KiB
Bash
135 lines
6.8 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Copyright (c) 2013-2026 Igor Pecovnik, igor@armbian.com
|
|
#
|
|
# This file is a part of the Armbian Build Framework
|
|
# https://github.com/armbian/build/
|
|
|
|
function run_kernel_make_internal() {
|
|
set -e
|
|
declare -a common_make_params_quoted common_make_envs full_command
|
|
|
|
# Prepare distcc, if enabled.
|
|
declare -a -g DISTCC_EXTRA_ENVS=()
|
|
declare -a -g DISTCC_CROSS_COMPILE_PREFIX=()
|
|
declare -a -g DISTCC_MAKE_J_PARALLEL=()
|
|
prepare_distcc_compilation_config
|
|
|
|
common_make_envs=(
|
|
"CCACHE_BASEDIR='$(pwd)'" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
|
|
"CCACHE_TEMPDIR='${CCACHE_TEMPDIR:?}'" # Temporary directory for ccache, under WORKDIR
|
|
"PATH='${PYTHON3_INFO[USERBASE]}/bin:${PATH}'" # Insert the pip binaries into the PATH
|
|
"PYTHONPATH='${PYTHON3_INFO[MODULES_PATH]}:${PYTHONPATH}'" # Insert the pip modules downloaded by Armbian into PYTHONPATH (needed for dtb checks)
|
|
"DPKG_COLORS=always" # Use colors for dpkg @TODO no dpkg is done anymore, remove?
|
|
"XZ_OPT='--threads=0'" # Use parallel XZ compression
|
|
"TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work.
|
|
"COLUMNS='${COLUMNS:-160}'"
|
|
"COLORFGBG='${COLORFGBG}'"
|
|
)
|
|
|
|
# If CCACHE_DIR is set, pass it to the kernel build; Pass the ccache dir explicitly, since we'll run under "env -i"
|
|
if [[ -n "${CCACHE_DIR}" ]]; then
|
|
common_make_envs+=("CCACHE_DIR=${CCACHE_DIR@Q}")
|
|
fi
|
|
|
|
# Add the distcc envs, if any.
|
|
common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}")
|
|
|
|
if [[ "${KERNEL_COMPILER}" == "clang" ]]; then
|
|
llvm_flag="LLVM=1"
|
|
cc_name="CC"
|
|
# Only suppress unused-command-line-argument errors
|
|
# Do NOT add -Wno-error=unknown-warning-option here - it breaks cc-option detection
|
|
# in kernel Makefiles (btrfs, drm, coresight) causing GCC-specific flags to be
|
|
# incorrectly added when building with clang
|
|
extra_warnings="-fcolor-diagnostics -Wno-error=unused-command-line-argument"
|
|
else
|
|
cc_name="CROSS_COMPILE"
|
|
extra_warnings=""
|
|
fi
|
|
common_make_params_quoted=(
|
|
# @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir; this has high impact in headers packaging though.
|
|
|
|
"${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled.
|
|
|
|
"ARCH=${ARCHITECTURE}" # Key param. Everything depends on this.
|
|
"LOCALVERSION=-${BRANCH}-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change hack at .config; that might handles mtime better
|
|
|
|
"${cc_name}=${CCACHE} ${DISTCC_CROSS_COMPILE_PREFIX[@]} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make
|
|
"KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation ${extra_warnings} ${KERNEL_EXTRA_CFLAGS:-""}" # Force GCC colored messages, downgrade misleading indentation to warning
|
|
|
|
"SOURCE_DATE_EPOCH=${kernel_base_revision_ts}" # https://reproducible-builds.org/docs/source-date-epoch/ and https://www.kernel.org/doc/html/latest/kbuild/reproducible-builds.html
|
|
"KBUILD_BUILD_TIMESTAMP=${kernel_base_revision_date}" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-timestamp
|
|
"KBUILD_BUILD_USER=build" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host
|
|
"KBUILD_BUILD_HOST=armbian" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host
|
|
|
|
# Parallel compression, use explicit parallel compressors https://lore.kernel.org/lkml/20200901151002.988547791@linuxfoundation.org/
|
|
"KGZIP=pigz"
|
|
"KBZIP2=pbzip2"
|
|
# Parallel compression for `xz` if needed can be added with "XZ_OPT=\"--threads=0\""
|
|
)
|
|
if [[ -n "${llvm_flag}" ]]; then
|
|
common_make_params_quoted+=("${llvm_flag}")
|
|
fi
|
|
|
|
# Hook order: kernel_make_config runs first (generic extension config),
|
|
# then custom_kernel_make_params (user/board overrides can take precedence).
|
|
call_extension_method "kernel_make_config" <<- 'KERNEL_MAKE_CONFIG'
|
|
*Hook to customize kernel make environment and parameters*
|
|
Called right before invoking make for kernel compilation.
|
|
Available arrays to modify:
|
|
- common_make_envs[@]: environment variables passed via "env -i" (e.g., CCACHE_REMOTE_STORAGE)
|
|
- common_make_params_quoted[@]: make command parameters (e.g., custom flags)
|
|
Available read-only variables:
|
|
- KERNEL_COMPILER, ARCHITECTURE, BRANCH, LINUXFAMILY
|
|
KERNEL_MAKE_CONFIG
|
|
|
|
# Runs after kernel_make_config — allows user/board overrides to take precedence
|
|
call_extension_method "custom_kernel_make_params" <<- 'CUSTOM_KERNEL_MAKE_PARAMS'
|
|
*Customize kernel make parameters before compilation*
|
|
Called after all standard make parameters are set but before invoking make.
|
|
Extensions can modify the following arrays:
|
|
- `common_make_params_quoted` - parameters passed to make (e.g., CROSS_COMPILE_COMPAT)
|
|
- `common_make_envs` - environment variables for make
|
|
CUSTOM_KERNEL_MAKE_PARAMS
|
|
|
|
# last statement, so it passes the result to calling function. "env -i" is used for empty env
|
|
full_command=("${KERNEL_MAKE_RUNNER:-run_host_command_logged}" "env" "-i" "${common_make_envs[@]}"
|
|
make "${common_make_params_quoted[@]@Q}" "$@")
|
|
"${full_command[@]}" # and exit with it's code, since it's the last statement
|
|
}
|
|
|
|
function run_kernel_make() {
|
|
KERNEL_MAKE_RUNNER="run_host_command_logged" KERNEL_MAKE_UNBUFFER="unbuffer" run_kernel_make_internal "$@"
|
|
}
|
|
|
|
function run_kernel_make_dialog() {
|
|
KERNEL_MAKE_RUNNER="run_host_command_dialog" run_kernel_make_internal "$@"
|
|
}
|
|
|
|
function run_kernel_make_long_running() {
|
|
local seconds_start=${SECONDS} # Bash has a builtin SECONDS that is seconds since start of script
|
|
KERNEL_MAKE_UNBUFFER="unbuffer" run_kernel_make_internal "$@"
|
|
display_alert "Kernel Make '$*' took" "$((SECONDS - seconds_start)) seconds" "debug"
|
|
}
|
|
|
|
function kernel_determine_toolchain() {
|
|
# compare with the architecture of the current Debian node
|
|
if dpkg-architecture -e "${ARCH}"; then
|
|
display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
|
|
else
|
|
display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
|
|
fi
|
|
|
|
declare kernel_compiler_full kernel_compiler_version
|
|
if [[ "${KERNEL_COMPILER}" == "clang" ]]; then
|
|
kernel_compiler_full="${KERNEL_COMPILER}"
|
|
else
|
|
kernel_compiler_full="${KERNEL_COMPILER}gcc"
|
|
fi
|
|
kernel_compiler_version="$(eval env "${kernel_compiler_full}" -dumpfullversion -dumpversion)"
|
|
display_alert "Compiler version" "${kernel_compiler_full} ${kernel_compiler_version}" "info"
|
|
}
|