From 770d508ca016f891f2fd135bfde6d1cb47123647 Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Sat, 24 Dec 2022 15:33:59 +0100 Subject: [PATCH] armbian-next: Python patching delusion pt 5 + EXTRAWIFI patch generator/harness + consistent patch mtime targets - Python patching: tune some logging all-around - Python patching: *FINALLY* set the dates on the patched files to `max(patch_date, root_makefile_date)` thus making lighting-fast rebuilds again - new EXTRAWIFI patch generator harness; Python patching EXTRA_PATCH_FILES_FIRST - Python patching: use temp file for patching rejects; clean it up - new EXTRAWIFI patch generator harness; Python: mark drivers as autogen, don't split or parse them, read as bytes, apply bytes directly - new EXTRAWIFI patch generator harness; somewhat-works, but patches are per-family - Python patching: add the `cache/patch` as a mountpoint - Darwin Docker performance is 20x with namedvolume; the cached patches are very large --- lib/functions/compilation/kernel-patching.sh | 15 +- lib/functions/compilation/kernel.sh | 10 +- .../compilation/patch/drivers-harness.sh | 140 ++++++++++++++++++ lib/functions/host/mountpoints.sh | 2 + lib/tools/common/armbian_utils.py | 6 +- lib/tools/common/patching_utils.py | 123 ++++++++++++--- lib/tools/patching.py | 41 ++++- 7 files changed, 301 insertions(+), 36 deletions(-) create mode 100644 lib/functions/compilation/patch/drivers-harness.sh diff --git a/lib/functions/compilation/kernel-patching.sh b/lib/functions/compilation/kernel-patching.sh index bc7c24d386..f31124174e 100644 --- a/lib/functions/compilation/kernel-patching.sh +++ b/lib/functions/compilation/kernel-patching.sh @@ -3,6 +3,9 @@ function kernel_main_patching_python() { prepare_pip_packages_for_python_tools + # outer scope variables + declare -I kernel_drivers_patch_file kernel_drivers_patch_hash + declare patch_debug="${SHOW_DEBUG:-${DEBUG_PATCHING:-"no"}}" declare temp_file_for_output="$(mktemp)" # Get a temporary file for the output. # array with all parameters; will be auto-quoted by bash's @Q modifier below @@ -13,9 +16,9 @@ function kernel_main_patching_python() { "ASSET_LOG_BASE=$(print_current_asset_log_base_file)" # base file name for the asset log; to write .md summaries. "PATCH_TYPE=kernel" # or, u-boot, or, atf "PATCH_DIRS_TO_APPLY=${KERNELPATCHDIR}" # A space-separated list of directories to apply... - "BOARD=" # BOARD is needed for the patchset selection logic; mostly for u-boot. empty for kernel. - "TARGET=" # TARGET is need for u-boot's SPI/SATA etc selection logic. empty for kernel "USERPATCHES_PATH=${USERPATCHES_PATH}" # Needed to find the userpatches. + #"BOARD=" # BOARD is needed for the patchset selection logic; mostly for u-boot. empty for kernel. + #"TARGET=" # TARGET is need for u-boot's SPI/SATA etc selection logic. empty for kernel # What to do? "APPLY_PATCHES=yes" # Apply the patches to the filesystem. Does not imply git commiting. If no, still exports the hash. "PATCHES_TO_GIT=${PATCHES_TO_GIT:-no}" # Commit to git after applying the patches. @@ -30,6 +33,9 @@ function kernel_main_patching_python() { # Pass the maintainer info, used for commits. "MAINTAINER_NAME=${MAINTAINER}" # Name of the maintainer "MAINTAINER_EMAIL=${MAINTAINERMAIL}" # Email of the maintainer + # Pass in the drivers extra patches and hashes; will be applied _first_, before series. + "EXTRA_PATCH_FILES_FIRST=${kernel_drivers_patch_file}" # Is a space-separated list. + "EXTRA_PATCH_HASHES_FIRST=${kernel_drivers_patch_hash}" # Is a space-separated list. ) display_alert "Calling Python patching script" "for kernel" "info" run_host_command_logged env -i "${params_quoted[@]@Q}" python3 "${SRC}/lib/tools/patching.py" @@ -41,6 +47,11 @@ function kernel_main_patching_python() { } function kernel_main_patching() { + # kernel_drivers_create_patches will fill the variables below + declare kernel_drivers_patch_file kernel_drivers_patch_hash + LOG_SECTION="kernel_drivers_create_patches" do_with_logging do_with_hooks kernel_drivers_create_patches "${kernel_work_dir}" "${kernel_git_revision}" + + # Python patching will git reset to the kernel SHA1 git revision, and remove all untracked files. LOG_SECTION="kernel_main_patching_python" do_with_logging do_with_hooks kernel_main_patching_python # The old way... diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index f0e5ed1fe4..342929e636 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -3,13 +3,6 @@ function compile_kernel() { local kernel_work_dir="${SRC}/cache/sources/${LINUXSOURCEDIR}" display_alert "Kernel build starting" "${LINUXSOURCEDIR}" "info" - # Extension hook: fetch_sources_for_kernel_driver - call_extension_method "fetch_sources_for_kernel_driver" <<- 'FETCH_SOURCES_FOR_KERNEL_DRIVER' - *fetch external drivers from source, before fetching kernel git sources* - Do your kernel driver fetching from external sources here. - `${kernel_work_dir}` is set, but not yet populated with kernel sources. - FETCH_SOURCES_FOR_KERNEL_DRIVER - # Prepare the git bare repo for the kernel; shared between all kernel builds declare kernel_git_bare_tree # alternative # LOG_SECTION="kernel_prepare_bare_repo_from_bundle" do_with_logging_unless_user_terminal do_with_hooks \ @@ -156,7 +149,7 @@ function kernel_build_and_package() { install_make_params_quoted+=("${value}") done - display_alert "Building kernel" "${LINUXCONFIG} ${build_targets[*]}" "info" + display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets[*]}" "info" fasthash_debug "build" make_filter="| grep --line-buffered -v -e 'LD' -e 'AR' -e 'INSTALL' -e 'SIGN' -e 'XZ' " \ do_with_ccache_statistics \ @@ -164,6 +157,7 @@ function kernel_build_and_package() { fasthash_debug "build" cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" + display_alert "Packaging kernel" "${LINUXFAMILY} ${LINUXCONFIG}" "info" prepare_kernel_packaging_debs "${kernel_work_dir}" "${kernel_dest_install_dir}" "${version}" kernel_install_dirs display_alert "Kernel built and packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info" diff --git a/lib/functions/compilation/patch/drivers-harness.sh b/lib/functions/compilation/patch/drivers-harness.sh new file mode 100644 index 0000000000..c1d8674655 --- /dev/null +++ b/lib/functions/compilation/patch/drivers-harness.sh @@ -0,0 +1,140 @@ +function calculate_hash_for_files() { + declare -a hashes=() + for file in "$@"; do + hash="$(sha256sum "${file}" | cut -d' ' -f1)" + hashes+=("$hash") + done + hash_files="$(echo "${hashes[@]}" | sha256sum | cut -d' ' -f1)" # now, hash the hashes + hash_files="${hash_files:0:16}" # shorten it to 16 characters + display_alert "Hash for files:" "$hash_files" "debug" +} + +function kernel_drivers_create_patches() { + declare kernel_work_dir="${1}" + declare kernel_git_revision="${2}" + display_alert "Creating patches for kernel drivers" "version: '${KERNEL_MAJOR_MINOR}' kernel_work_dir:'${kernel_work_dir}'" "info" + + declare hash_files # any changes in these two files will trigger a cache miss. + calculate_hash_for_files "${SRC}/lib/functions/compilation/patch/drivers_network.sh" "${SRC}/lib/functions/compilation/patch/drivers-harness.sh" + + declare cache_key_base="${KERNEL_MAJOR_MINOR}_${LINUXFAMILY}" + declare cache_key="${cache_key_base}_${hash_files}" + display_alert "Cache key base:" "$cache_key_base" "debug" + display_alert "Cache key:" "$cache_key" "debug" + + declare cache_dir_base="${SRC}/cache/patch/kernel-drivers" + mkdir -p "${cache_dir_base}" + + declare cache_target_file="${cache_dir_base}/${cache_key}.patch" + + # outer scope variables: + kernel_drivers_patch_file="${cache_target_file}" + kernel_drivers_patch_hash="${cache_key}" + + # If the target file exists, we can skip the patch creation. + if [[ -f "${cache_target_file}" ]]; then + display_alert "Using cached drivers patch file for ${LINUXFAMILY}-${KERNEL_MAJOR_MINOR}" "${cache_key}" "cachehit" + return + fi + + # if it does _not_ exist, fist clear the base, so no old patches are left over + run_host_command_logged rm -fv "${cache_dir_base}/${cache_key_base}*" + + # since it does not exist, go create it. this requires working tree. + declare target_patch_file="${cache_target_file}" + + # grab the date of the kernel kernel_git_revision into kernel_driver_commit_date, which will be used to commit later + declare kernel_driver_commit_date + kernel_driver_commit_date=$(git -C "$kernel_work_dir" show -s --format=%ci "$kernel_git_revision") + display_alert "Kernel driver commit date" "$kernel_driver_commit_date" "debug" + + display_alert "Preparing patch for drivers" "version: ${KERNEL_MAJOR_MINOR} kernel_work_dir: ${kernel_work_dir}" "info" + + kernel_drivers_prepare_harness "${kernel_work_dir}" "${kernel_git_revision}" +} + +function kernel_drivers_prepare_harness() { + declare kernel_work_dir="${1}" + declare kernel_git_revision="${2}" + declare -I kernel_driver_commit_date target_patch_file # outer scope variables + + declare -a drivers=( + driver_rtl8152_rtl8153 + driver_rtl8189ES + driver_rtl8189FS + driver_rtl8192EU + driver_rtl8811_rtl8812_rtl8814_rtl8821 + driver_xradio_xr819 + driver_rtl8811CU_rtl8821C + driver_rtl8188EU_rtl8188ETV + driver_rtl88x2bu + driver_rtl88x2cs + driver_rtl8822cs_bt + driver_rtl8723DS + driver_rtl8723DU + driver_rtl8822BS + ) + + # change cwd to the kernel working dir + cd "${kernel_work_dir}" || exit_with_error "Failed to change directory to ${kernel_work_dir}" + + #run_host_command_logged git status + run_host_command_logged git reset --hard "${kernel_git_revision}" + # git: remove tracked files, but not those in .gitignore + run_host_command_logged git clean -fd # no -x here + + for driver in "${drivers[@]}"; do + display_alert "Preparing driver" "${driver}" "info" + + # reset variables used by each driver + declare version="${KERNEL_MAJOR_MINOR}" + declare kernel_work_dir="${1}" + declare kernel_git_revision="${2}" + # for compatibility with `master`-based code + declare kerneldir="${kernel_work_dir}" + declare EXTRAWIFI="yes" # forced! @TODO not really? + + # change cwd to the kernel working dir + cd "${kernel_work_dir}" || exit_with_error "Failed to change directory to ${kernel_work_dir}" + + # invoke the driver; non-armbian-next code. + "${driver}" + + # recover from possible cwd changes in the driver code + cd "${kernel_work_dir}" || exit_with_error "Failed to change directory to ${kernel_work_dir}" + done + + # git: check if there are modifications + if [[ -n "$(git status --porcelain)" ]]; then + display_alert "Drivers have modifications" "exporting patch into ${target_patch_file}" "info" + export_changes_as_patch_via_git_format_patch + else + exit_with_error "Applying drivers didn't produce changes." + fi +} + +function export_changes_as_patch_via_git_format_patch() { + # git: add all modifications + run_host_command_logged git add . "&>/dev/null" + + # git: commit the changes + declare -a commit_params=( + -m "drivers for ${LINUXFAMILY} version ${KERNEL_MAJOR_MINOR}" + --date="${kernel_driver_commit_date}" + --author="${MAINTAINER} <${MAINTAINERMAIL}>" + ) + GIT_COMMITTER_NAME="${MAINTAINER}" GIT_COMMITTER_EMAIL="${MAINTAINERMAIL}" git commit "${commit_params[@]}" &> /dev/null + + # export the commit as a patch; first to a temporary file, then move it to the target location if they're not the same + declare formatpatch_params=( + "-1" "--stdout" + "--unified=3" # force 3 lines of diff context + "--keep-subject" # do not add a prefix to the subject "[PATCH] " + "--no-encode-email-headers" # do not encode email headers + '--signature' "Armbian generated patch from drivers for kernel ${version} and family ${LINUXFAMILY}" + '--stat=120' # 'wider' stat output; default is 80 + '--stat-graph-width=10' # shorten the diffgraph graph part, it's too long + "--zero-commit" # Output an all-zero hash in each patch’s From header instead of the hash of the commit. + ) + git format-patch "${formatpatch_params[@]}" > "${target_patch_file}" +} diff --git a/lib/functions/host/mountpoints.sh b/lib/functions/host/mountpoints.sh index d4002a9541..e414698fc2 100644 --- a/lib/functions/host/mountpoints.sh +++ b/lib/functions/host/mountpoints.sh @@ -16,6 +16,7 @@ function prepare_armbian_mountpoints_description_dict() { "cache/sources/linux-kernel-worktree" "cache/sources/u-boot-worktree" "cache/ccache" + "cache/patch" ) declare -A -g ARMBIAN_MOUNTPOINTS_DESC_DICT=( @@ -35,6 +36,7 @@ function prepare_armbian_mountpoints_description_dict() { ["cache/sources/linux-kernel-worktree"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for kernel builds. huge. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. ["cache/sources/u-boot-worktree"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for u-boot. large. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. ["cache/ccache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # ccache object store. limited to 5gb by default. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/patch"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # auto-generated patches (for kernel drivers, etc); large patches so keep it as local as possible. ) # These, if found, will be removed on `dockerpurge` and other cleanups. diff --git a/lib/tools/common/armbian_utils.py b/lib/tools/common/armbian_utils.py index e48458f95e..19debabd17 100644 --- a/lib/tools/common/armbian_utils.py +++ b/lib/tools/common/armbian_utils.py @@ -54,9 +54,9 @@ def setup_logging(): level = "DEBUG" format = "%(message)s" styles = { - 'trace': {'color': 'white', }, - 'debug': {'color': 'white'}, - 'info': {'color': 'white', 'bold': True}, + 'trace': {'color': 'white', 'bold': False}, + 'debug': {'color': 'white', 'bold': False}, + 'info': {'color': 'green', 'bold': True}, 'warning': {'color': 'yellow', 'bold': True}, 'error': {'color': 'red'}, 'critical': {'bold': True, 'color': 'red'} diff --git a/lib/tools/common/patching_utils.py b/lib/tools/common/patching_utils.py index 79bdc92c3f..4190f5a1dd 100644 --- a/lib/tools/common/patching_utils.py +++ b/lib/tools/common/patching_utils.py @@ -5,11 +5,13 @@ import mailbox import os import re import subprocess +import tempfile import git # GitPython from unidecode import unidecode from unidiff import PatchSet +REGEX_PATCH_FILENAMES = r"^patching file \"(.+)\"" log: logging.Logger = logging.getLogger("patching_utils") @@ -36,6 +38,7 @@ class PatchDir: self.root_type = self.patch_root_dir.root_type self.sub_type = self.patch_sub_dir.sub_type self.patch_files: list[PatchFileInDir] = [] + self.is_autogen_dir: bool = False def __str__(self) -> str: return "" @@ -117,6 +120,18 @@ class PatchFileInDir: return os.path.join(self.patch_dir.rel_dir, self.file_name) def split_patches_from_file(self) -> list["PatchInPatchFile"]: + # Hack: for autogen dirs, we just need to be as fast as possible, don't parse anything. + if self.patch_dir.is_autogen_dir: + contents_bytes = read_file_as_bytes(self.full_file_path()) + # @TODO: date? + bare_patch = PatchInPatchFile( + self, 1, "", f"Autogenerated patch", + f"Armbian Autopatcher ", + f"[AUTOGEN] {self.relative_dirs_and_base_file_name}", None) + bare_patch.diff_bytes = contents_bytes + log.warning(f"Patch file {self.full_file_path()} is autogenerated.") + return [bare_patch] + counter: int = 1 mbox: mailbox.mbox = mailbox.mbox(self.full_file_path()) is_invalid_mbox: bool = False @@ -199,6 +214,13 @@ def shorten_patched_file_name_for_stats(path): return os.path.basename(path) +def parse_patch_stdout_for_files(stdout_output: str): + # run the REGEX_PATCH_FILENAMES on the output; get the group 1 (the filename) for each match + ret: list[str] = re.findall(REGEX_PATCH_FILENAMES, stdout_output, re.MULTILINE) + # log.debug(f"Found {len(ret)} patched files in patch output: {','.join(ret)}.") + return ret + + class PatchInPatchFile: def __init__(self, parent: PatchFileInDir, counter: int, diff: str, desc, from_hdr, sbj_hdr, date_hdr): @@ -206,10 +228,12 @@ class PatchInPatchFile: self.applied_ok: bool = False self.rewritten_patch: str | None = None self.git_commit_hash: str | None = None + self.actually_patched_files: list[str] = [] self.parent: PatchFileInDir = parent self.counter: int = counter - self.diff: str = diff + self.diff: str | None = diff + self.diff_bytes: bytes | None = None self.failed_to_parse: bool = False @@ -254,15 +278,22 @@ class PatchInPatchFile: return f"(+{self.total_additions}/-{self.total_deletions})[{', '.join(operations)}]" def parse_patch(self): - # parse the patch, using the unidiff package - try: - patch = PatchSet(self.diff, encoding=None) - except Exception as e: - self.problems.append("invalid_diff") - self.failed_to_parse = True - log.error( - f"Failed to parse unidiff for file {self.parent.full_file_path()}(:{self.counter}): {str(e).strip()}") - return # no point in continuing; the patch is invalid; might be recovered during apply + # Hack: don't parse if autogenned; this could also be "don't parse if larger than X megabytes" since + # large patches cause trouble + if self.parent.patch_dir.is_autogen_dir: + log.warning( + f"Skipping parsing of auto-generated patch {self.counter} in file {self.parent.full_file_path()}") + return + else: + # parse the patch, using the unidiff package + try: + patch = PatchSet(self.diff, encoding=None) + except Exception as e: + self.problems.append("invalid_diff") + self.failed_to_parse = True + log.error( + f"Failed to parse unidiff for file {self.parent.full_file_path()}(:{self.counter}): {str(e).strip()}") + return # no point in continuing; the patch is invalid; might be recovered during apply self.total_additions = 0 self.total_deletions = 0 @@ -315,20 +346,38 @@ class PatchInPatchFile: os.remove(full_path) # Use the 'patch' utility to apply the patch. + if self.diff_bytes is None: + real_input = self.diff.encode("utf-8") + else: + real_input = self.diff_bytes + + # create a temporary filename (don't create the file yet: patch will maybe create it) + rejects_file = tempfile.mktemp() + #log.debug(f"Rejects file is going to be '{rejects_file}'...") + proc = subprocess.run( - ["patch", "--batch", "-p1", "-N", "--reject-file=patching.rejects"], + ["patch", "--batch", "-p1", "-N", f"--reject-file={rejects_file}", "--quoting-style=c"], cwd=working_dir, - input=self.diff.encode("utf-8"), + input=real_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) # read the output of the patch command stdout_output = proc.stdout.decode("utf-8").strip() stderr_output = proc.stderr.decode("utf-8").strip() - if stdout_output != "": - log.debug(f"patch stdout: {stdout_output}") - if stderr_output != "": - log.warning(f"patch stderr: {stderr_output}") + # if stdout_output != "": + # log.debug(f"patch stdout: {stdout_output}") + # if stderr_output != "": + # log.warning(f"patch stderr: {stderr_output}") + + # Check if the rejects exists: + if os.path.exists(rejects_file): + log.warning(f"Rejects file {rejects_file} exists.") + # Show its contents + with open(rejects_file, "r") as f: + log.warning(f"Rejects file contents: {f.read()}") + # delete it + os.remove(rejects_file) # Look at stdout. If it contains: if " (offset" in stdout_output or " with fuzz " in stdout_output: @@ -339,6 +388,11 @@ class PatchInPatchFile: log.warning(f"Patch {self} needs review: can't find file to patch.") self.problems.append("missing_file") + # parse the stdout output for the files actually patched. + if options["set_patch_date"]: + self.actually_patched_files = parse_patch_stdout_for_files(stdout_output) + self.apply_patch_date_to_files(working_dir, options) + # Check if the exit code is not zero and bomb if proc.returncode != 0: # prefix each line of the stderr_output with "STDERR: ", then join again @@ -426,6 +480,9 @@ class PatchInPatchFile: if self.parent.from_series: ret.append(f" 📜 ") + if self.parent.patch_dir.is_autogen_dir: + ret.append(f" 🤖 ") + if len(self.problems) == 0: ret.append("✅ ") @@ -433,6 +490,8 @@ class PatchInPatchFile: if problem in ["not_mbox", "needs_rebase"]: # warning emoji ret.append(f"⚠️`[{problem}]` ") + elif problem in ["autogen"]: + ret.append(f"ℹ️`[{problem}]` ") else: ret.append(f"❌`[{problem}]` ") @@ -485,6 +544,26 @@ class PatchInPatchFile: ret.append(f"`{patch_name}`") return " ".join(ret) + def apply_patch_date_to_files(self, working_dir, options): + # The date applied to the patched files is: + # 1) The date of the root Makefile + # 2) The date of the patch file + # And date is 2, unless 1 is higher. + patch_mtime = os.path.getmtime(self.parent.full_file_path()) + makefile_mtime = options["root_makefile_date"] + final_mtime = makefile_mtime + if patch_mtime > makefile_mtime: + log.debug(f"Patch {self.parent.full_file_path()} is newer than root Makefile, using patch date") + final_mtime = patch_mtime + else: + log.warn( + f"Root Makefile is newer than patch '{self.parent.full_file_path()}', using Makefile date") + # Apply the date to all files that were touched by the patch + for file_name in self.actually_patched_files: + # log.debug(f"Setting mtime of '{file_name}' to '{final_mtime}'.") + file_path = os.path.join(working_dir, file_name) + os.utime(file_path, (final_mtime, final_mtime)) + def fix_patch_subject(subject): # replace newlines with one space @@ -547,10 +626,6 @@ def export_commit_as_patch(repo: git.Repo, commit: str): # read the output of the patch command stdout_output = proc.stdout.decode("utf-8") stderr_output = proc.stderr.decode("utf-8") - # if stdout_output != "": - # print(f"git format-patch stdout: \n{stdout_output}", file=sys.stderr) - # if stderr_output != "": - # print(f"git format-patch stderr: {stderr_output}", file=sys.stderr) # Check if the exit code is not zero and bomb if proc.returncode != 0: raise Exception(f"Failed to export commit {commit} to patch: {stderr_output}") @@ -574,11 +649,17 @@ def read_file_as_utf8(file_name: str) -> tuple[str, list[str]]: content = f.read() # Read the file as bytes try: return content.decode("utf-8"), [] # no problems if this worked - except UnicodeDecodeError: + except UnicodeDecodeError as ude: + log.warning(f"File '{file_name}' is not valid utf-8, trying to fix it...: '{ude}'") # If decoding failed, try to decode as iso-8859-1 return content.decode("iso-8859-1"), ["invalid_utf8"] # utf-8 problems +def read_file_as_bytes(file_name: str) -> bytes: + with open(file_name, "rb") as f: + return f.read() # Read the file as bytes + + # Extremely Armbian-specific. def perform_git_archeology( base_armbian_src_dir: str, armbian_git_repo: git.Repo, patch: PatchInPatchFile, diff --git a/lib/tools/patching.py b/lib/tools/patching.py index cfd114a9dc..30e682aec5 100755 --- a/lib/tools/patching.py +++ b/lib/tools/patching.py @@ -16,6 +16,8 @@ log: logging.Logger = logging.getLogger("patching") # Show the environment variables we've been called with armbian_utils.show_incoming_environment() +# @TODO: test that "patch --version" is >= 2.7.6 using a subprocess and parsing the output. + # Let's start by reading environment variables. # Those are always needed, and we should bomb if they're not set. SRC = armbian_utils.get_from_env_or_bomb("SRC") @@ -32,7 +34,10 @@ apply_patches_to_git = PATCHES_TO_GIT == "yes" git_archeology = GIT_ARCHEOLOGY == "yes" fast_archeology = FAST_ARCHEOLOGY == "yes" rewrite_patches_in_place = REWRITE_PATCHES == "yes" -apply_options = {"allow_recreate_existing_files": (ALLOW_RECREATE_EXISTING_FILES == "yes")} +apply_options = { + "allow_recreate_existing_files": (ALLOW_RECREATE_EXISTING_FILES == "yes"), + "set_patch_date": True, +} # Those are optional. GIT_WORK_DIR = armbian_utils.get_from_env("GIT_WORK_DIR") @@ -42,12 +47,15 @@ USERPATCHES_PATH = armbian_utils.get_from_env("USERPATCHES_PATH") # Some path possibilities CONST_PATCH_ROOT_DIRS = [] + for patch_dir_to_apply in PATCH_DIRS_TO_APPLY: if USERPATCHES_PATH is not None: CONST_PATCH_ROOT_DIRS.append( patching_utils.PatchRootDir( f"{USERPATCHES_PATH}/{PATCH_TYPE}/{patch_dir_to_apply}", "user", PATCH_TYPE, USERPATCHES_PATH)) + + # regular patchset CONST_PATCH_ROOT_DIRS.append( patching_utils.PatchRootDir(f"{SRC}/patch/{PATCH_TYPE}/{patch_dir_to_apply}", "core", PATCH_TYPE, SRC)) @@ -65,6 +73,27 @@ for patch_root_dir in CONST_PATCH_ROOT_DIRS: for patch_sub_dir in CONST_PATCH_SUB_DIRS: ALL_DIRS.append(patching_utils.PatchDir(patch_root_dir, patch_sub_dir, SRC)) +PATCH_FILES_FIRST: list[patching_utils.PatchFileInDir] = [] +EXTRA_PATCH_FILES_FIRST: list[str] = armbian_utils.parse_env_for_tokens("EXTRA_PATCH_FILES_FIRST") +EXTRA_PATCH_HASHES_FIRST: list[str] = armbian_utils.parse_env_for_tokens("EXTRA_PATCH_HASHES_FIRST") + +for patch_file in EXTRA_PATCH_FILES_FIRST: + # if the file does not exist, bomb. + if not os.path.isfile(patch_file): + raise Exception(f"File {patch_file} does not exist.") + + # get the directory name of the file path + patch_dir = os.path.dirname(patch_file) + + # Fabricate fake dirs... + driver_root_dir = patching_utils.PatchRootDir(patch_dir, "extra-first", PATCH_TYPE, SRC) + driver_sub_dir = patching_utils.PatchSubDir("", "extra-first") + driver_dir = patching_utils.PatchDir(driver_root_dir, driver_sub_dir, SRC) + driver_dir.is_autogen_dir = True + PATCH_FILES_FIRST.append(patching_utils.PatchFileInDir(patch_file, driver_dir)) + +log.info(f"Found {len(PATCH_FILES_FIRST)} kernel driver patches") + SERIES_PATCH_FILES: list[patching_utils.PatchFileInDir] = [] # Now, loop over ALL_DIRS, and find the patch files in each directory for one_dir in ALL_DIRS: @@ -92,7 +121,8 @@ for one_patch_file in ALL_DIR_PATCH_FILES: # This reflects the order in which we want to apply the patches. # For series-based patches, we want to apply the serie'd patches first. # The other patches are separately sorted. -ALL_PATCH_FILES_SORTED = SERIES_PATCH_FILES + list(dict(sorted(ALL_DIR_PATCH_FILES_BY_NAME.items())).values()) +ALL_PATCH_FILES_SORTED = PATCH_FILES_FIRST + SERIES_PATCH_FILES + \ + list(dict(sorted(ALL_DIR_PATCH_FILES_BY_NAME.items())).values()) # Now, actually read the patch files. # Patch files might be in mailbox format, and in that case contain more than one "patch". @@ -100,6 +130,7 @@ ALL_PATCH_FILES_SORTED = SERIES_PATCH_FILES + list(dict(sorted(ALL_DIR_PATCH_FIL # We need to read the file, and see if it's a mailbox file; if so, split into multiple patches. # If not, just use the whole file as a single patch. # We'll store the patches in a list of Patch objects. +log.info("Splitting patch files into patches") VALID_PATCHES: list[patching_utils.PatchInPatchFile] = [] patch_file_in_dir: patching_utils.PatchFileInDir for patch_file_in_dir in ALL_PATCH_FILES_SORTED: @@ -112,10 +143,12 @@ for patch_file_in_dir in ALL_PATCH_FILES_SORTED: f"Can't continue; please fix the patch file {patch_file_in_dir.full_file_path()} manually. Sorry." , exc_info=True) exit(1) +log.info("Done splitting patch files into patches") # Now, some patches might not be mbox-formatted, or somehow else invalid. We can try and recover those. # That is only possible if we're applying patches to git. # Rebuilding description is only possible if we've the git repo where the patches themselves reside. +log.info("Parsing patches...") for patch in VALID_PATCHES: try: patch.parse_patch() # this handles diff-level parsing; modifies itself; throws exception if invalid @@ -165,6 +198,10 @@ if apply_patches: # Loop over the VALID_PATCHES, and apply them log.info(f"- Applying {len(VALID_PATCHES)} patches...") + # Grab the date of the root Makefile; that is the minimum date for the patched files. + root_makefile = os.path.join(GIT_WORK_DIR, "Makefile") + apply_options["root_makefile_date"] = os.path.getmtime(root_makefile) + log.info(f"- Root Makefile '{root_makefile}' date: '{os.path.getmtime(root_makefile)}'") for one_patch in VALID_PATCHES: log.info(f"Applying patch {one_patch}") one_patch.applied_ok = False