From 43c7710bfda422f720cee0422d67f235cefe3bdc Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 8 Jan 2026 12:23:40 +0100 Subject: [PATCH] tools/repository/extract-repo.sh: simplify extraction by copying directly from pool (#9174) * tools/repository/extract-repo.sh: simplify extraction by copying directly from pool Remove dependency on Packages index files. Instead of parsing package metadata to find file locations, directly scan the pool/ directory structure and copy all .deb files found in each component subdirectory. This simplifies the code and makes it more robust since it doesn't rely on index files being present or correctly formatted. Signed-off-by: Igor Pecovnik * Add helper script: recursively clean Armbian Debian package artifacts * Update tools/repository/cleanup-debs.sh Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --------- Signed-off-by: Igor Pecovnik Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- tools/repository/cleanup-debs.sh | 94 ++++++++++++++++++++++++++++++++ tools/repository/extract-repo.sh | 76 +++++++++++++------------- 2 files changed, 132 insertions(+), 38 deletions(-) create mode 100644 tools/repository/cleanup-debs.sh diff --git a/tools/repository/cleanup-debs.sh b/tools/repository/cleanup-debs.sh new file mode 100644 index 0000000000..8ad93f23ae --- /dev/null +++ b/tools/repository/cleanup-debs.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# cleanup-debs.sh +# +# Recursively clean Armbian Debian package artifacts. +# +# This script scans the given directory (and all subdirectories) for +# Debian packages whose filenames start with `armbian-` and end with `.deb`. +# +# For each logical package base (the part of the filename before the first +# underscore), it keeps only the newest version as determined by +# `dpkg --compare-versions` and removes all older versions. +# +# Key properties: +# - Operates recursively on all subfolders +# - Affects ONLY files matching `armbian-*.deb` +# - Keeps the most recent version per package base +# - Uses proper Debian version comparison (not lexical sorting) +# - Safe by default: dry-run mode enabled unless DRYRUN=0 is set +# +# Usage: +# ./armbian-deb-cleanup.sh /path/to/repository +# +# To actually delete files: +# DRYRUN=0 ./armbian-deb-cleanup.sh /path/to/repository +# +# Notes: +# - If the same package exists in multiple subdirectories, only the newest +# version is kept globally (not per directory). +# - Files not matching `armbian-*.deb` are ignored. +# +set -euo pipefail + +ROOT="${1:-.}" +DRYRUN="${DRYRUN:-1}" # DRYRUN=0 to actually delete + +shopt -s nullglob + +declare -A best_ver best_file + +# Extract base + version from armbian-*.deb +extract_base_ver() { + local f="$1" bn base ver + bn="$(basename -- "$f")" + + [[ "$bn" == armbian-*.deb ]] || return 1 + + base="${bn%%_*}" # before first underscore + ver="${bn#*_}"; ver="${ver%%_*}" # between first and second underscore + + [[ -n "$base" && -n "$ver" ]] || return 1 + printf '%s\t%s\n' "$base" "$ver" +} + +# First pass: find newest version per base (across ALL subfolders) +while IFS= read -r -d '' f; do + read -r base ver < <(extract_base_ver "$f") || continue + + if [[ -z "${best_ver[$base]:-}" ]]; then + best_ver["$base"]="$ver" + best_file["$base"]="$f" + else + if dpkg --compare-versions "$ver" gt "${best_ver[$base]}"; then + best_ver["$base"]="$ver" + best_file["$base"]="$f" + fi + fi +done < <(find "$ROOT" -type f -name 'armbian-*.deb' -print0) + +echo "Keeping newest armbian-* package per base (recursive):" +for base in "${!best_file[@]}"; do + echo " $base -> ${best_ver[$base]} ($(basename -- "${best_file[$base]}"))" +done +echo + +# Second pass: remove older versions +echo "Removing older armbian-* packages:" +while IFS= read -r -d '' f; do + read -r base ver < <(extract_base_ver "$f") || continue + if [[ "${best_file[$base]}" != "$f" ]]; then + if [[ "$DRYRUN" == "1" ]]; then + echo " DRYRUN rm -f -- $f" + else + rm -f -- "$f" + echo " rm -f -- $f" + fi + fi +done < <(find "$ROOT" -type f -name 'armbian-*.deb' -print0) + +if [[ "$DRYRUN" == "1" ]]; then + echo + echo "Dry-run mode. To actually delete:" + echo " DRYRUN=0 $0 \"$ROOT\"" +fi diff --git a/tools/repository/extract-repo.sh b/tools/repository/extract-repo.sh index c5f0782d58..122b5ab564 100755 --- a/tools/repository/extract-repo.sh +++ b/tools/repository/extract-repo.sh @@ -162,26 +162,19 @@ detect_releases() { fi } -# Get package list from Packages file -get_packages_from_component() { - local repo_base="$1" - local release="$2" - local component="$3" +# Copy all .deb files from a directory recursively +copy_debs_from_dir() { + local source_dir="$1" + local target_dir="$2" - # Find Packages file - try different architectures - local component_dir="$repo_base/dists/$release/$component" - - if [[ ! -d "$component_dir" ]]; then + if [[ ! -d "$source_dir" ]]; then return fi - # Find any Packages file in the component directory - local packages_file=$(find "$component_dir" -type f -name "Packages" | head -1) - - if [[ -n "$packages_file" && -f "$packages_file" ]]; then - # Extract package filenames from Packages file - grep -E '^Filename:' "$packages_file" | sed 's/Filename: //' || true - fi + # Find all .deb files recursively in the source directory + while IFS= read -r -d '' deb_file; do + echo "$deb_file" + done < <(find "$source_dir" -type f -name "*.deb" -print0 2>/dev/null) } # Extract packages from repository @@ -233,36 +226,43 @@ extract_packages() { for component in "${components[@]}"; do log_verbose "Processing component: $release/$component" - # Get package list - mapfile -t packages < <(get_packages_from_component "$repo_base" "$release" "$component") + # Determine source and target directories + local source_dir="" + local target_dir="" - if [[ ${#packages[@]} -eq 0 ]]; then - log_verbose "No packages found for $release/$component" + if [[ "$component" == "main" ]]; then + # Main component packages go to root + source_dir="$repo_base/pool/main" + target_dir="$output_base" + else + # Release-specific components go to extra/ + source_dir="$repo_base/pool/$component" + target_dir="$output_base/extra/$component" + if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$target_dir" + fi + fi + + if [[ ! -d "$source_dir" ]]; then + log_verbose "Source directory not found: $source_dir" continue fi - log "Found ${#packages[@]} packages in $release/$component" + # Get list of all .deb files recursively + mapfile -t packages < <(copy_debs_from_dir "$source_dir" "$target_dir") + + if [[ ${#packages[@]} -eq 0 ]]; then + log_verbose "No packages found in $source_dir" + continue + fi + + log "Found ${#packages[@]} packages in $source_dir" # Process each package - for package_path in "${packages[@]}"; do + for source_path in "${packages[@]}"; do ((total_packages++)) || true - local package_name=$(basename "$package_path") - local source_path="$repo_base/$package_path" - - # Determine target directory based on component - local target_dir="" - if [[ "$component" == "main" ]]; then - # Main component packages go to root - target_dir="$output_base" - else - # Release-specific components go to extra/ - target_dir="$output_base/extra/$component" - if [[ "$DRY_RUN" == false ]]; then - mkdir -p "$target_dir" - fi - fi - + local package_name=$(basename "$source_path") local target_path="$target_dir/$package_name" # Copy package