Compare commits
10 Commits
main
...
singlemode
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
95baded3b2 | ||
|
|
997a4e9ea5 | ||
|
|
d10295ad8c | ||
|
|
492e431131 | ||
|
|
a6c9b99509 | ||
|
|
e1f55e5d7a | ||
|
|
92052384aa | ||
|
|
80ebbe24b6 | ||
|
|
63fe441107 | ||
|
|
72ec2b171b |
@ -2,22 +2,17 @@
|
|||||||
|
|
||||||
# Global variables
|
# Global variables
|
||||||
DRY_RUN=false # Full dry-run: don't make any repository changes
|
DRY_RUN=false # Full dry-run: don't make any repository changes
|
||||||
KEEP_SOURCES=false # Keep source packages when adding to repo (don't delete)
|
KEEP_SOURCES=true # Keep source packages when adding to repo (don't delete)
|
||||||
SINGLE_RELEASE="" # Process only a single release (for GitHub Actions parallel workflow)
|
|
||||||
FORCE_ADD=false # Force re-adding packages even if they already exist in repo
|
FORCE_ADD=false # Force re-adding packages even if they already exist in repo
|
||||||
FORCE_PUBLISH=true # Force publishing even when no packages to add
|
FORCE_PUBLISH=true # Force publishing even when no packages to add
|
||||||
|
GPG_PARAMS=() # Global GPG parameters array (set by get_gpg_signing_params)
|
||||||
|
|
||||||
# Logging function - uses syslog, view logs with: journalctl -t repo-management -f
|
# Log message to syslog (view with: journalctl -t repo-management -f)
|
||||||
# Arguments:
|
|
||||||
# $* - Message to log
|
|
||||||
log() {
|
log() {
|
||||||
logger -t repo-management "$*"
|
logger -t repo-management "$*"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Execute aptly command and check for errors
|
# Execute aptly command, exit on failure (unless dry-run)
|
||||||
# Exits with status 1 if the command fails (unless in dry-run mode)
|
|
||||||
# Arguments:
|
|
||||||
# $* - Aptly command to execute (without 'aptly' prefix)
|
|
||||||
run_aptly() {
|
run_aptly() {
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
log "[DRY-RUN] Would execute: aptly $*"
|
log "[DRY-RUN] Would execute: aptly $*"
|
||||||
@ -31,11 +26,9 @@ run_aptly() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Drop published repositories that are no longer supported
|
# Drop published repositories for unsupported releases
|
||||||
# Identifies and removes published repositories for releases that are no longer
|
|
||||||
# in config/distributions/*/support (excluding 'eos')
|
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - "all" to drop all published repositories, otherwise drops only unsupported ones
|
# $1 - "all" to drop all, otherwise only drops unsupported ones
|
||||||
drop_unsupported_releases() {
|
drop_unsupported_releases() {
|
||||||
local supported_releases=()
|
local supported_releases=()
|
||||||
local published_repos=()
|
local published_repos=()
|
||||||
@ -68,82 +61,34 @@ drop_unsupported_releases() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
# Display contents of all repositories
|
# Display contents of all repositories
|
||||||
# Shows packages in the common repository and release-specific repositories (utils, desktop)
|
|
||||||
# In single-release mode, shows content from isolated database
|
|
||||||
# Otherwise, shows content from main database and any existing isolated databases
|
|
||||||
# Uses global DISTROS array for iteration, or discovers repos automatically if DISTROS is empty
|
|
||||||
showall() {
|
showall() {
|
||||||
echo "Displaying common repository contents"
|
echo "Displaying common repository contents"
|
||||||
aptly repo show -with-packages -config="${CONFIG}" common 2>/dev/null | tail -n +7
|
aptly repo show -with-packages -config="${CONFIG}" common 2>/dev/null | tail -n +7
|
||||||
|
|
||||||
# If DISTROS array is empty, discover repos from the database
|
|
||||||
local releases_to_show=("${DISTROS[@]}")
|
local releases_to_show=("${DISTROS[@]}")
|
||||||
if [[ ${#DISTROS[@]} -eq 0 ]]; then
|
if [[ ${#DISTROS[@]} -eq 0 ]]; then
|
||||||
# First, discover releases from isolated databases
|
releases_to_show=($(aptly repo list -config="${CONFIG}" -raw 2>/dev/null | awk '{print $NF}' | grep -E '^.+-(utils|desktop)$' | sed 's/-(utils|desktop)$//' | sort -u))
|
||||||
local all_repos=()
|
|
||||||
if [[ -d "$output" ]]; then
|
|
||||||
for isolated_dir in "$output"/aptly-isolated-*; do
|
|
||||||
if [[ -d "$isolated_dir" ]]; then
|
|
||||||
local release_name=$(basename "$isolated_dir" | sed 's/aptly-isolated-//')
|
|
||||||
all_repos+=("$release_name")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
# Also get repos from main database (for non-isolated repos)
|
|
||||||
local main_repos
|
|
||||||
main_repos=($(aptly repo list -config="${CONFIG}" -raw 2>/dev/null | awk '{print $NF}' | grep -E '^.+-(utils|desktop)$' | sed 's/-(utils|desktop)$//' | sort -u))
|
|
||||||
# Merge and deduplicate
|
|
||||||
all_repos+=("${main_repos[@]}")
|
|
||||||
releases_to_show=($(echo "${all_repos[@]}" | tr ' ' '\n' | sort -u))
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
for release in "${releases_to_show[@]}"; do
|
for release in "${releases_to_show[@]}"; do
|
||||||
# In single-release mode, only show that specific release from the isolated database
|
if aptly repo show -config="${CONFIG}" "${release}-utils" &>/dev/null; then
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
|
||||||
if [[ "$release" != "$SINGLE_RELEASE" ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if there's an isolated database for this release
|
|
||||||
local isolated_db="${output}/aptly-isolated-${release}"
|
|
||||||
local show_config="$CONFIG"
|
|
||||||
|
|
||||||
if [[ -d "$isolated_db" ]]; then
|
|
||||||
# Create temporary config for the isolated database
|
|
||||||
local temp_config
|
|
||||||
temp_config="$(mktemp)"
|
|
||||||
sed 's|"rootDir": ".*"|"rootDir": "'$isolated_db'"|g' tools/repository/aptly.conf > "$temp_config"
|
|
||||||
show_config="$temp_config"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Show utils repo if it exists
|
|
||||||
if aptly repo show -config="${show_config}" "${release}-utils" &>/dev/null; then
|
|
||||||
echo "Displaying repository contents for $release-utils"
|
echo "Displaying repository contents for $release-utils"
|
||||||
aptly repo show -with-packages -config="${show_config}" "${release}-utils" | tail -n +7
|
aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Show desktop repo if it exists
|
if aptly repo show -config="${CONFIG}" "${release}-desktop" &>/dev/null; then
|
||||||
if aptly repo show -config="${show_config}" "${release}-desktop" &>/dev/null; then
|
|
||||||
echo "Displaying repository contents for $release-desktop"
|
echo "Displaying repository contents for $release-desktop"
|
||||||
aptly repo show -with-packages -config="${show_config}" "${release}-desktop" | tail -n +7
|
aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
|
||||||
fi
|
|
||||||
|
|
||||||
# Clean up temp config if we created one
|
|
||||||
if [[ -n "$temp_config" && -f "$temp_config" ]]; then
|
|
||||||
rm -f "$temp_config"
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Add packages to an aptly repository component
|
# Add .deb packages to repository component
|
||||||
# Processes .deb files from a source directory, optionally repacking BSP packages
|
|
||||||
# to pin kernel versions, then adds them to the specified repository
|
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - Repository component name (e.g., "common", "jammy-utils")
|
# $1 - Component name (e.g., "common", "jammy-utils")
|
||||||
# $2 - Subdirectory path relative to input folder (e.g., "", "/extra/jammy-utils")
|
# $2 - Subdirectory path (e.g., "", "/extra/jammy-utils")
|
||||||
# $3 - Description (unused, for documentation only)
|
# $3 - Description (unused)
|
||||||
# $4 - Base input folder containing packages
|
# $4 - Base input folder containing packages
|
||||||
adding_packages() {
|
adding_packages() {
|
||||||
local component="$1"
|
local component="$1"
|
||||||
@ -204,6 +149,23 @@ adding_packages() {
|
|||||||
|
|
||||||
log "Checking package: $deb_display"
|
log "Checking package: $deb_display"
|
||||||
|
|
||||||
|
# If package with same name+arch but different version exists in repo, remove it first
|
||||||
|
# This prevents "file already exists and is different" errors during publish
|
||||||
|
if [[ "$FORCE_ADD" != true ]]; then
|
||||||
|
for existing_key in "${!repo_packages_map[@]}"; do
|
||||||
|
# existing_key format: name|version|arch
|
||||||
|
local existing_name existing_version existing_arch
|
||||||
|
IFS='|' read -r existing_name existing_version existing_arch <<< "$existing_key"
|
||||||
|
# Check if same name and arch but different version
|
||||||
|
if [[ "$existing_name" == "$deb_name" && "$existing_arch" == "$deb_arch" && "$existing_version" != "$deb_version" ]]; then
|
||||||
|
log "Removing old version ${existing_name}_${existing_version}_${existing_arch} before adding new version"
|
||||||
|
run_aptly repo remove -config="${CONFIG}" "${component}" "${existing_name}_${existing_version}_${existing_arch}"
|
||||||
|
# Remove from map so we don't try to remove it again
|
||||||
|
unset "repo_packages_map[$existing_key]"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
# Skip if exact package (name+version+arch) already exists in repo (unless FORCE_ADD is true)
|
# Skip if exact package (name+version+arch) already exists in repo (unless FORCE_ADD is true)
|
||||||
if [[ "$FORCE_ADD" != true && -n "${repo_packages_map[$deb_key]}" ]]; then
|
if [[ "$FORCE_ADD" != true && -n "${repo_packages_map[$deb_key]}" ]]; then
|
||||||
echo "[-] SKIP: $deb_display"
|
echo "[-] SKIP: $deb_display"
|
||||||
@ -213,13 +175,10 @@ adding_packages() {
|
|||||||
|
|
||||||
# Repack BSP packages if last-known-good kernel map exists
|
# Repack BSP packages if last-known-good kernel map exists
|
||||||
# This prevents upgrading to kernels that may break the board
|
# This prevents upgrading to kernels that may break the board
|
||||||
if [[ -f userpatches/last-known-good.map ]]; then
|
if [[ -f userpatches/last-known-good-kernel-pkg.map ]]; then
|
||||||
local package_name
|
|
||||||
package_name=$(dpkg-deb -W "$deb_file" | awk '{ print $1 }')
|
|
||||||
|
|
||||||
# Read kernel pinning mappings from file
|
# Read kernel pinning mappings from file
|
||||||
while IFS='|' read -r board branch linux_family last_kernel; do
|
while IFS='|' read -r board branch linux_family last_kernel; do
|
||||||
if [[ "${package_name}" == "armbian-bsp-cli-${board}-${branch}" ]]; then
|
if [[ "${deb_name}" == "armbian-bsp-cli-${board}-${branch}" ]]; then
|
||||||
echo "Setting last kernel upgrade for $board to linux-image-$branch-$board=${last_kernel}"
|
echo "Setting last kernel upgrade for $board to linux-image-$branch-$board=${last_kernel}"
|
||||||
|
|
||||||
# Extract, modify control file, and repackage
|
# Extract, modify control file, and repackage
|
||||||
@ -236,9 +195,8 @@ adding_packages() {
|
|||||||
# Determine whether to remove source files after adding to repo
|
# Determine whether to remove source files after adding to repo
|
||||||
# KEEP_SOURCES mode preserves source packages
|
# KEEP_SOURCES mode preserves source packages
|
||||||
# DRY_RUN mode also preserves sources (and skips all repo modifications)
|
# DRY_RUN mode also preserves sources (and skips all repo modifications)
|
||||||
# SINGLE_RELEASE mode preserves sources so parallel workers don't delete files needed by other workers
|
|
||||||
local remove_flag="-remove-files"
|
local remove_flag="-remove-files"
|
||||||
if [[ "$KEEP_SOURCES" == true ]] || [[ "$DRY_RUN" == true ]] || [[ -n "$SINGLE_RELEASE" ]]; then
|
if [[ "$KEEP_SOURCES" == true ]] || [[ "$DRY_RUN" == true ]]; then
|
||||||
remove_flag=""
|
remove_flag=""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@ -249,53 +207,7 @@ adding_packages() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Build the common (main) repository component
|
# Process a single release: create repos, publish, and sign
|
||||||
# Creates/updates the common repository that contains packages shared across all releases
|
|
||||||
# Should be run once before processing individual releases in parallel
|
|
||||||
# Arguments:
|
|
||||||
# $1 - Input folder containing packages
|
|
||||||
# $2 - Output folder for published repository
|
|
||||||
# $3 - GPG password for signing (currently unused, signing is done separately)
|
|
||||||
update_main() {
|
|
||||||
local input_folder="$1"
|
|
||||||
local output_folder="$2"
|
|
||||||
local gpg_password="$3"
|
|
||||||
|
|
||||||
log "Building common (main) component"
|
|
||||||
|
|
||||||
# Create common repo if it doesn't exist
|
|
||||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
|
|
||||||
run_aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | logger -t repo-management >/dev/null
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add packages from main folder
|
|
||||||
adding_packages "common" "" "main" "$input_folder"
|
|
||||||
|
|
||||||
# Drop old snapshot if it exists and is not published
|
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
|
|
||||||
# Check if snapshot is published
|
|
||||||
if ! aptly publish list -config="${CONFIG}" 2>/dev/null | grep -q "common"; then
|
|
||||||
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management >/dev/null
|
|
||||||
else
|
|
||||||
log "WARNING: common snapshot is published, cannot drop. Packages added to repo but snapshot not updated."
|
|
||||||
log "Run 'update' command to update all releases with new packages."
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Create new snapshot if it doesn't exist or was dropped
|
|
||||||
if [[ -z $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
|
|
||||||
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
|
|
||||||
else
|
|
||||||
log "common snapshot already exists, skipping creation"
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Common component built successfully"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Process a single release distribution
|
|
||||||
# Creates/updates release-specific repositories (utils, desktop), publishes them,
|
|
||||||
# and signs the Release files. Can be run in parallel for different releases.
|
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - Release name (e.g., "jammy", "noble")
|
# $1 - Release name (e.g., "jammy", "noble")
|
||||||
# $2 - Input folder containing packages
|
# $2 - Input folder containing packages
|
||||||
@ -309,29 +221,6 @@ process_release() {
|
|||||||
|
|
||||||
log "Processing release: $release"
|
log "Processing release: $release"
|
||||||
|
|
||||||
# In isolated mode (SINGLE_RELEASE), ensure common snapshot exists
|
|
||||||
# It should have been created by 'update-main' command, but if not, create it from input packages
|
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
|
||||||
# Create common repo if it doesn't exist
|
|
||||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
|
|
||||||
run_aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | logger -t repo-management >/dev/null
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add packages from main input folder to common repo
|
|
||||||
# This ensures each isolated worker has the common packages
|
|
||||||
log "Populating common repo from input folder: $input_folder"
|
|
||||||
adding_packages "common" "" "main" "$input_folder"
|
|
||||||
|
|
||||||
# Drop old common snapshot if it exists (in isolated DB, snapshots aren't published yet)
|
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
|
|
||||||
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management >/dev/null
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Create snapshot with packages
|
|
||||||
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
|
|
||||||
log "Created common snapshot with packages for isolated mode"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Create release-specific repositories if they don't exist
|
# Create release-specific repositories if they don't exist
|
||||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
||||||
run_aptly repo create -config="${CONFIG}" -component="${release}-utils" -distribution="${release}" -comment="Armbian ${release}-utils repository" "${release}-utils" | logger -t repo-management >/dev/null
|
run_aptly repo create -config="${CONFIG}" -component="${release}-utils" -distribution="${release}" -comment="Armbian ${release}-utils repository" "${release}-utils" | logger -t repo-management >/dev/null
|
||||||
@ -340,19 +229,26 @@ process_release() {
|
|||||||
run_aptly repo create -config="${CONFIG}" -component="${release}-desktop" -distribution="${release}" -comment="Armbian ${release}-desktop repository" "${release}-desktop" | logger -t repo-management >/dev/null
|
run_aptly repo create -config="${CONFIG}" -component="${release}-desktop" -distribution="${release}" -comment="Armbian ${release}-desktop repository" "${release}-desktop" | logger -t repo-management >/dev/null
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Run db cleanup before adding packages to avoid "file already exists and is different" errors
|
||||||
|
# This removes unreferenced packages from previous runs that may have the same filename
|
||||||
|
log "Running database cleanup before adding release packages"
|
||||||
|
run_aptly db cleanup -config="${CONFIG}"
|
||||||
|
|
||||||
# Add packages ONLY from release-specific extra folders
|
# Add packages ONLY from release-specific extra folders
|
||||||
adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$input_folder"
|
adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$input_folder"
|
||||||
adding_packages "${release}-desktop" "/extra/${release}-desktop" "release desktop" "$input_folder"
|
adding_packages "${release}-desktop" "/extra/${release}-desktop" "release desktop" "$input_folder"
|
||||||
|
|
||||||
# Run db cleanup before publishing to remove unreferenced packages
|
# Run db cleanup again after adding packages to remove any old package files
|
||||||
# This helps avoid "file already exists and is different" errors
|
# This is critical after removing old versions of packages to prevent
|
||||||
log "Running database cleanup before publishing"
|
# "file already exists and is different" errors during publish
|
||||||
|
log "Running database cleanup after adding packages"
|
||||||
run_aptly db cleanup -config="${CONFIG}"
|
run_aptly db cleanup -config="${CONFIG}"
|
||||||
|
|
||||||
# Check if we have any packages to publish
|
# Check if we have any packages to publish
|
||||||
# Get package counts in each repo
|
# Get package counts in each repo
|
||||||
local utils_count=$(aptly repo show -config="${CONFIG}" "${release}-utils" 2>/dev/null | grep "Number of packages" | awk '{print $4}' || echo "0")
|
local utils_count desktop_count
|
||||||
local desktop_count=$(aptly repo show -config="${CONFIG}" "${release}-desktop" 2>/dev/null | grep "Number of packages" | awk '{print $4}' || echo "0")
|
utils_count=$(aptly repo show -config="${CONFIG}" "${release}-utils" 2>/dev/null | grep "Number of packages" | awk '{print $4}') || utils_count="0"
|
||||||
|
desktop_count=$(aptly repo show -config="${CONFIG}" "${release}-desktop" 2>/dev/null | grep "Number of packages" | awk '{print $4}') || desktop_count="0"
|
||||||
|
|
||||||
log "Package counts for $release: utils=$utils_count, desktop=$desktop_count"
|
log "Package counts for $release: utils=$utils_count, desktop=$desktop_count"
|
||||||
|
|
||||||
@ -370,102 +266,48 @@ process_release() {
|
|||||||
log "Force publish enabled: will publish even with no packages"
|
log "Force publish enabled: will publish even with no packages"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Drop old snapshots if we have new packages to add OR if FORCE_PUBLISH is enabled
|
# Always drop and recreate snapshots for fresh publish
|
||||||
# This ensures fresh snapshots are created for force-publish scenarios
|
# This ensures that even empty repos are properly published
|
||||||
if [[ "$utils_count" -gt 0 || "$FORCE_PUBLISH" == true ]]; then
|
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
log "Dropping existing ${release}-utils snapshot"
|
||||||
log "Dropping existing ${release}-utils snapshot"
|
run_aptly -config="${CONFIG}" snapshot drop ${release}-utils | logger -t repo-management 2>/dev/null
|
||||||
run_aptly -config="${CONFIG}" snapshot drop ${release}-utils | logger -t repo-management 2>/dev/null
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
if [[ "$desktop_count" -gt 0 || "$FORCE_PUBLISH" == true ]]; then
|
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
|
log "Dropping existing ${release}-desktop snapshot"
|
||||||
log "Dropping existing ${release}-desktop snapshot"
|
run_aptly -config="${CONFIG}" snapshot drop ${release}-desktop | logger -t repo-management 2>/dev/null
|
||||||
run_aptly -config="${CONFIG}" snapshot drop ${release}-desktop | logger -t repo-management 2>/dev/null
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Create snapshots only for repos that have packages
|
# Create snapshots for all repos (even empty ones) to ensure they're included in publish
|
||||||
# OR when FORCE_PUBLISH is enabled (then we publish whatever exists in the DB)
|
local components_to_publish=()
|
||||||
local components_to_publish=("main")
|
local snapshots_to_publish=()
|
||||||
local snapshots_to_publish=("common")
|
|
||||||
|
|
||||||
if [[ "$utils_count" -gt 0 || "$FORCE_PUBLISH" == true ]]; then
|
# Add common/main component
|
||||||
# Only create snapshot if repo has packages, or if force-publishing
|
components_to_publish=("main")
|
||||||
if [[ "$utils_count" -gt 0 ]]; then
|
snapshots_to_publish=("common")
|
||||||
run_aptly -config="${CONFIG}" snapshot create ${release}-utils from repo ${release}-utils | logger -t repo-management >/dev/null
|
|
||||||
components_to_publish+=("${release}-utils")
|
|
||||||
snapshots_to_publish+=("${release}-utils")
|
|
||||||
elif [[ "$FORCE_PUBLISH" == true ]]; then
|
|
||||||
log "Force publish: checking for existing ${release}-utils snapshot in DB"
|
|
||||||
# Try to use existing snapshot if it exists
|
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
|
|
||||||
components_to_publish+=("${release}-utils")
|
|
||||||
snapshots_to_publish+=("${release}-utils")
|
|
||||||
log "Using existing ${release}-utils snapshot"
|
|
||||||
else
|
|
||||||
# Create empty snapshot from empty repo
|
|
||||||
run_aptly -config="${CONFIG}" snapshot create ${release}-utils from repo ${release}-utils | logger -t repo-management >/dev/null
|
|
||||||
components_to_publish+=("${release}-utils")
|
|
||||||
snapshots_to_publish+=("${release}-utils")
|
|
||||||
log "Created empty ${release}-utils snapshot for force publish"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$desktop_count" -gt 0 || "$FORCE_PUBLISH" == true ]]; then
|
# Always create utils snapshot and include in publish (even if empty)
|
||||||
# Only create snapshot if repo has packages, or if force-publishing
|
log "Creating ${release}-utils snapshot (packages: $utils_count)"
|
||||||
if [[ "$desktop_count" -gt 0 ]]; then
|
run_aptly -config="${CONFIG}" snapshot create ${release}-utils from repo ${release}-utils | logger -t repo-management >/dev/null
|
||||||
run_aptly -config="${CONFIG}" snapshot create ${release}-desktop from repo ${release}-desktop | logger -t repo-management >/dev/null
|
components_to_publish+=("${release}-utils")
|
||||||
components_to_publish+=("${release}-desktop")
|
snapshots_to_publish+=("${release}-utils")
|
||||||
snapshots_to_publish+=("${release}-desktop")
|
|
||||||
elif [[ "$FORCE_PUBLISH" == true ]]; then
|
# Always create desktop snapshot and include in publish (even if empty)
|
||||||
log "Force publish: checking for existing ${release}-desktop snapshot in DB"
|
log "Creating ${release}-desktop snapshot (packages: $desktop_count)"
|
||||||
# Try to use existing snapshot if it exists
|
run_aptly -config="${CONFIG}" snapshot create ${release}-desktop from repo ${release}-desktop | logger -t repo-management >/dev/null
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
|
components_to_publish+=("${release}-desktop")
|
||||||
components_to_publish+=("${release}-desktop")
|
snapshots_to_publish+=("${release}-desktop")
|
||||||
snapshots_to_publish+=("${release}-desktop")
|
|
||||||
log "Using existing ${release}-desktop snapshot"
|
|
||||||
else
|
|
||||||
# Create empty snapshot from empty repo
|
|
||||||
run_aptly -config="${CONFIG}" snapshot create ${release}-desktop from repo ${release}-desktop | logger -t repo-management >/dev/null
|
|
||||||
components_to_publish+=("${release}-desktop")
|
|
||||||
snapshots_to_publish+=("${release}-desktop")
|
|
||||||
log "Created empty ${release}-desktop snapshot for force publish"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Publishing $release with components: ${components_to_publish[*]}"
|
log "Publishing $release with components: ${components_to_publish[*]}"
|
||||||
|
|
||||||
# Determine publish directory based on mode
|
|
||||||
local publish_dir="$output_folder"
|
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
|
||||||
publish_dir="$IsolatedRootDir"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Publish - include common snapshot for main component
|
# Publish - include common snapshot for main component
|
||||||
log "Publishing $release"
|
log "Publishing $release"
|
||||||
|
|
||||||
# Drop existing publish for this release if it exists to avoid "file already exists" errors
|
# Drop existing publish for this release if it exists to avoid "file already exists" errors
|
||||||
if aptly publish list -config="${CONFIG}" 2>/dev/null | grep -q "^\[${release}\]"; then
|
if aptly publish list -config="${CONFIG}" 2>/dev/null | grep -q "^\[${release}\]"; then
|
||||||
log "Dropping existing publish for $release from isolated DB"
|
log "Dropping existing publish for $release"
|
||||||
run_aptly publish drop -config="${CONFIG}" "${release}"
|
run_aptly publish drop -config="${CONFIG}" "${release}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# When using isolated DB, only clean up the isolated DB's published files
|
|
||||||
# DO NOT clean up shared output - other parallel workers might be using it
|
|
||||||
# The rsync copy will overwrite as needed, preserving other releases' files
|
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
|
||||||
# Clean up isolated DB's published files only
|
|
||||||
if [[ -d "${IsolatedRootDir}/public/dists/${release}" ]]; then
|
|
||||||
log "Cleaning up existing published files for $release in isolated DB"
|
|
||||||
rm -rf "${IsolatedRootDir}/public/dists/${release}"
|
|
||||||
# Clean up pool entries for this release in isolated DB
|
|
||||||
find "${IsolatedRootDir}/public/pool" -type d -name "${release}-*" 2>/dev/null | xargs -r rm -rf
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Build publish command with only components that have packages
|
# Build publish command with only components that have packages
|
||||||
local component_list=$(IFS=,; echo "${components_to_publish[*]}")
|
local component_list=$(IFS=,; echo "${components_to_publish[*]}")
|
||||||
local snapshot_list="${snapshots_to_publish[*]}"
|
local snapshot_list="${snapshots_to_publish[*]}"
|
||||||
@ -473,6 +315,12 @@ process_release() {
|
|||||||
log "Publishing with components: $component_list"
|
log "Publishing with components: $component_list"
|
||||||
log "Publishing with snapshots: $snapshot_list"
|
log "Publishing with snapshots: $snapshot_list"
|
||||||
|
|
||||||
|
# Skip publishing if no components to publish (shouldn't happen, but safety check)
|
||||||
|
if [[ ${#components_to_publish[@]} -eq 0 ]]; then
|
||||||
|
log "WARNING: No components to publish for $release"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
run_aptly publish \
|
run_aptly publish \
|
||||||
-skip-signing \
|
-skip-signing \
|
||||||
-skip-contents \
|
-skip-contents \
|
||||||
@ -484,94 +332,35 @@ process_release() {
|
|||||||
-component="$component_list" \
|
-component="$component_list" \
|
||||||
-distribution="${release}" snapshot $snapshot_list
|
-distribution="${release}" snapshot $snapshot_list
|
||||||
|
|
||||||
# If using isolated DB, copy published files to shared output location FIRST
|
|
||||||
log "Isolated mode check: SINGLE_RELEASE='$SINGLE_RELEASE' publish_dir='$publish_dir' output_folder='$output_folder'"
|
|
||||||
if [[ -n "$SINGLE_RELEASE" && "$publish_dir" != "$output_folder" ]]; then
|
|
||||||
log "Copying published files from isolated DB to shared output"
|
|
||||||
log "Source: ${publish_dir}/public"
|
|
||||||
log "Destination: ${output_folder}/public"
|
|
||||||
if [[ -d "${publish_dir}/public" ]]; then
|
|
||||||
mkdir -p "${output_folder}/public"
|
|
||||||
# Use rsync to copy published repo files to shared location
|
|
||||||
# NO --delete flag - we want to preserve other releases' files
|
|
||||||
if ! rsync -a "${publish_dir}/public/" "${output_folder}/public/" 2>&1 | logger -t repo-management; then
|
|
||||||
log "ERROR: Failed to copy published files for $release"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
log "Copied files for $release to ${output_folder}/public/"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Sign Release files for this release
|
# Sign Release files for this release
|
||||||
# This includes:
|
# This includes:
|
||||||
# 1. Top-level Release file (dists/{release}/Release)
|
# 1. Top-level Release file (dists/{release}/Release)
|
||||||
# 2. Component-level Release files (dists/{release}/{component}/Release)
|
# 2. Component-level Release files (dists/{release}/{component}/Release)
|
||||||
# Sign AFTER copying so signed files end up in the shared output location
|
|
||||||
log "Starting signing process for $release"
|
log "Starting signing process for $release"
|
||||||
# Use shared output location for signing, not isolated directory
|
|
||||||
local release_pub_dir="${output_folder}/public/dists/${release}"
|
local release_pub_dir="${output_folder}/public/dists/${release}"
|
||||||
|
|
||||||
# Get GPG keys from environment or use defaults
|
if ! get_gpg_signing_params "$gpg_password"; then
|
||||||
# Use BOTH keys for signing, just like the signing() function does
|
|
||||||
local gpg_keys=()
|
|
||||||
if [[ -n "$GPG_KEY" ]]; then
|
|
||||||
gpg_keys=("$GPG_KEY")
|
|
||||||
else
|
|
||||||
gpg_keys=("DF00FAF1C577104B50BF1D0093D6889F9F0E78D5" "8CFA83D13EB2181EEF5843E41EB30FAF236099FE")
|
|
||||||
fi
|
|
||||||
|
|
||||||
local gpg_params=("--yes" "--armor")
|
|
||||||
local keys_found=0
|
|
||||||
|
|
||||||
# Add all available keys to GPG parameters
|
|
||||||
for gpg_key in "${gpg_keys[@]}"; do
|
|
||||||
# Try to find the actual key in the keyring
|
|
||||||
local actual_key=""
|
|
||||||
if gpg --list-secret-keys "$gpg_key" >/dev/null 2>&1; then
|
|
||||||
actual_key="$gpg_key"
|
|
||||||
else
|
|
||||||
# Try to find by email or partial match
|
|
||||||
actual_key=$(gpg --list-secret-keys --keyid-format LONG 2>/dev/null | grep -B1 "$gpg_key" | grep "sec" | awk '{print $2}' | cut -d'/' -f2 || echo "")
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -n "$actual_key" ]]; then
|
|
||||||
gpg_params+=("-u" "$actual_key")
|
|
||||||
log "Adding GPG key for signing: $actual_key (requested: $gpg_key)"
|
|
||||||
((keys_found++))
|
|
||||||
else
|
|
||||||
log "WARNING: GPG key $gpg_key not found in keyring"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ $keys_found -eq 0 ]]; then
|
|
||||||
log "ERROR: No GPG keys found in keyring"
|
|
||||||
log "Available keys:"
|
|
||||||
gpg --list-secret-keys --keyid-format LONG 2>&1 | logger -t repo-management
|
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
log "Using $keys_found GPG key(s) for signing"
|
|
||||||
|
|
||||||
# First, create component-level Release files by copying from binary-amd64 Release
|
# First, create component-level Release files by copying from binary-amd64 Release
|
||||||
# This is needed because aptly only creates Release files in binary-* subdirs
|
# This is needed because aptly only creates Release files in binary-* subdirs
|
||||||
for component in main ${release}-utils ${release}-desktop; do
|
for component in main ${release}-utils ${release}-desktop; do
|
||||||
local component_dir="${release_pub_dir}/${component}"
|
local component_dir="${release_pub_dir}/${component}"
|
||||||
if [[ -d "$component_dir" ]]; then
|
if [[ -d "$component_dir" ]]; then
|
||||||
# Use the binary-amd64 Release file as the component Release file
|
|
||||||
local source_release="${component_dir}/binary-amd64/Release"
|
local source_release="${component_dir}/binary-amd64/Release"
|
||||||
local target_release="${component_dir}/Release"
|
local target_release="${component_dir}/Release"
|
||||||
|
|
||||||
if [[ -f "$source_release" && ! -f "$target_release" ]]; then
|
if [[ -f "$source_release" && ! -f "$target_release" ]]; then
|
||||||
log "Creating component Release file: ${target_release}"
|
log "Creating component Release file: ${target_release}"
|
||||||
cp "$source_release" "$target_release" 2>&1 | logger -t repo-management
|
cp "$source_release" "$target_release"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# Now sign all Release files (both top-level and component-level)
|
# Sign all Release files (both top-level and component-level)
|
||||||
# Find all Release files except those in binary-* subdirectories
|
# Skip binary-* subdirectories
|
||||||
find "${release_pub_dir}" -type f -name "Release" | while read -r release_file; do
|
find "${release_pub_dir}" -type f -name "Release" | while read -r release_file; do
|
||||||
# Skip binary-* subdirectories
|
|
||||||
if [[ "$release_file" =~ /binary-[^/]+/Release$ ]]; then
|
if [[ "$release_file" =~ /binary-[^/]+/Release$ ]]; then
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
@ -579,64 +368,65 @@ process_release() {
|
|||||||
log "Signing: ${release_file}"
|
log "Signing: ${release_file}"
|
||||||
local sign_dir="$(dirname "$release_file")"
|
local sign_dir="$(dirname "$release_file")"
|
||||||
|
|
||||||
if gpg "${gpg_params[@]}" --clear-sign -o "${sign_dir}/InRelease" "$release_file" 2>&1 | logger -t repo-management >/dev/null; then
|
# Sign with InRelease (clear-sign) - capture output for logging
|
||||||
gpg "${gpg_params[@]}" --detach-sign -o "${sign_dir}/Release.gpg" "$release_file" 2>&1 | logger -t repo-management >/dev/null
|
if gpg "${GPG_PARAMS[@]}" --clear-sign -o "${sign_dir}/InRelease" "$release_file" 2>&1; then
|
||||||
log "Successfully signed: ${release_file}"
|
log "Created InRelease for: ${release_file}"
|
||||||
|
# Sign with Release.gpg (detach-sign)
|
||||||
|
if gpg "${GPG_PARAMS[@]}" --detach-sign -o "${sign_dir}/Release.gpg" "$release_file" 2>&1; then
|
||||||
|
log "Successfully signed: ${release_file}"
|
||||||
|
else
|
||||||
|
log "ERROR: Failed to create Release.gpg for: ${release_file}"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
log "ERROR: Failed to sign: ${release_file}"
|
log "ERROR: Failed to create InRelease for: ${release_file}"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
log "Completed processing release: $release"
|
log "Completed processing release: $release"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Publish repositories for all configured releases
|
# Build common component, process all releases, and finalize repository
|
||||||
# Builds common component, processes each release, and finalizes the repository
|
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - Input folder containing packages
|
# $1 - Input folder containing packages
|
||||||
# $2 - Output folder for published repository
|
# $2 - Output folder for published repository
|
||||||
# $3 - Command name (unused, for compatibility)
|
# $3 - Command name (unused)
|
||||||
# $4 - GPG password for signing
|
# $4 - GPG password for signing
|
||||||
# $5 - Comma-separated list of releases (unused, determined from config)
|
# $5 - Comma-separated list of releases (unused, determined from config)
|
||||||
publishing() {
|
publishing() {
|
||||||
# Only build common repo if NOT in single-release mode
|
# Build common repo - this repository contains packages that are the same in all releases
|
||||||
# In single-release mode, common should be built separately with 'update-main' command
|
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
|
||||||
if [[ -z "$SINGLE_RELEASE" ]]; then
|
run_aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | logger -t repo-management >/dev/null
|
||||||
# This repository contains packages that are the same in all releases
|
|
||||||
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
|
|
||||||
run_aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | logger -t repo-management >/dev/null
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add packages from main folder
|
|
||||||
adding_packages "common" "" "main" "$1"
|
|
||||||
|
|
||||||
# Create snapshot
|
|
||||||
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
|
|
||||||
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management >/dev/null
|
|
||||||
fi
|
|
||||||
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
|
|
||||||
else
|
|
||||||
# Single-release mode: ensure common snapshot exists (should be created by update-main)
|
|
||||||
if [[ -z $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
|
|
||||||
log "WARNING: Common snapshot not found. Run 'update-main' command first!"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Get all distributions or use single release if specified
|
# Run db cleanup before adding packages to avoid "file already exists and is different" errors
|
||||||
local distributions=()
|
# This removes unreferenced packages from previous runs that may have the same filename
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
log "Running database cleanup before adding common packages"
|
||||||
distributions=("$SINGLE_RELEASE")
|
run_aptly db cleanup -config="${CONFIG}"
|
||||||
log "Single release mode: processing only $SINGLE_RELEASE"
|
|
||||||
else
|
# Add packages from main folder
|
||||||
distributions=($(grep -rw config/distributions/*/support -ve '' | cut -d"/" -f3))
|
adding_packages "common" "" "main" "$1"
|
||||||
|
|
||||||
|
# Run db cleanup after adding packages to remove any old package files
|
||||||
|
# This is critical after removing old versions of packages to prevent
|
||||||
|
# "file already exists and is different" errors during publish
|
||||||
|
log "Running database cleanup after adding common packages"
|
||||||
|
run_aptly db cleanup -config="${CONFIG}"
|
||||||
|
|
||||||
|
# Create or update the common snapshot
|
||||||
|
# Drop existing snapshot if it exists
|
||||||
|
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "^common$") ]]; then
|
||||||
|
log "Dropping existing common snapshot"
|
||||||
|
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management 2>/dev/null
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
log "Creating common snapshot"
|
||||||
|
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
|
||||||
|
|
||||||
|
# Get all distributions
|
||||||
|
local distributions=($(grep -rw config/distributions/*/support -ve '' | cut -d"/" -f3))
|
||||||
|
|
||||||
# Process releases sequentially
|
# Process releases sequentially
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
log "Processing ${#distributions[@]} releases sequentially"
|
||||||
log "Processing single release: ${distributions[0]}"
|
|
||||||
else
|
|
||||||
log "Processing ${#distributions[@]} releases sequentially"
|
|
||||||
fi
|
|
||||||
for release in "${distributions[@]}"; do
|
for release in "${distributions[@]}"; do
|
||||||
process_release "$release" "$1" "$2" "$4"
|
process_release "$release" "$1" "$2" "$4"
|
||||||
done
|
done
|
||||||
@ -658,109 +448,89 @@ publishing() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Sign repository Release files using GPG
|
# Resolve GPG keys and build signing parameters
|
||||||
# Creates InRelease and Release.gpg signature files for component-level Release files
|
# Sets global GPG_PARAMS array
|
||||||
|
# Arguments:
|
||||||
|
# $1 - GPG password (optional, currently unused)
|
||||||
|
# Returns:
|
||||||
|
# 0 on success, 1 if no keys found
|
||||||
|
get_gpg_signing_params() {
|
||||||
|
local gpg_password="${1:-}"
|
||||||
|
local gpg_keys=()
|
||||||
|
|
||||||
|
# Get GPG keys from environment or use defaults
|
||||||
|
if [[ -n "$GPG_KEY" ]]; then
|
||||||
|
gpg_keys=("$GPG_KEY")
|
||||||
|
else
|
||||||
|
gpg_keys=("DF00FAF1C577104B50BF1D0093D6889F9F0E78D5" "8CFA83D13EB2181EEF5843E41EB30FAF236099FE")
|
||||||
|
fi
|
||||||
|
|
||||||
|
GPG_PARAMS=("--yes" "--armor")
|
||||||
|
local keys_found=0
|
||||||
|
|
||||||
|
# Add all available keys to GPG parameters
|
||||||
|
for gpg_key in "${gpg_keys[@]}"; do
|
||||||
|
local actual_key=""
|
||||||
|
if gpg --list-secret-keys "$gpg_key" >/dev/null 2>&1; then
|
||||||
|
actual_key="$gpg_key"
|
||||||
|
else
|
||||||
|
# Try to find by email or partial match
|
||||||
|
actual_key=$(gpg --list-secret-keys --keyid-format LONG 2>/dev/null | grep -B1 "$gpg_key" | grep "sec" | awk '{print $2}' | cut -d'/' -f2 || echo "")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$actual_key" ]]; then
|
||||||
|
GPG_PARAMS+=("-u" "$actual_key")
|
||||||
|
log "Adding GPG key for signing: $actual_key (requested: $gpg_key)"
|
||||||
|
((keys_found++))
|
||||||
|
else
|
||||||
|
log "WARNING: GPG key $gpg_key not found in keyring"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ $keys_found -eq 0 ]]; then
|
||||||
|
log "ERROR: No GPG keys found in keyring"
|
||||||
|
log "Available keys:"
|
||||||
|
gpg --list-secret-keys --keyid-format LONG 2>&1 | logger -t repo-management
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Using $keys_found GPG key(s) for signing"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Sign Release files with GPG (creates InRelease and Release.gpg)
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - Output folder path containing published repository
|
# $1 - Output folder path containing published repository
|
||||||
# $@ - GPG key IDs to use for signing
|
|
||||||
signing() {
|
signing() {
|
||||||
local output_folder="$1"
|
local output_folder="$1"
|
||||||
shift
|
|
||||||
local gpg_keys=("$@")
|
|
||||||
|
|
||||||
if [[ ${#gpg_keys[@]} -eq 0 ]]; then
|
if ! get_gpg_signing_params; then
|
||||||
echo "No GPG keys provided for signing." >&2
|
return 1
|
||||||
return 1
|
fi
|
||||||
fi
|
|
||||||
|
|
||||||
# Build GPG parameters with available keys
|
# Sign top-level Release files for each distribution
|
||||||
local gpg_params=("--yes" "--armor")
|
find "$output_folder/public/dists" -maxdepth 2 -type f -name Release | while read -r release_file; do
|
||||||
for key in "${gpg_keys[@]}"; do
|
local rel_path="${release_file#$output_folder/public/dists/}"
|
||||||
# Try to find the actual key in the keyring
|
local slash_count=$(echo "$rel_path" | tr -cd '/' | wc -c)
|
||||||
local actual_key=""
|
|
||||||
if gpg --list-secret-keys "$key" >/dev/null 2>&1; then
|
|
||||||
actual_key="$key"
|
|
||||||
else
|
|
||||||
# Try to find by email or partial match
|
|
||||||
actual_key=$(gpg --list-secret-keys --keyid-format LONG 2>/dev/null | grep -B1 "$key" | grep "sec" | awk '{print $2}' | cut -d'/' -f2 || echo "")
|
|
||||||
if [[ -z "$actual_key" ]]; then
|
|
||||||
echo "Warning: GPG key $key not found on this system." >&2
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
gpg_params+=("-u" "$actual_key")
|
|
||||||
echo "Using GPG key: $actual_key (requested: $key)" >&2
|
|
||||||
done
|
|
||||||
|
|
||||||
# Only sign Release files at component level, NOT binary subdirs
|
if [[ $slash_count -eq 1 ]]; then
|
||||||
# Sign: dists/{release}/{component}/Release
|
local distro_path
|
||||||
# Skip: dists/{release}/Release (top-level, not needed)
|
distro_path="$(dirname "$release_file")"
|
||||||
# Skip: dists/{release}/*/binary-*/Release (subdirs, not needed)
|
log "Signing release at: $distro_path"
|
||||||
find "$output_folder/public/dists" -type f -name Release | while read -r release_file; do
|
gpg "${GPG_PARAMS[@]}" --clear-sign -o "$distro_path/InRelease" "$release_file"
|
||||||
# Skip if file is in a binary-* subdirectory
|
gpg "${GPG_PARAMS[@]}" --detach-sign -o "$distro_path/Release.gpg" "$release_file"
|
||||||
if [[ "$release_file" =~ /binary-[^/]+/Release$ ]]; then
|
fi
|
||||||
continue
|
done
|
||||||
fi
|
|
||||||
|
|
||||||
# Skip top-level Release files (dists/{release}/Release)
|
|
||||||
# Only sign component-level Release files (dists/{release}/{component}/Release)
|
|
||||||
local rel_path="${release_file#$output_folder/public/dists/}"
|
|
||||||
# Count slashes - should have exactly 2 for component level: {release}/{component}/Release
|
|
||||||
local slash_count=$(echo "$rel_path" | tr -cd '/' | wc -c)
|
|
||||||
|
|
||||||
if [[ $slash_count -eq 2 ]]; then
|
|
||||||
local distro_path
|
|
||||||
distro_path="$(dirname "$release_file")"
|
|
||||||
echo "Signing release at: $distro_path" | logger -t repo-management
|
|
||||||
gpg "${gpg_params[@]}" --clear-sign -o "$distro_path/InRelease" "$release_file"
|
|
||||||
gpg "${gpg_params[@]}" --detach-sign -o "$distro_path/Release.gpg" "$release_file"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Finalize repository after parallel GitHub Actions workers have built individual releases
|
# Main command dispatcher
|
||||||
# Workers have already built and signed repos in isolated databases, so this just
|
|
||||||
# ensures the GPG key and control file are in place
|
|
||||||
# Arguments:
|
|
||||||
# $1 - Base input folder (contains package sources, for consistency)
|
|
||||||
# $2 - Output folder containing combined repository
|
|
||||||
merge_repos() {
|
|
||||||
local input_folder="$1"
|
|
||||||
local output_folder="$2"
|
|
||||||
|
|
||||||
log "Merge mode: finalizing combined repository"
|
|
||||||
log "Workers have already built and signed individual releases"
|
|
||||||
|
|
||||||
# Repositories are already built and signed by parallel workers
|
|
||||||
# Just need to ensure the key and control file are in place
|
|
||||||
|
|
||||||
# Copy GPG key to repository
|
|
||||||
mkdir -p "${output_folder}"/public/
|
|
||||||
# Remove existing key file if it exists to avoid permission issues
|
|
||||||
rm -f "${output_folder}"/public/armbian.key
|
|
||||||
cp config/armbian.key "${output_folder}"/public/
|
|
||||||
log "Copied GPG key to repository"
|
|
||||||
|
|
||||||
# Write repository sync control file
|
|
||||||
date +%s > ${output_folder}/public/control
|
|
||||||
log "Updated repository control file"
|
|
||||||
|
|
||||||
# Display repository contents
|
|
||||||
showall
|
|
||||||
|
|
||||||
log "Merge complete - repository is ready"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Main repository manipulation dispatcher
|
|
||||||
# Routes commands to appropriate repository management functions
|
|
||||||
# Arguments:
|
# Arguments:
|
||||||
# $1 - Input folder containing packages
|
# $1 - Input folder containing packages
|
||||||
# $2 - Output folder for published repository
|
# $2 - Output folder for published repository
|
||||||
# $3 - Command to execute (update-main, serve, html, delete, show, unique, update, merge)
|
# $3 - Command to execute (serve, html, delete, show, unique, update)
|
||||||
# $4 - GPG password for signing
|
# $4 - GPG password for signing
|
||||||
# $5 - Comma-separated list of releases (used by some commands)
|
# $5 - Comma-separated list of releases
|
||||||
# $6 - List of packages to delete (used by delete command)
|
# $6 - List of packages to delete (used by delete command)
|
||||||
repo-manipulate() {
|
repo-manipulate() {
|
||||||
# Read comma-delimited distros into array
|
# Read comma-delimited distros into array
|
||||||
@ -768,12 +538,6 @@ repo-manipulate() {
|
|||||||
|
|
||||||
case "$3" in
|
case "$3" in
|
||||||
|
|
||||||
update-main)
|
|
||||||
# Build common (main) component - runs once before parallel workers
|
|
||||||
update_main "$1" "$2" "$4"
|
|
||||||
return 0
|
|
||||||
;;
|
|
||||||
|
|
||||||
serve)
|
serve)
|
||||||
# Serve the published repository
|
# Serve the published repository
|
||||||
# Since aptly serve requires published repos in its database, and we use
|
# Since aptly serve requires published repos in its database, and we use
|
||||||
@ -884,17 +648,6 @@ repo-manipulate() {
|
|||||||
# remove old releases from publishing (only drops unsupported releases, not all)
|
# remove old releases from publishing (only drops unsupported releases, not all)
|
||||||
drop_unsupported_releases ""
|
drop_unsupported_releases ""
|
||||||
publishing "$1" "$2" "$3" "$4" "$5"
|
publishing "$1" "$2" "$3" "$4" "$5"
|
||||||
# Only use signing function for non-single-release mode
|
|
||||||
# In single-release mode, workers already signed their components
|
|
||||||
if [[ -z "$SINGLE_RELEASE" ]]; then
|
|
||||||
signing "$2" "DF00FAF1C577104B50BF1D0093D6889F9F0E78D5" "8CFA83D13EB2181EEF5843E41EB30FAF236099FE"
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
|
|
||||||
merge)
|
|
||||||
# Merge repositories from parallel per-release runs
|
|
||||||
# Workers have already signed their releases, just finalize
|
|
||||||
merge_repos "$1" "$2"
|
|
||||||
;;
|
;;
|
||||||
|
|
||||||
*)
|
*)
|
||||||
@ -936,17 +689,12 @@ Usage: $0 [ -short | --long ]
|
|||||||
-r --repository [jammy,sid,bullseye,...] comma-separated list of releases
|
-r --repository [jammy,sid,bullseye,...] comma-separated list of releases
|
||||||
-l --list [\"Name (% linux*)|armbian-config\"] list of packages
|
-l --list [\"Name (% linux*)|armbian-config\"] list of packages
|
||||||
-c --command command to execute
|
-c --command command to execute
|
||||||
-R --single-release [name] process only a single release (for parallel GitHub Actions)
|
|
||||||
example: -R jammy or -R noble
|
|
||||||
|
|
||||||
[show] displays packages in each repository
|
[show] displays packages in each repository
|
||||||
[sign] sign repository
|
|
||||||
[html] displays packages in each repository in html form
|
[html] displays packages in each repository in html form
|
||||||
[serve] serve repository - useful for local diagnostics
|
[serve] serve repository - useful for local diagnostics
|
||||||
[unique] manually select which package should be removed from all repositories
|
[unique] manually select which package should be removed from all repositories
|
||||||
[update] search for packages in input folder and create/update repository
|
[update] search for packages in input folder and create/update repository
|
||||||
[update-main] build common (main) component - run once before parallel workers
|
|
||||||
[merge] merge repositories from parallel per-release runs into final repo
|
|
||||||
[delete] delete package from -l LIST of packages
|
[delete] delete package from -l LIST of packages
|
||||||
|
|
||||||
-d --dry-run perform a full trial run without making any repository changes
|
-d --dry-run perform a full trial run without making any repository changes
|
||||||
@ -957,27 +705,12 @@ Usage: $0 [ -short | --long ]
|
|||||||
(by default, skips packages that are already in the repo)
|
(by default, skips packages that are already in the repo)
|
||||||
-P --force-publish force publishing even when there are no packages to add
|
-P --force-publish force publishing even when there are no packages to add
|
||||||
(by default, skips publishing empty releases)
|
(by default, skips publishing empty releases)
|
||||||
|
|
||||||
GitHub Actions parallel workflow example:
|
|
||||||
# Step 1: Build common (main) component once (optional - workers will create it if missing)
|
|
||||||
./repo.sh -c update-main -i /shared/packages -o /shared/output
|
|
||||||
|
|
||||||
# Step 2: Workers build release-specific components in parallel (isolated DBs)
|
|
||||||
# Worker 1: ./repo.sh -c update -R jammy -k -i /shared/packages -o /shared/output
|
|
||||||
# Worker 2: ./repo.sh -c update -R noble -k -i /shared/packages -o /shared/output
|
|
||||||
# Worker 3: ./repo.sh -c update -R bookworm -k -i /shared/packages -o /shared/output
|
|
||||||
|
|
||||||
# Step 3: Final merge to combine all outputs
|
|
||||||
./repo.sh -c merge -i /shared/packages -o /shared/output
|
|
||||||
|
|
||||||
Note: Each worker uses isolated DB (aptly-isolated-<release>) to avoid locking.
|
|
||||||
Common snapshot is created in each worker's isolated DB from root packages.
|
|
||||||
"
|
"
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
SHORT=i:,l:,o:,c:,p:,r:,h,d,k,R:,F:,P:
|
SHORT=i:,l:,o:,c:,p:,r:,h,d,k,F:,P:
|
||||||
LONG=input:,list:,output:,command:,password:,releases:,help,dry-run,keep-sources,single-release:,force-add:,force-publish:
|
LONG=input:,list:,output:,command:,password:,releases:,help,dry-run,keep-sources,force-add:,force-publish:
|
||||||
if ! OPTS=$(getopt -a -n repo --options $SHORT --longoptions $LONG -- "$@"); then
|
if ! OPTS=$(getopt -a -n repo --options $SHORT --longoptions $LONG -- "$@"); then
|
||||||
help
|
help
|
||||||
exit 1
|
exit 1
|
||||||
@ -1020,10 +753,6 @@ do
|
|||||||
KEEP_SOURCES=true
|
KEEP_SOURCES=true
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-R | --single-release )
|
|
||||||
SINGLE_RELEASE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-F | --force-add )
|
-F | --force-add )
|
||||||
FORCE_ADD=true
|
FORCE_ADD=true
|
||||||
shift
|
shift
|
||||||
@ -1053,41 +782,9 @@ do
|
|||||||
done
|
done
|
||||||
|
|
||||||
# redefine output folder in Aptly
|
# redefine output folder in Aptly
|
||||||
# Use isolated database for single-release mode to avoid DB locking during parallel execution
|
TempDir="$(mktemp -d || exit 1)"
|
||||||
# Use shared database for regular (non-parallel) mode
|
sed 's|"rootDir": ".*"|"rootDir": "'$output'"|g' tools/repository/aptly.conf > "${TempDir}"/aptly.conf
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
CONFIG="${TempDir}/aptly.conf"
|
||||||
# Create isolated aptly directory for this release
|
|
||||||
IsolatedRootDir="${output}/aptly-isolated-${SINGLE_RELEASE}"
|
|
||||||
|
|
||||||
# Create the isolated directory if it doesn't exist
|
|
||||||
if ! mkdir -p "$IsolatedRootDir"; then
|
|
||||||
log "ERROR: mkdir $IsolatedRootDir: permission denied"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Do NOT copy the shared database to isolated DB
|
|
||||||
# This prevents "key not found" errors when the copied DB references packages
|
|
||||||
# that don't exist in the isolated pool. Instead, each worker creates a fresh DB
|
|
||||||
# and builds the common component from packages in the shared input folder.
|
|
||||||
|
|
||||||
# Do NOT link the shared pool either - each isolated DB should have its own pool
|
|
||||||
# Packages will be copied to the isolated pool when they're added via 'aptly repo add'
|
|
||||||
# This prevents hard link issues and "no such file or directory" errors during publish
|
|
||||||
|
|
||||||
# Create temp config file
|
|
||||||
TempDir="$(mktemp -d || exit 1)"
|
|
||||||
|
|
||||||
# Create config with isolated rootDir
|
|
||||||
cat tools/repository/aptly.conf | \
|
|
||||||
sed 's|"rootDir": ".*"|"rootDir": "'$IsolatedRootDir'"|g' > "${TempDir}"/aptly.conf
|
|
||||||
|
|
||||||
CONFIG="${TempDir}/aptly.conf"
|
|
||||||
log "Using isolated aptly root for $SINGLE_RELEASE at: $IsolatedRootDir"
|
|
||||||
else
|
|
||||||
TempDir="$(mktemp -d || exit 1)"
|
|
||||||
sed 's|"rootDir": ".*"|"rootDir": "'$output'"|g' tools/repository/aptly.conf > "${TempDir}"/aptly.conf
|
|
||||||
CONFIG="${TempDir}/aptly.conf"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Display configuration status
|
# Display configuration status
|
||||||
echo "=========================================="
|
echo "=========================================="
|
||||||
@ -1096,14 +793,9 @@ echo " DRY-RUN: $([ "$DRY_RUN" == true ] && echo 'ENABLED' || echo 'disab
|
|||||||
echo " KEEP-SOURCES: $([ "$KEEP_SOURCES" == true ] && echo 'ENABLED' || echo 'disabled')"
|
echo " KEEP-SOURCES: $([ "$KEEP_SOURCES" == true ] && echo 'ENABLED' || echo 'disabled')"
|
||||||
echo " FORCE-ADD: $([ "$FORCE_ADD" == true ] && echo 'ENABLED' || echo 'disabled')"
|
echo " FORCE-ADD: $([ "$FORCE_ADD" == true ] && echo 'ENABLED' || echo 'disabled')"
|
||||||
echo " FORCE-PUBLISH: $([ "$FORCE_PUBLISH" == true ] && echo 'ENABLED' || echo 'disabled')"
|
echo " FORCE-PUBLISH: $([ "$FORCE_PUBLISH" == true ] && echo 'ENABLED' || echo 'disabled')"
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
|
||||||
echo " SINGLE-RELEASE: ENABLED ($SINGLE_RELEASE)"
|
|
||||||
else
|
|
||||||
echo " SINGLE-RELEASE: disabled"
|
|
||||||
fi
|
|
||||||
echo "=========================================="
|
echo "=========================================="
|
||||||
|
|
||||||
log "Configuration: DRY_RUN=$DRY_RUN, KEEP_SOURCES=$KEEP_SOURCES, FORCE_ADD=$FORCE_ADD, FORCE_PUBLISH=$FORCE_PUBLISH, SINGLE_RELEASE=$SINGLE_RELEASE"
|
log "Configuration: DRY_RUN=$DRY_RUN, KEEP_SOURCES=$KEEP_SOURCES, FORCE_ADD=$FORCE_ADD, FORCE_PUBLISH=$FORCE_PUBLISH"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
echo "=========================================="
|
echo "=========================================="
|
||||||
@ -1126,13 +818,6 @@ if [[ "$FORCE_ADD" == true ]]; then
|
|||||||
echo "=========================================="
|
echo "=========================================="
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -n "$SINGLE_RELEASE" ]]; then
|
|
||||||
echo "=========================================="
|
|
||||||
echo "SINGLE RELEASE MODE"
|
|
||||||
echo "Processing only: $SINGLE_RELEASE"
|
|
||||||
echo "=========================================="
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
# main
|
# main
|
||||||
repo-manipulate "$input" "$output" "$command" "$password" "$releases" "$list"
|
repo-manipulate "$input" "$output" "$command" "$password" "$releases" "$list"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user