From 882d7e3dfd8e496999f8cc00927979128758fa21 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sat, 17 Jan 2026 10:56:46 +0100 Subject: [PATCH] docker: add automatic image pull cronjob and cleanup system - Add docker_cleanup_old_images() to remove dangling images and keep only 2 most recent per tag - Add docker_pull_with_marker() to pull images and update marker files tracking last pull time - Add docker_setup_auto_pull_cronjob() to create/update system cronjob and wrapper script via hash-based detection - Add docker_ensure_auto_pull_cronjob() to ensure cronjob is installed and up-to-date - Create self-contained wrapper script at /usr/local/bin/armbian-docker-pull for cron execution - Store configuration hash in /var/lib/armbian/docker-pull.hash for smart update detection - Install cronjob at /etc/cron.d/armbian-docker-pull to pull images every 12 hours - Move cronjob setup from docker_cli_prepare() to requirements command - Cronjob is now only installed when users explicitly run ./compile.sh requirements - Prevents "12 hours since last pull, pulling again" delay during builds Signed-off-by: Igor Pecovnik --- lib/functions/cli/cli-requirements.sh | 6 + lib/functions/host/docker.sh | 221 ++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) diff --git a/lib/functions/cli/cli-requirements.sh b/lib/functions/cli/cli-requirements.sh index 3c5ad4b9be..ffd4665192 100644 --- a/lib/functions/cli/cli-requirements.sh +++ b/lib/functions/cli/cli-requirements.sh @@ -47,4 +47,10 @@ function cli_requirements_run() { fi display_alert "Done with" "@host dependencies" "cachehit" + + # Ensure Docker auto-pull cronjob is installed if Docker is available + if [[ -n "$(command -v docker)" ]]; then + display_alert "Docker" "ensuring auto-pull cronjob is installed" "info" + docker_ensure_auto_pull_cronjob + fi } diff --git a/lib/functions/host/docker.sh b/lib/functions/host/docker.sh index 6b1e394a4a..6096480098 100644 --- a/lib/functions/host/docker.sh +++ b/lib/functions/host/docker.sh @@ -154,6 +154,10 @@ function docker_cli_prepare() { exit 56 fi + ############################################################################################################# + # Cleanup old Docker images to free disk space + docker_cleanup_old_images + ############################################################################################################# # Detect some docker info; use cached. get_docker_info_once @@ -640,3 +644,220 @@ function docker_purge_deprecated_volumes() { fi done } + +# Clean old/unused Docker images to free disk space +# Removes dangling images and keeps only the 2 most recent armbian images per tag +function docker_cleanup_old_images() { + display_alert "Cleaning old Docker images" "removing dangling and keeping only 2 most recent per tag" "info" + + # Remove dangling images (layers with no tags) + display_alert "Pruning dangling images" "docker image prune -f" "debug" + docker image prune -f > /dev/null 2>&1 || true + + # For each armbian image tag, keep only the 2 most recent + declare image_tags=() + while IFS= read -r line; do + image_tags+=("$line") + done < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep "docker-armbian-build" | sort -u) + + for image_tag in "${image_tags[@]}"; do + # Get all image IDs for this tag, sorted by creation date (newest first) + declare -a image_ids=() + while IFS= read -r line; do + image_ids+=("$line") + done < <(docker images --format '{{.ID}} {{.CreatedAt}}' "${image_tag}" | sort -r -k2,2 -k3,3 -k4,4 -k5,5 | awk '{print $1}') + + # Remove images beyond the first 2 (keep newest 2) + if [[ ${#image_ids[@]} -gt 2 ]]; then + for ((i=2; i<${#image_ids[@]}; i++)); do + display_alert "Removing old image" "${image_tag}:${image_ids[$i]}" "debug" + docker rmi "${image_ids[$i]}" > /dev/null 2>&1 || true + done + fi + done + + display_alert "Docker cleanup complete" "dangling images removed, old armbian images pruned" "info" +} + +# Pull a Docker image and update the marker file to track when it was last pulled +# Usage: docker_pull_with_marker +function docker_pull_with_marker() { + declare image_name="$1" + declare docker_marker_dir="${SRC}"/cache/docker + + # If cache dir exists, but we can't write to cache dir... + if [[ -d "${SRC}"/cache ]] && [[ ! -w "${SRC}"/cache ]]; then + docker_marker_dir="${SRC}"/.tmp/docker + fi + + run_host_command_logged mkdir -p "${docker_marker_dir}" + + display_alert "Pulling Docker image" "${image_name}" "info" + + if docker pull "${image_name}"; then + # Update marker file after successful pull + declare local_image_sha + local_image_sha="$(docker images --no-trunc --quiet "${image_name}")" + if [[ -n "${local_image_sha}" ]]; then + echo "${image_name}|${local_image_sha}|$(date +%s)" >> "${docker_marker_dir}"/last-pull + display_alert "Updated pull marker" "${image_name}" "debug" + fi + return 0 + else + display_alert "Failed to pull" "${image_name}" "wrn" + return 1 + fi +} + +# Setup or update system cronjob to automatically pull Docker images +# This ensures images are always fresh before builds start +function docker_setup_auto_pull_cronjob() { + if [[ ! -d /etc/cron.d ]]; then + exit_with_error "Docker auto-pull cronjob" "cron not available; /etc/cron.d does not exist on this system" + fi + declare cron_file="/etc/cron.d/armbian-docker-pull" + declare wrapper_script="/usr/local/bin/armbian-docker-pull" + declare hash_file="/var/lib/armbian/docker-pull.hash" + + # Determine which images to pull based on common base images + declare -a images_to_pull=( + "ghcr.io/armbian/docker-armbian-build:armbian-ubuntu-noble-latest" + "ghcr.io/armbian/docker-armbian-build:armbian-debian-trixie-latest" + ) + + # Generate the wrapper script content (self-contained) + declare wrapper_content + wrapper_content=$(cat <<- 'EOT' + #!/usr/bin/env bash + # Auto-generated by Armbian build framework + # Pulls Docker images and updates markers to prevent unnecessary re-pulls + # DO NOT EDIT MANUALLY - this file is regenerated by the build system + + set -e + set -o pipefail + + SRC="__SRC_PLACEHOLDER__" + MARKER_DIR="${SRC}/cache/docker" + + # Fallback to .tmp if cache is not writable + if [[ -d "${SRC}/cache" ]] && [[ ! -w "${SRC}/cache" ]]; then + MARKER_DIR="${SRC}/.tmp/docker" + fi + + mkdir -p "${MARKER_DIR}" + + # Simple logging function + log() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | logger -t armbian-docker-pull + } + + # Pull a Docker image and update the marker file + pull_with_marker() { + local image_name="$1" + + log "Pulling Docker image: ${image_name}" + + if docker pull "${image_name}" 2>&1 | logger -t armbian-docker-pull; then + # Update marker file after successful pull + local local_image_sha + local_image_sha="$(docker images --no-trunc --quiet "${image_name}")" + if [[ -n "${local_image_sha}" ]]; then + echo "${image_name}|${local_image_sha}|$(date +%s)" >> "${MARKER_DIR}/last-pull" + log "Updated pull marker for: ${image_name}" + fi + return 0 + else + log "Failed to pull: ${image_name}" + return 1 + fi + } + + # Pull each image + __IMAGE_COMMANDS__ + EOT + ) + + # Replace placeholders with actual values + wrapper_content="${wrapper_content//__SRC_PLACEHOLDER__/${SRC}}" + declare image_commands="" + for image in "${images_to_pull[@]}"; do + image_commands+="pull_with_marker \"${image}\""$'\n' + done + wrapper_content="${wrapper_content//__IMAGE_COMMANDS__/${image_commands}}" + + # Calculate hash of the wrapper content + declare current_wrapper_hash + current_wrapper_hash="$(echo "${wrapper_content}" | sha256sum | cut -d' ' -f1)" + + # Generate the cron file content + declare cron_content + cron_content=$(cat <<- 'EOT' + # Armbian Docker image auto-pull + # Pulls Docker images every 12 hours to keep them fresh + # This prevents the '12 hours since last pull, pulling again' delay during builds + # DO NOT EDIT MANUALLY - this file is regenerated by the build system + EOT + ) + declare cron_user="${ARMBIAN_DOCKER_PULL_USER:-${SUDO_USER:-$(whoami)}}" + cron_content="${cron_content}"$'\n'"0 */12 * * * ${cron_user} ${wrapper_script} 2>&1 | logger -t armbian-docker-pull" + + # Calculate combined hash (wrapper + cron content) + declare current_hash="${current_wrapper_hash}" + cron_hash="$(echo "${cron_content}" | sha256sum | cut -d' ' -f1)" + current_hash="$(echo "${current_hash}${cron_hash}" | sha256sum | cut -d' ' -f1)" + + # Check if we need to update + declare needs_update="yes" + if [[ -f "${hash_file}" ]]; then + declare stored_hash + stored_hash="$(cat "${hash_file}")" + if [[ "${stored_hash}" == "${current_hash}" ]]; then + needs_update="no" + display_alert "Docker auto-pull" "configuration unchanged, no update needed" "debug" + else + display_alert "Docker auto-pull" "configuration changed, updating" "info" + fi + fi + + if [[ "${needs_update}" == "yes" ]]; then + # Create/update wrapper script + display_alert "Creating/updating Docker auto-pull wrapper script" "${wrapper_script}" "info" + if ! echo "${wrapper_content}" | sudo tee "${wrapper_script}" > /dev/null 2>&1; then + display_alert "Docker auto-pull" "failed to create wrapper script (sudo required)" "warn" + return 0 + fi + sudo chmod +x "${wrapper_script}" || true + + # Create/update cron file + display_alert "Creating/updating Docker auto-pull cronjob" "${cron_file}" "info" + echo "${cron_content}" | sudo tee "${cron_file}" > /dev/null + sudo chmod 600 "${cron_file}" + + # Store hash for next time + sudo mkdir -p "$(dirname "${hash_file}")" + echo "${current_hash}" | sudo tee "${hash_file}" > /dev/null + sudo chmod 644 "${hash_file}" + + # Verify cron service is running + if systemctl is-active --quiet cron || systemctl is-active --quiet crond; then + display_alert "Docker auto-pull cronjob" "installed/updated successfully - images will be pulled every 12 hours" "info" + else + display_alert "Docker auto-pull cronjob" "installed/updated but cron service not active" "warn" + fi + fi +} + +# Check if auto-pull cronjob is installed, and install if not or outdated +function docker_ensure_auto_pull_cronjob() { + declare wrapper_script="/usr/local/bin/armbian-docker-pull" + declare hash_file="/var/lib/armbian/docker-pull.hash" + + # Always call docker_setup_auto_pull_cronjob - it will check hashes and only update if needed + if [[ ! -f "${wrapper_script}" ]] || [[ ! -f "${hash_file}" ]]; then + display_alert "Docker auto-pull cronjob" "wrapper or hash file missing, installing now" "info" + docker_setup_auto_pull_cronjob + else + # Still call setup to check for updates via hash comparison + docker_setup_auto_pull_cronjob + fi +}