docker: add automatic image pull cronjob and cleanup system
- Add docker_cleanup_old_images() to remove dangling images and keep only 2 most recent per tag - Add docker_pull_with_marker() to pull images and update marker files tracking last pull time - Add docker_setup_auto_pull_cronjob() to create/update system cronjob and wrapper script via hash-based detection - Add docker_ensure_auto_pull_cronjob() to ensure cronjob is installed and up-to-date - Create self-contained wrapper script at /usr/local/bin/armbian-docker-pull for cron execution - Store configuration hash in /var/lib/armbian/docker-pull.hash for smart update detection - Install cronjob at /etc/cron.d/armbian-docker-pull to pull images every 12 hours - Move cronjob setup from docker_cli_prepare() to requirements command - Cronjob is now only installed when users explicitly run ./compile.sh requirements - Prevents "12 hours since last pull, pulling again" delay during builds Signed-off-by: Igor Pecovnik <igor@armbian.com>
This commit is contained in:
Igor Pecovnik
Igor
parent
a01887d50d
commit
882d7e3dfd
@ -47,4 +47,10 @@ function cli_requirements_run() {
|
||||
fi
|
||||
|
||||
display_alert "Done with" "@host dependencies" "cachehit"
|
||||
|
||||
# Ensure Docker auto-pull cronjob is installed if Docker is available
|
||||
if [[ -n "$(command -v docker)" ]]; then
|
||||
display_alert "Docker" "ensuring auto-pull cronjob is installed" "info"
|
||||
docker_ensure_auto_pull_cronjob
|
||||
fi
|
||||
}
|
||||
|
||||
@ -154,6 +154,10 @@ function docker_cli_prepare() {
|
||||
exit 56
|
||||
fi
|
||||
|
||||
#############################################################################################################
|
||||
# Cleanup old Docker images to free disk space
|
||||
docker_cleanup_old_images
|
||||
|
||||
#############################################################################################################
|
||||
# Detect some docker info; use cached.
|
||||
get_docker_info_once
|
||||
@ -640,3 +644,220 @@ function docker_purge_deprecated_volumes() {
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Clean old/unused Docker images to free disk space
|
||||
# Removes dangling images and keeps only the 2 most recent armbian images per tag
|
||||
function docker_cleanup_old_images() {
|
||||
display_alert "Cleaning old Docker images" "removing dangling and keeping only 2 most recent per tag" "info"
|
||||
|
||||
# Remove dangling images (layers with no tags)
|
||||
display_alert "Pruning dangling images" "docker image prune -f" "debug"
|
||||
docker image prune -f > /dev/null 2>&1 || true
|
||||
|
||||
# For each armbian image tag, keep only the 2 most recent
|
||||
declare image_tags=()
|
||||
while IFS= read -r line; do
|
||||
image_tags+=("$line")
|
||||
done < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep "docker-armbian-build" | sort -u)
|
||||
|
||||
for image_tag in "${image_tags[@]}"; do
|
||||
# Get all image IDs for this tag, sorted by creation date (newest first)
|
||||
declare -a image_ids=()
|
||||
while IFS= read -r line; do
|
||||
image_ids+=("$line")
|
||||
done < <(docker images --format '{{.ID}} {{.CreatedAt}}' "${image_tag}" | sort -r -k2,2 -k3,3 -k4,4 -k5,5 | awk '{print $1}')
|
||||
|
||||
# Remove images beyond the first 2 (keep newest 2)
|
||||
if [[ ${#image_ids[@]} -gt 2 ]]; then
|
||||
for ((i=2; i<${#image_ids[@]}; i++)); do
|
||||
display_alert "Removing old image" "${image_tag}:${image_ids[$i]}" "debug"
|
||||
docker rmi "${image_ids[$i]}" > /dev/null 2>&1 || true
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
display_alert "Docker cleanup complete" "dangling images removed, old armbian images pruned" "info"
|
||||
}
|
||||
|
||||
# Pull a Docker image and update the marker file to track when it was last pulled
|
||||
# Usage: docker_pull_with_marker <image_name>
|
||||
function docker_pull_with_marker() {
|
||||
declare image_name="$1"
|
||||
declare docker_marker_dir="${SRC}"/cache/docker
|
||||
|
||||
# If cache dir exists, but we can't write to cache dir...
|
||||
if [[ -d "${SRC}"/cache ]] && [[ ! -w "${SRC}"/cache ]]; then
|
||||
docker_marker_dir="${SRC}"/.tmp/docker
|
||||
fi
|
||||
|
||||
run_host_command_logged mkdir -p "${docker_marker_dir}"
|
||||
|
||||
display_alert "Pulling Docker image" "${image_name}" "info"
|
||||
|
||||
if docker pull "${image_name}"; then
|
||||
# Update marker file after successful pull
|
||||
declare local_image_sha
|
||||
local_image_sha="$(docker images --no-trunc --quiet "${image_name}")"
|
||||
if [[ -n "${local_image_sha}" ]]; then
|
||||
echo "${image_name}|${local_image_sha}|$(date +%s)" >> "${docker_marker_dir}"/last-pull
|
||||
display_alert "Updated pull marker" "${image_name}" "debug"
|
||||
fi
|
||||
return 0
|
||||
else
|
||||
display_alert "Failed to pull" "${image_name}" "wrn"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup or update system cronjob to automatically pull Docker images
|
||||
# This ensures images are always fresh before builds start
|
||||
function docker_setup_auto_pull_cronjob() {
|
||||
if [[ ! -d /etc/cron.d ]]; then
|
||||
exit_with_error "Docker auto-pull cronjob" "cron not available; /etc/cron.d does not exist on this system"
|
||||
fi
|
||||
declare cron_file="/etc/cron.d/armbian-docker-pull"
|
||||
declare wrapper_script="/usr/local/bin/armbian-docker-pull"
|
||||
declare hash_file="/var/lib/armbian/docker-pull.hash"
|
||||
|
||||
# Determine which images to pull based on common base images
|
||||
declare -a images_to_pull=(
|
||||
"ghcr.io/armbian/docker-armbian-build:armbian-ubuntu-noble-latest"
|
||||
"ghcr.io/armbian/docker-armbian-build:armbian-debian-trixie-latest"
|
||||
)
|
||||
|
||||
# Generate the wrapper script content (self-contained)
|
||||
declare wrapper_content
|
||||
wrapper_content=$(cat <<- 'EOT'
|
||||
#!/usr/bin/env bash
|
||||
# Auto-generated by Armbian build framework
|
||||
# Pulls Docker images and updates markers to prevent unnecessary re-pulls
|
||||
# DO NOT EDIT MANUALLY - this file is regenerated by the build system
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
SRC="__SRC_PLACEHOLDER__"
|
||||
MARKER_DIR="${SRC}/cache/docker"
|
||||
|
||||
# Fallback to .tmp if cache is not writable
|
||||
if [[ -d "${SRC}/cache" ]] && [[ ! -w "${SRC}/cache" ]]; then
|
||||
MARKER_DIR="${SRC}/.tmp/docker"
|
||||
fi
|
||||
|
||||
mkdir -p "${MARKER_DIR}"
|
||||
|
||||
# Simple logging function
|
||||
log() {
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | logger -t armbian-docker-pull
|
||||
}
|
||||
|
||||
# Pull a Docker image and update the marker file
|
||||
pull_with_marker() {
|
||||
local image_name="$1"
|
||||
|
||||
log "Pulling Docker image: ${image_name}"
|
||||
|
||||
if docker pull "${image_name}" 2>&1 | logger -t armbian-docker-pull; then
|
||||
# Update marker file after successful pull
|
||||
local local_image_sha
|
||||
local_image_sha="$(docker images --no-trunc --quiet "${image_name}")"
|
||||
if [[ -n "${local_image_sha}" ]]; then
|
||||
echo "${image_name}|${local_image_sha}|$(date +%s)" >> "${MARKER_DIR}/last-pull"
|
||||
log "Updated pull marker for: ${image_name}"
|
||||
fi
|
||||
return 0
|
||||
else
|
||||
log "Failed to pull: ${image_name}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Pull each image
|
||||
__IMAGE_COMMANDS__
|
||||
EOT
|
||||
)
|
||||
|
||||
# Replace placeholders with actual values
|
||||
wrapper_content="${wrapper_content//__SRC_PLACEHOLDER__/${SRC}}"
|
||||
declare image_commands=""
|
||||
for image in "${images_to_pull[@]}"; do
|
||||
image_commands+="pull_with_marker \"${image}\""$'\n'
|
||||
done
|
||||
wrapper_content="${wrapper_content//__IMAGE_COMMANDS__/${image_commands}}"
|
||||
|
||||
# Calculate hash of the wrapper content
|
||||
declare current_wrapper_hash
|
||||
current_wrapper_hash="$(echo "${wrapper_content}" | sha256sum | cut -d' ' -f1)"
|
||||
|
||||
# Generate the cron file content
|
||||
declare cron_content
|
||||
cron_content=$(cat <<- 'EOT'
|
||||
# Armbian Docker image auto-pull
|
||||
# Pulls Docker images every 12 hours to keep them fresh
|
||||
# This prevents the '12 hours since last pull, pulling again' delay during builds
|
||||
# DO NOT EDIT MANUALLY - this file is regenerated by the build system
|
||||
EOT
|
||||
)
|
||||
declare cron_user="${ARMBIAN_DOCKER_PULL_USER:-${SUDO_USER:-$(whoami)}}"
|
||||
cron_content="${cron_content}"$'\n'"0 */12 * * * ${cron_user} ${wrapper_script} 2>&1 | logger -t armbian-docker-pull"
|
||||
|
||||
# Calculate combined hash (wrapper + cron content)
|
||||
declare current_hash="${current_wrapper_hash}"
|
||||
cron_hash="$(echo "${cron_content}" | sha256sum | cut -d' ' -f1)"
|
||||
current_hash="$(echo "${current_hash}${cron_hash}" | sha256sum | cut -d' ' -f1)"
|
||||
|
||||
# Check if we need to update
|
||||
declare needs_update="yes"
|
||||
if [[ -f "${hash_file}" ]]; then
|
||||
declare stored_hash
|
||||
stored_hash="$(cat "${hash_file}")"
|
||||
if [[ "${stored_hash}" == "${current_hash}" ]]; then
|
||||
needs_update="no"
|
||||
display_alert "Docker auto-pull" "configuration unchanged, no update needed" "debug"
|
||||
else
|
||||
display_alert "Docker auto-pull" "configuration changed, updating" "info"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${needs_update}" == "yes" ]]; then
|
||||
# Create/update wrapper script
|
||||
display_alert "Creating/updating Docker auto-pull wrapper script" "${wrapper_script}" "info"
|
||||
if ! echo "${wrapper_content}" | sudo tee "${wrapper_script}" > /dev/null 2>&1; then
|
||||
display_alert "Docker auto-pull" "failed to create wrapper script (sudo required)" "warn"
|
||||
return 0
|
||||
fi
|
||||
sudo chmod +x "${wrapper_script}" || true
|
||||
|
||||
# Create/update cron file
|
||||
display_alert "Creating/updating Docker auto-pull cronjob" "${cron_file}" "info"
|
||||
echo "${cron_content}" | sudo tee "${cron_file}" > /dev/null
|
||||
sudo chmod 600 "${cron_file}"
|
||||
|
||||
# Store hash for next time
|
||||
sudo mkdir -p "$(dirname "${hash_file}")"
|
||||
echo "${current_hash}" | sudo tee "${hash_file}" > /dev/null
|
||||
sudo chmod 644 "${hash_file}"
|
||||
|
||||
# Verify cron service is running
|
||||
if systemctl is-active --quiet cron || systemctl is-active --quiet crond; then
|
||||
display_alert "Docker auto-pull cronjob" "installed/updated successfully - images will be pulled every 12 hours" "info"
|
||||
else
|
||||
display_alert "Docker auto-pull cronjob" "installed/updated but cron service not active" "warn"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if auto-pull cronjob is installed, and install if not or outdated
|
||||
function docker_ensure_auto_pull_cronjob() {
|
||||
declare wrapper_script="/usr/local/bin/armbian-docker-pull"
|
||||
declare hash_file="/var/lib/armbian/docker-pull.hash"
|
||||
|
||||
# Always call docker_setup_auto_pull_cronjob - it will check hashes and only update if needed
|
||||
if [[ ! -f "${wrapper_script}" ]] || [[ ! -f "${hash_file}" ]]; then
|
||||
display_alert "Docker auto-pull cronjob" "wrapper or hash file missing, installing now" "info"
|
||||
docker_setup_auto_pull_cronjob
|
||||
else
|
||||
# Still call setup to check for updates via hash comparison
|
||||
docker_setup_auto_pull_cronjob
|
||||
fi
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user