Add support for month cache, limit push to master branch (#3939)

* Add support for month cache, limit push to master branch * Bump cache version * Add calculator * Control file can't be zero lenght * Remove non-existing package * Remove non-existing pkg * Wrong secret used * Fix broken packages * Improve download methods. No more errors * Fix mechanism for using old cache files
2022-06-28 18:04:31 +02:00 · 2022-06-28 18:04:31 +02:00 · 20ee8c5450
commit 20ee8c5450
parent 19d1ce656e
13 changed files with 169 additions and 34 deletions
--- a/.github/workflows/update-cache.yml
+++ b/.github/workflows/update-cache.yml
@ -1,17 +1,29 @@
 name: Build Rootfs Cache
 #
-# Generates rootfs cache and uploads it to https://github.com/armbian/mirror/releases/tag/_rootfs and https://dl.armbian.com/_rootfs/
+# Generates rootfs cache and uploads it to:
+#
+#     https://github.com/armbian/mirror/releases/tag/rootfs 
+#     https://cache.armbian.com/rootfs/
 #

 on:
  schedule:
-    - cron: "30 0 1 * *"
+      - cron: "30 0 1 * *" # Runs at 00:30 UTC on the 1st of every month.
+    
  push:
-      paths:
+    branches:
+      - 'master'
+    paths:
      - 'config/distributions/**'
      - 'config/cli/**'
      - 'config/desktop/**'
+
  workflow_dispatch:
+    inputs:
+      monthoffset:
+        description: Forced month offset
+        required: true
+        default: 0

 jobs:

@ -20,8 +32,20 @@ jobs:
      contents: none
    name: Release start
    runs-on: [X64]
+    outputs:
+      rootfscache_version: ${{ steps.env-vars.outputs.rootfscache_version }}
    if: ${{ github.repository_owner == 'Armbian' }}
    steps:
+    
+      # Synyching procedure expects this value
+      - run: |
+          echo "not empty" > changes
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: changes
+          name: changes
+          if-no-files-found: ignore

      - name: Checkout repository
        uses: actions/checkout@v3
@ -30,11 +54,48 @@ jobs:
          path: build
          clean: false

-      - name: Updating releases
-        uses: ncipollo/release-action@v1
+      - name: Remove current status
+        run: |
+          sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
+          sudo apt-get -y -qq install sshfs
+          sudo mkdir -p build/cache/rootfs.upload || true
+          
+          # locally mount via NFS
+          if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
+              sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload
+          else
+              sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
+          fi
+          # remove true in sshfs when all runners are on jammy
+
+          sudo rm build/cache/rootfs.upload/* || true
+          sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
+
+      - id: env-vars
+        name: Read current version
+        run: |
+          
+          ROOTFSCACHE_VERSION=$(wget --tries=10 -O - -o /dev/null https://github.com/armbian/mirror/releases/download/rootfs/rootfscache.version || true)
+          ROOTFSCACHE_VERSION=$(( ${ROOTFSCACHE_VERSION:-"0"} + 1 ))
+          echo "${ROOTFSCACHE_VERSION}"
+          echo "$ROOTFSCACHE_VERSION" | sudo tee rootfscache.version
+          echo ::set-output name=rootfscache_version::$(echo ${ROOTFSCACHE_VERSION}) || true
+
+      - uses: dev-drprasad/delete-tag-and-release@v0.2.0
+        if: ${{ github.repository_owner == 'Armbian' }}
        with:
+          delete_release: true
+          repo: 'armbian/mirror'
+          tag_name: "rootfs"
+        env:
+          GITHUB_TOKEN: ${{ secrets.CR_PAT }}
+
+      - name: Updating releases
+        uses: ncipollo/release-action@v1.10.0
+        with:
+          artifacts: "rootfscache.version"
          repo: "mirror"
-          tag: "_rootfs"
+          tag: "rootfs"
          name: "Build in progress"
          bodyFile: "build/.github/Releases-wip.md"
          allowUpdates: true
@ -50,6 +111,8 @@ jobs:
    with:

      variant: 'minimal:uefi-x86'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -70,6 +133,8 @@ jobs:
    with:

      variant: 'server:uefi-x86'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -90,6 +155,8 @@ jobs:
    with:

      variant: 'desktop:uefi-x86'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -110,6 +177,8 @@ jobs:
    with:

      variant: 'minimal:tinkerboard'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -131,6 +200,8 @@ jobs:
    with:

      variant: 'server:tinkerboard'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -150,6 +221,8 @@ jobs:

    with:
      variant: 'desktop:tinkerboard'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -170,6 +243,8 @@ jobs:
    with:

      variant: 'minimal:uefi-arm64'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -190,6 +265,8 @@ jobs:
    with:

      variant: 'server:uefi-arm64'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -209,6 +286,8 @@ jobs:

    with:
      variant: 'desktop:uefi-arm64'
+      monthoffset: ${{ inputs.monthoffset }}
+      rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }}

    secrets:
      PAT1: ${{ secrets.CR_PAT }}
@ -223,7 +302,7 @@ jobs:
  jobsend:
    permissions:
      contents: none
-    name: finish
+    name: Release finish
    needs: [x86-min,x86-srv,x86-x,armhf-min,armhf-srv,armhf-x,arm64-min,arm64-srv,arm64-x]
    runs-on: [fast]
    if: ${{ github.repository_owner == 'Armbian' }}
@ -283,9 +362,10 @@ jobs:
          sudo apt-get -y -qq install parallel buildtorrent

          ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | head -1)ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1)
-          TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-)
-          WEBSEEDS="--webseeds=https://github.com/armbian/mirror/releases/download/_rootfs/$FILE,https://imola.armbian.com/dl/_rootfs/$FILE,https://stpete-mirror.armbian.com/dl/_rootfs/$FILE"
+          TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-)          
+          WEBSEEDS="--webseeds="https://github.com/armbian/mirror/releases/download/rootfs/\$FILE,"$(curl -s https://cache.armbian.com/mirrors | jq -r '.'default' | .[] | values' | sed -e 's/$/rootfs\/$FILE/' |  tr '\n' , | sed 's/.$//')"
          cd build/cache/rootfs.upload
+
          FILES=$(ls -1 *.lz4)
          for FILE in ${FILES[@]}
          do
@ -297,21 +377,48 @@ jobs:
              echo "Generating {}.torrent "; buildtorrent -q -s -m '$WEBSEEDS' --announce="'$ANNOUNCE'" --announcelist="'$TRACKERS'" {} -c "Armbian rootfs cache" {}.torrent >/dev/null
          '

+          # Update version
+          echo "${{ needs.release-start.outputs.rootfscache_version }}" | sudo tee rootfscache.version
+
      - name: Upload
        uses: ncipollo/release-action@v1
        if: ${{ github.repository_owner == 'Armbian' }}
        with:
          repo: "mirror"
-          artifacts: "build/cache/rootfs/*.torrent,build/cache/rootfs/*.asc"
-          tag: "_rootfs"
+          artifacts: "build/cache/rootfs.upload/*.torrent,build/cache/rootfs.upload/*.asc"
+          tag: "rootfs"
          bodyFile: "build/.github/Releases.md"
          name: "Rootfs cache"
          allowUpdates: true
-          token: ${{ secrets.PAT }}
+          token: ${{ secrets.CR_PAT }}
+
+      - name: Delete obsolete
+        run: |
+
+          BRISI=($(diff <(find build/cache/rootfs.upload -name "*.lz4.current" | sed "s/.current//" | sort) <(find build/cache/rootfs.upload -name "*.lz4" | sort) | grep ">" | sed "s/> //"))
+          for brisi in "${BRISI[@]}"; do
+              sudo rm $brisi.*
+          done

      - name: Unmount folders
-
        run: |

          sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
          sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
+
+  final-sync:
+    permissions:
+      contents: none
+    name: "Sync servers"
+    needs: [jobsend]
+    if: ${{ success() && github.repository_owner == 'Armbian' }}
+    uses: armbian/scripts/.github/workflows/sync-servers.yml@master
+
+    with:
+      KEY_ID: 'upload'
+
+    secrets:
+      KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }}
+      USER_REPOSITORY: ${{ secrets.USER_REPOSITORY }}
+      HOST_REPOSITORY: ${{ secrets.HOST_REPOSITORY }}
+      KNOWN_HOSTS_REPOSITORY: ${{ secrets.KNOWN_HOSTS_REPOSITORY }}
--- a/config/desktop/focal/appgroups/desktop_tools/packages
+++ b/config/desktop/focal/appgroups/desktop_tools/packages
@ -1,4 +1,3 @@
 bleachbit
 fbi
 gparted
-system-monitoring-center
--- a/config/desktop/focal/environments/gnome/config_base/packages
+++ b/config/desktop/focal/environments/gnome/config_base/packages
@ -200,6 +200,7 @@ python3-distupgrade
 python3-update-manager
 python3-xkit
 python3-yaml
+qalculate-gtk
 slick-greeter
 software-properties-gtk
 spice-vdagent
--- a/config/desktop/jammy/appgroups/desktop_tools
+++ b/config/desktop/jammy/appgroups/desktop_tools
@ -1 +0,0 @@
-../../focal/appgroups/desktop_tools
--- a/config/desktop/jammy/appgroups/desktop_tools/packages
+++ b/config/desktop/jammy/appgroups/desktop_tools/packages
@ -0,0 +1,4 @@
+bleachbit
+fbi
+gparted
+system-monitoring-center
--- a/config/desktop/jammy/environments/gnome/config_base/packages
+++ b/config/desktop/jammy/environments/gnome/config_base/packages
@ -120,6 +120,7 @@ python3-distupgrade
 python3-update-manager
 python3-xkit
 python3-yaml
+qalculate-gtk
 slick-greeter
 software-properties-gtk
 spice-vdagent
--- a/config/desktop/jammy/environments/xfce/config_base/packages
+++ b/config/desktop/jammy/environments/xfce/config_base/packages
@ -114,6 +114,7 @@ policykit-desktop-privileges
 printer-driver-all
 profile-sync-daemon
 pulseaudio-module-bluetooth
+qalculate-gtk
 redshift
 slick-greeter
 smbclient
--- a/config/desktop/sid/environments/budgie/config_base/packages
+++ b/config/desktop/sid/environments/budgie/config_base/packages
@ -122,6 +122,7 @@ policykit-1-gnome
 printer-driver-all
 profile-sync-daemon
 pulseaudio-module-bluetooth
+qalculate-gtk
 redshift
 samba
 slick-greeter
--- a/config/desktop/sid/environments/gnome/config_base/packages
+++ b/config/desktop/sid/environments/gnome/config_base/packages
@ -106,6 +106,7 @@ python3-debian
 python3-distutils
 python3-distutils-extra
 python3-yaml
+qalculate-gtk
 software-properties-gtk
 spice-vdagent
 system-config-printer
--- a/config/desktop/sid/environments/xfce/config_base/packages
+++ b/config/desktop/sid/environments/xfce/config_base/packages
@ -32,6 +32,7 @@ printer-driver-all
 profile-sync-daemon
 pulseaudio
 pulseaudio-module-bluetooth
+qalculate-gtk
 samba
 slick-greeter
 smbclient
--- a/lib/configuration.sh
+++ b/lib/configuration.sh
@ -29,7 +29,13 @@ HOSTRELEASE=$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2)
 [[ -z $EXIT_PATCHING_ERROR ]] && EXIT_PATCHING_ERROR="" # exit patching if failed
 [[ -z $HOST ]] && HOST="$BOARD" # set hostname to the board
 cd "${SRC}" || exit
-[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=20
+
+# if variable not provided, check which is current version in the cache storage
+if [[ -z "${ROOTFSCACHE_VERSION}" ]]; then
+	ROOTFSCACHE_VERSION=$(wget --tries=10 -O - -o /dev/null https://github.com/armbian/mirror/releases/download/rootfs/rootfscache.version || true)
+	ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-"0"}
+fi
+
 [[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7
 BUILD_REPOSITORY_URL=$(improved_git remote get-url $(improved_git remote 2>/dev/null | grep origin) 2>/dev/null)
 BUILD_REPOSITORY_COMMIT=$(improved_git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null)
--- a/lib/debootstrap.sh
+++ b/lib/debootstrap.sh
@ -128,8 +128,9 @@ create_rootfs_cache()
 	# seek last cache, proceed to previous otherwise build it
 	for ((n=0;n<${cycles};n++)); do

-		[[ -z ${FORCED_MONTH_OFFSET} ]] && FORCED_MONTH_OFFSET=${n}
-		local packages_hash=$(get_package_list_hash "$(date -d "$D +${FORCED_MONTH_OFFSET} month" +"%Y-%m-module$ROOTFSCACHE_VERSION" | sed 's/^0*//')")
+		FORCED_MONTH_OFFSET=${n}
+
+		local packages_hash=$(get_package_list_hash "$(date -d "$D -${FORCED_MONTH_OFFSET} month" +"%Y-%m-module$ROOTFSCACHE_VERSION" | sed 's/^0*//')")
 		local cache_type="cli"
 		[[ ${BUILD_DESKTOP} == yes ]] && local cache_type="xfce-desktop"
 		[[ -n ${DESKTOP_ENVIRONMENT} ]] && local cache_type="${DESKTOP_ENVIRONMENT}"
@ -149,7 +150,7 @@ create_rootfs_cache()
 		display_alert "Checking local cache" "$display_name" "info"

 		if [[ -f ${cache_fname} && -n "$ROOT_FS_CREATE_ONLY" ]]; then
-			touch $cache_fname.current
+			echo "$cache_fname" > $cache_fname.current
 			display_alert "Checking cache integrity" "$display_name" "info"
 			sudo lz4 -tqq ${cache_fname}
 			[[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Please restart!"
@ -164,6 +165,7 @@ create_rootfs_cache()
 		else
 			display_alert "searching on servers"
 			download_and_verify "_rootfs" "$cache_name"
+			[[ -f ${cache_fname} ]] && break
 		fi

 		if [[ ! -f $cache_fname ]]; then
@ -176,7 +178,7 @@ create_rootfs_cache()

 		# speed up checking
 		if [[ -n "$ROOT_FS_CREATE_ONLY" ]]; then
-			touch $cache_fname.current
+			echo "$cache_fname" > $cache_fname.current
 			umount --lazy "$SDCARD"
 			rm -rf $SDCARD
 			# remove exit trap
@ -393,7 +395,7 @@ create_rootfs_cache()
 		fi

 		# needed for backend to keep current only
-		touch $cache_fname.current
+		echo "$cache_fname" > $cache_fname.current

 	fi

--- a/lib/general.sh
+++ b/lib/general.sh
@ -1619,21 +1619,26 @@ prepare_host()

 function webseed ()
 {
+
 	# list of mirrors that host our files
 	unset text
-	# Hardcoded to EU mirrors since
 	local CCODE=$(curl -s redirect.armbian.com/geoip | jq '.continent.code' -r)
-	WEBSEED=($(curl -s https://redirect.armbian.com/mirrors | jq -r '.'${CCODE}' | .[] | values'))
+
+	if [[ "$2" == "rootfs" ]]; then
+		WEBSEED=($(curl -s ${1}mirrors | jq -r '.'${CCODE}' | .[] | values'))
+		else
+		WEBSEED=($(curl -s https://redirect.armbian.com/mirrors | jq -r '.'${CCODE}' | .[] | values'))
+	fi
+
 	# remove dead mirrors to suppress download errors
-	FILE=".control"
 	while read -r line
 	do
 		REMOVE=$(echo $line | egrep -o 'https?://[^ ]+/')
-        WEBSEED=( "${WEBSEED[@]/$REMOVE}" )
+		WEBSEED=( "${WEBSEED[@]/$REMOVE}" )
 	done < <(
 	for k in ${WEBSEED[@]}
 	do
-	echo "$k$FILE"
+	echo "$k$2/$3"
 	done | parallel --halt soon,fail=10 --jobs 32 wget -q --spider --timeout=15 --tries=4 --retry-connrefused {} 2>&1 >/dev/null)

 	# aria2 simply split chunks based on sources count not depending on download speed
@ -1647,8 +1652,9 @@ function webseed ()
 		https://mirrors.bfsu.edu.cn/armbian-releases/
 		)
 	fi
+
 	for toolchain in ${WEBSEED[@]}; do
-		text="${text} ${toolchain}${1}"
+		text="${text} ${toolchain}"$2/"${3}"
 	done
 	text="${text:1}"
 	echo "${text}"
@ -1680,22 +1686,28 @@ download_and_verify()
 	fi

 	# switch to china mirror if US timeouts
-	timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
+	timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
 	if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then
 		display_alert "Timeout from $server" "retrying" "info"
 		server="https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/"

 		# switch to another china mirror if tuna timeouts
-		timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
+		timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
 		if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then
 			display_alert "Timeout from $server" "retrying" "info"
 			server="https://mirrors.bfsu.edu.cn/armbian-releases/"
 		fi
 	fi

+	# rootfs has its own infra
+	if [[ "${remotedir}" == "_rootfs" ]]; then
+		local server="https://cache.armbian.com/"
+		remotedir="rootfs"
+	fi

 	# check if file exists on remote server before running aria2 downloader
-	[[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename}` ]] && return
+	timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
+	[[ $? -ne 0 ]] && return

 	cd "${localdir}" || exit

@ -1703,13 +1715,13 @@ download_and_verify()
 	if [[ -f "${SRC}"/config/torrents/${filename}.asc ]]; then
 		local torrent="${SRC}"/config/torrents/${filename}.torrent
 		ln -sf "${SRC}/config/torrents/${filename}.asc" "${localdir}/${filename}.asc"
-	elif [[ ! `timeout 10 curl --head --fail --silent "${server}${remotedir}/${filename}.asc"` ]]; then
+	elif [[ ! `timeout 10 curl --location --head --fail --silent "${server}${remotedir}/${filename}.asc"` ]]; then
 		return
 	else
 		# download control file
 		local torrent=${server}$remotedir/${filename}.torrent
 		aria2c --download-result=hide --disable-ipv6=$DISABLE_IPV6 --summary-interval=0 --console-log-level=error --auto-file-renaming=false \
-		--continue=false --allow-overwrite=true --dir="${localdir}" ${server}${remotedir}/${filename}.asc $(webseed "$remotedir/${filename}.asc") -o "${filename}.asc"
+		--continue=false --allow-overwrite=true --dir="${localdir}" ${server}${remotedir}/${filename}.asc $(webseed "${server}" "${remotedir}" "${filename}.asc") -o "${filename}.asc"
 		[[ $? -ne 0 ]] && display_alert "Failed to download control file" "" "wrn"
 	fi

@ -1738,10 +1750,10 @@ download_and_verify()

 	# direct download if torrent fails
 	if [[ ! -f "${localdir}/${filename}.complete" ]]; then
-		if [[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null` ]]; then
+		if [[ ! `timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null` ]]; then
 			display_alert "downloading using http(s) network" "$filename"
 			aria2c --download-result=hide --rpc-save-upload-metadata=false --console-log-level=error \
-			--dht-file-path="${SRC}"/cache/.aria2/dht.dat --disable-ipv6=$DISABLE_IPV6 --summary-interval=0 --auto-file-renaming=false --dir="${localdir}" ${server}${remotedir}/${filename} $(webseed "${remotedir}/${filename}") -o "${filename}"
+			--dht-file-path="${SRC}"/cache/.aria2/dht.dat --disable-ipv6=$DISABLE_IPV6 --summary-interval=0 --auto-file-renaming=false --dir="${localdir}" ${server}${remotedir}/${filename} $(webseed "${server}" "${remotedir}" "${filename}") -o "${filename}"
 			# mark complete
 			[[ $? -eq 0 ]] && touch "${localdir}/${filename}.complete" && echo ""