diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml index e540f9ad9e..df3fc29dce 100644 --- a/.github/workflows/update-cache.yml +++ b/.github/workflows/update-cache.yml @@ -1,17 +1,29 @@ name: Build Rootfs Cache # -# Generates rootfs cache and uploads it to https://github.com/armbian/mirror/releases/tag/_rootfs and https://dl.armbian.com/_rootfs/ +# Generates rootfs cache and uploads it to: +# +# https://github.com/armbian/mirror/releases/tag/rootfs +# https://cache.armbian.com/rootfs/ # on: schedule: - - cron: "30 0 1 * *" + - cron: "30 0 1 * *" # Runs at 00:30 UTC on the 1st of every month. + push: - paths: + branches: + - 'master' + paths: - 'config/distributions/**' - 'config/cli/**' - 'config/desktop/**' + workflow_dispatch: + inputs: + monthoffset: + description: Forced month offset + required: true + default: 0 jobs: @@ -20,8 +32,20 @@ jobs: contents: none name: Release start runs-on: [X64] + outputs: + rootfscache_version: ${{ steps.env-vars.outputs.rootfscache_version }} if: ${{ github.repository_owner == 'Armbian' }} steps: + + # Synyching procedure expects this value + - run: | + echo "not empty" > changes + + - uses: actions/upload-artifact@v3 + with: + path: changes + name: changes + if-no-files-found: ignore - name: Checkout repository uses: actions/checkout@v3 @@ -30,11 +54,48 @@ jobs: path: build clean: false - - name: Updating releases - uses: ncipollo/release-action@v1 + - name: Remove current status + run: | + sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true + sudo apt-get -y -qq install sshfs + sudo mkdir -p build/cache/rootfs.upload || true + + # locally mount via NFS + if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then + sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload + else + sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true + fi + # remove true in sshfs when all runners are on jammy + + sudo rm build/cache/rootfs.upload/* || true + sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true + + - id: env-vars + name: Read current version + run: | + + ROOTFSCACHE_VERSION=$(wget --tries=10 -O - -o /dev/null https://github.com/armbian/mirror/releases/download/rootfs/rootfscache.version || true) + ROOTFSCACHE_VERSION=$(( ${ROOTFSCACHE_VERSION:-"0"} + 1 )) + echo "${ROOTFSCACHE_VERSION}" + echo "$ROOTFSCACHE_VERSION" | sudo tee rootfscache.version + echo ::set-output name=rootfscache_version::$(echo ${ROOTFSCACHE_VERSION}) || true + + - uses: dev-drprasad/delete-tag-and-release@v0.2.0 + if: ${{ github.repository_owner == 'Armbian' }} with: + delete_release: true + repo: 'armbian/mirror' + tag_name: "rootfs" + env: + GITHUB_TOKEN: ${{ secrets.CR_PAT }} + + - name: Updating releases + uses: ncipollo/release-action@v1.10.0 + with: + artifacts: "rootfscache.version" repo: "mirror" - tag: "_rootfs" + tag: "rootfs" name: "Build in progress" bodyFile: "build/.github/Releases-wip.md" allowUpdates: true @@ -50,6 +111,8 @@ jobs: with: variant: 'minimal:uefi-x86' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -70,6 +133,8 @@ jobs: with: variant: 'server:uefi-x86' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -90,6 +155,8 @@ jobs: with: variant: 'desktop:uefi-x86' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -110,6 +177,8 @@ jobs: with: variant: 'minimal:tinkerboard' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -131,6 +200,8 @@ jobs: with: variant: 'server:tinkerboard' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -150,6 +221,8 @@ jobs: with: variant: 'desktop:tinkerboard' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -170,6 +243,8 @@ jobs: with: variant: 'minimal:uefi-arm64' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -190,6 +265,8 @@ jobs: with: variant: 'server:uefi-arm64' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -209,6 +286,8 @@ jobs: with: variant: 'desktop:uefi-arm64' + monthoffset: ${{ inputs.monthoffset }} + rootfscache_version: ${{ needs.release-start.outputs.rootfscache_version }} secrets: PAT1: ${{ secrets.CR_PAT }} @@ -223,7 +302,7 @@ jobs: jobsend: permissions: contents: none - name: finish + name: Release finish needs: [x86-min,x86-srv,x86-x,armhf-min,armhf-srv,armhf-x,arm64-min,arm64-srv,arm64-x] runs-on: [fast] if: ${{ github.repository_owner == 'Armbian' }} @@ -283,9 +362,10 @@ jobs: sudo apt-get -y -qq install parallel buildtorrent ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | head -1)ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1) - TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-) - WEBSEEDS="--webseeds=https://github.com/armbian/mirror/releases/download/_rootfs/$FILE,https://imola.armbian.com/dl/_rootfs/$FILE,https://stpete-mirror.armbian.com/dl/_rootfs/$FILE" + TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-) + WEBSEEDS="--webseeds="https://github.com/armbian/mirror/releases/download/rootfs/\$FILE,"$(curl -s https://cache.armbian.com/mirrors | jq -r '.'default' | .[] | values' | sed -e 's/$/rootfs\/$FILE/' | tr '\n' , | sed 's/.$//')" cd build/cache/rootfs.upload + FILES=$(ls -1 *.lz4) for FILE in ${FILES[@]} do @@ -297,21 +377,48 @@ jobs: echo "Generating {}.torrent "; buildtorrent -q -s -m '$WEBSEEDS' --announce="'$ANNOUNCE'" --announcelist="'$TRACKERS'" {} -c "Armbian rootfs cache" {}.torrent >/dev/null ' + # Update version + echo "${{ needs.release-start.outputs.rootfscache_version }}" | sudo tee rootfscache.version + - name: Upload uses: ncipollo/release-action@v1 if: ${{ github.repository_owner == 'Armbian' }} with: repo: "mirror" - artifacts: "build/cache/rootfs/*.torrent,build/cache/rootfs/*.asc" - tag: "_rootfs" + artifacts: "build/cache/rootfs.upload/*.torrent,build/cache/rootfs.upload/*.asc" + tag: "rootfs" bodyFile: "build/.github/Releases.md" name: "Rootfs cache" allowUpdates: true - token: ${{ secrets.PAT }} + token: ${{ secrets.CR_PAT }} + + - name: Delete obsolete + run: | + + BRISI=($(diff <(find build/cache/rootfs.upload -name "*.lz4.current" | sed "s/.current//" | sort) <(find build/cache/rootfs.upload -name "*.lz4" | sort) | grep ">" | sed "s/> //")) + for brisi in "${BRISI[@]}"; do + sudo rm $brisi.* + done - name: Unmount folders - run: | sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true + + final-sync: + permissions: + contents: none + name: "Sync servers" + needs: [jobsend] + if: ${{ success() && github.repository_owner == 'Armbian' }} + uses: armbian/scripts/.github/workflows/sync-servers.yml@master + + with: + KEY_ID: 'upload' + + secrets: + KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }} + USER_REPOSITORY: ${{ secrets.USER_REPOSITORY }} + HOST_REPOSITORY: ${{ secrets.HOST_REPOSITORY }} + KNOWN_HOSTS_REPOSITORY: ${{ secrets.KNOWN_HOSTS_REPOSITORY }} diff --git a/config/desktop/focal/appgroups/desktop_tools/packages b/config/desktop/focal/appgroups/desktop_tools/packages index 3a5c657bd1..485ac5917f 100644 --- a/config/desktop/focal/appgroups/desktop_tools/packages +++ b/config/desktop/focal/appgroups/desktop_tools/packages @@ -1,4 +1,3 @@ bleachbit fbi gparted -system-monitoring-center diff --git a/config/desktop/focal/environments/gnome/config_base/packages b/config/desktop/focal/environments/gnome/config_base/packages index b9df40a7f9..8062fbaa28 100644 --- a/config/desktop/focal/environments/gnome/config_base/packages +++ b/config/desktop/focal/environments/gnome/config_base/packages @@ -200,6 +200,7 @@ python3-distupgrade python3-update-manager python3-xkit python3-yaml +qalculate-gtk slick-greeter software-properties-gtk spice-vdagent diff --git a/config/desktop/jammy/appgroups/desktop_tools b/config/desktop/jammy/appgroups/desktop_tools deleted file mode 120000 index 9135d28584..0000000000 --- a/config/desktop/jammy/appgroups/desktop_tools +++ /dev/null @@ -1 +0,0 @@ -../../focal/appgroups/desktop_tools \ No newline at end of file diff --git a/config/desktop/jammy/appgroups/desktop_tools/packages b/config/desktop/jammy/appgroups/desktop_tools/packages new file mode 100644 index 0000000000..3a5c657bd1 --- /dev/null +++ b/config/desktop/jammy/appgroups/desktop_tools/packages @@ -0,0 +1,4 @@ +bleachbit +fbi +gparted +system-monitoring-center diff --git a/config/desktop/jammy/environments/gnome/config_base/packages b/config/desktop/jammy/environments/gnome/config_base/packages index 5bdf0607b9..d472849828 100644 --- a/config/desktop/jammy/environments/gnome/config_base/packages +++ b/config/desktop/jammy/environments/gnome/config_base/packages @@ -120,6 +120,7 @@ python3-distupgrade python3-update-manager python3-xkit python3-yaml +qalculate-gtk slick-greeter software-properties-gtk spice-vdagent diff --git a/config/desktop/jammy/environments/xfce/config_base/packages b/config/desktop/jammy/environments/xfce/config_base/packages index ed7eae94a3..a17b789359 100644 --- a/config/desktop/jammy/environments/xfce/config_base/packages +++ b/config/desktop/jammy/environments/xfce/config_base/packages @@ -114,6 +114,7 @@ policykit-desktop-privileges printer-driver-all profile-sync-daemon pulseaudio-module-bluetooth +qalculate-gtk redshift slick-greeter smbclient diff --git a/config/desktop/sid/environments/budgie/config_base/packages b/config/desktop/sid/environments/budgie/config_base/packages index daaaac0def..98ebdbd8e0 100644 --- a/config/desktop/sid/environments/budgie/config_base/packages +++ b/config/desktop/sid/environments/budgie/config_base/packages @@ -122,6 +122,7 @@ policykit-1-gnome printer-driver-all profile-sync-daemon pulseaudio-module-bluetooth +qalculate-gtk redshift samba slick-greeter diff --git a/config/desktop/sid/environments/gnome/config_base/packages b/config/desktop/sid/environments/gnome/config_base/packages index 9cbec10b7c..9002b0e403 100644 --- a/config/desktop/sid/environments/gnome/config_base/packages +++ b/config/desktop/sid/environments/gnome/config_base/packages @@ -106,6 +106,7 @@ python3-debian python3-distutils python3-distutils-extra python3-yaml +qalculate-gtk software-properties-gtk spice-vdagent system-config-printer diff --git a/config/desktop/sid/environments/xfce/config_base/packages b/config/desktop/sid/environments/xfce/config_base/packages index 85773808eb..0e9622ed0f 100644 --- a/config/desktop/sid/environments/xfce/config_base/packages +++ b/config/desktop/sid/environments/xfce/config_base/packages @@ -32,6 +32,7 @@ printer-driver-all profile-sync-daemon pulseaudio pulseaudio-module-bluetooth +qalculate-gtk samba slick-greeter smbclient diff --git a/lib/configuration.sh b/lib/configuration.sh index a400244c3f..a3fa1097b7 100644 --- a/lib/configuration.sh +++ b/lib/configuration.sh @@ -29,7 +29,13 @@ HOSTRELEASE=$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2) [[ -z $EXIT_PATCHING_ERROR ]] && EXIT_PATCHING_ERROR="" # exit patching if failed [[ -z $HOST ]] && HOST="$BOARD" # set hostname to the board cd "${SRC}" || exit -[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=20 + +# if variable not provided, check which is current version in the cache storage +if [[ -z "${ROOTFSCACHE_VERSION}" ]]; then + ROOTFSCACHE_VERSION=$(wget --tries=10 -O - -o /dev/null https://github.com/armbian/mirror/releases/download/rootfs/rootfscache.version || true) + ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-"0"} +fi + [[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7 BUILD_REPOSITORY_URL=$(improved_git remote get-url $(improved_git remote 2>/dev/null | grep origin) 2>/dev/null) BUILD_REPOSITORY_COMMIT=$(improved_git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null) diff --git a/lib/debootstrap.sh b/lib/debootstrap.sh index 4f92765ee3..5c66d2980d 100644 --- a/lib/debootstrap.sh +++ b/lib/debootstrap.sh @@ -128,8 +128,9 @@ create_rootfs_cache() # seek last cache, proceed to previous otherwise build it for ((n=0;n<${cycles};n++)); do - [[ -z ${FORCED_MONTH_OFFSET} ]] && FORCED_MONTH_OFFSET=${n} - local packages_hash=$(get_package_list_hash "$(date -d "$D +${FORCED_MONTH_OFFSET} month" +"%Y-%m-module$ROOTFSCACHE_VERSION" | sed 's/^0*//')") + FORCED_MONTH_OFFSET=${n} + + local packages_hash=$(get_package_list_hash "$(date -d "$D -${FORCED_MONTH_OFFSET} month" +"%Y-%m-module$ROOTFSCACHE_VERSION" | sed 's/^0*//')") local cache_type="cli" [[ ${BUILD_DESKTOP} == yes ]] && local cache_type="xfce-desktop" [[ -n ${DESKTOP_ENVIRONMENT} ]] && local cache_type="${DESKTOP_ENVIRONMENT}" @@ -149,7 +150,7 @@ create_rootfs_cache() display_alert "Checking local cache" "$display_name" "info" if [[ -f ${cache_fname} && -n "$ROOT_FS_CREATE_ONLY" ]]; then - touch $cache_fname.current + echo "$cache_fname" > $cache_fname.current display_alert "Checking cache integrity" "$display_name" "info" sudo lz4 -tqq ${cache_fname} [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Please restart!" @@ -164,6 +165,7 @@ create_rootfs_cache() else display_alert "searching on servers" download_and_verify "_rootfs" "$cache_name" + [[ -f ${cache_fname} ]] && break fi if [[ ! -f $cache_fname ]]; then @@ -176,7 +178,7 @@ create_rootfs_cache() # speed up checking if [[ -n "$ROOT_FS_CREATE_ONLY" ]]; then - touch $cache_fname.current + echo "$cache_fname" > $cache_fname.current umount --lazy "$SDCARD" rm -rf $SDCARD # remove exit trap @@ -393,7 +395,7 @@ create_rootfs_cache() fi # needed for backend to keep current only - touch $cache_fname.current + echo "$cache_fname" > $cache_fname.current fi diff --git a/lib/general.sh b/lib/general.sh index e6bf6824f0..b3025a8dfa 100644 --- a/lib/general.sh +++ b/lib/general.sh @@ -1619,21 +1619,26 @@ prepare_host() function webseed () { + # list of mirrors that host our files unset text - # Hardcoded to EU mirrors since local CCODE=$(curl -s redirect.armbian.com/geoip | jq '.continent.code' -r) - WEBSEED=($(curl -s https://redirect.armbian.com/mirrors | jq -r '.'${CCODE}' | .[] | values')) + + if [[ "$2" == "rootfs" ]]; then + WEBSEED=($(curl -s ${1}mirrors | jq -r '.'${CCODE}' | .[] | values')) + else + WEBSEED=($(curl -s https://redirect.armbian.com/mirrors | jq -r '.'${CCODE}' | .[] | values')) + fi + # remove dead mirrors to suppress download errors - FILE=".control" while read -r line do REMOVE=$(echo $line | egrep -o 'https?://[^ ]+/') - WEBSEED=( "${WEBSEED[@]/$REMOVE}" ) + WEBSEED=( "${WEBSEED[@]/$REMOVE}" ) done < <( for k in ${WEBSEED[@]} do - echo "$k$FILE" + echo "$k$2/$3" done | parallel --halt soon,fail=10 --jobs 32 wget -q --spider --timeout=15 --tries=4 --retry-connrefused {} 2>&1 >/dev/null) # aria2 simply split chunks based on sources count not depending on download speed @@ -1647,8 +1652,9 @@ function webseed () https://mirrors.bfsu.edu.cn/armbian-releases/ ) fi + for toolchain in ${WEBSEED[@]}; do - text="${text} ${toolchain}${1}" + text="${text} ${toolchain}"$2/"${3}" done text="${text:1}" echo "${text}" @@ -1680,22 +1686,28 @@ download_and_verify() fi # switch to china mirror if US timeouts - timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null + timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then display_alert "Timeout from $server" "retrying" "info" server="https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/" # switch to another china mirror if tuna timeouts - timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null + timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then display_alert "Timeout from $server" "retrying" "info" server="https://mirrors.bfsu.edu.cn/armbian-releases/" fi fi + # rootfs has its own infra + if [[ "${remotedir}" == "_rootfs" ]]; then + local server="https://cache.armbian.com/" + remotedir="rootfs" + fi # check if file exists on remote server before running aria2 downloader - [[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename}` ]] && return + timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null + [[ $? -ne 0 ]] && return cd "${localdir}" || exit @@ -1703,13 +1715,13 @@ download_and_verify() if [[ -f "${SRC}"/config/torrents/${filename}.asc ]]; then local torrent="${SRC}"/config/torrents/${filename}.torrent ln -sf "${SRC}/config/torrents/${filename}.asc" "${localdir}/${filename}.asc" - elif [[ ! `timeout 10 curl --head --fail --silent "${server}${remotedir}/${filename}.asc"` ]]; then + elif [[ ! `timeout 10 curl --location --head --fail --silent "${server}${remotedir}/${filename}.asc"` ]]; then return else # download control file local torrent=${server}$remotedir/${filename}.torrent aria2c --download-result=hide --disable-ipv6=$DISABLE_IPV6 --summary-interval=0 --console-log-level=error --auto-file-renaming=false \ - --continue=false --allow-overwrite=true --dir="${localdir}" ${server}${remotedir}/${filename}.asc $(webseed "$remotedir/${filename}.asc") -o "${filename}.asc" + --continue=false --allow-overwrite=true --dir="${localdir}" ${server}${remotedir}/${filename}.asc $(webseed "${server}" "${remotedir}" "${filename}.asc") -o "${filename}.asc" [[ $? -ne 0 ]] && display_alert "Failed to download control file" "" "wrn" fi @@ -1738,10 +1750,10 @@ download_and_verify() # direct download if torrent fails if [[ ! -f "${localdir}/${filename}.complete" ]]; then - if [[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null` ]]; then + if [[ ! `timeout 10 curl --location --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null` ]]; then display_alert "downloading using http(s) network" "$filename" aria2c --download-result=hide --rpc-save-upload-metadata=false --console-log-level=error \ - --dht-file-path="${SRC}"/cache/.aria2/dht.dat --disable-ipv6=$DISABLE_IPV6 --summary-interval=0 --auto-file-renaming=false --dir="${localdir}" ${server}${remotedir}/${filename} $(webseed "${remotedir}/${filename}") -o "${filename}" + --dht-file-path="${SRC}"/cache/.aria2/dht.dat --disable-ipv6=$DISABLE_IPV6 --summary-interval=0 --auto-file-renaming=false --dir="${localdir}" ${server}${remotedir}/${filename} $(webseed "${server}" "${remotedir}" "${filename}") -o "${filename}" # mark complete [[ $? -eq 0 ]] && touch "${localdir}/${filename}.complete" && echo ""