Github Actions - improve beta build process (#3353)

.github/workflows/build-beta-desktop.yml

@@ -17,6 +17,8 @@ jobs:
       matrix: ${{steps.list_dirs.outputs.matrix}}
     steps:
 
+      - uses: ahmadnassri/action-workflow-run-wait@v1
+
       - name: Checkout repository
         uses: actions/checkout@v2
         with:

.github/workflows/build-beta-images.yml

@@ -1,295 +1,72 @@
-name: Build Beta Images
+name: Beta images
 
 on:
   workflow_dispatch:
-#    inputs:
-#      build_chunks:
-#        description: 'Build chunks'
-#        required: false
-#        default: '22'
-#  workflow_run:
-#    workflows: ["Build beta kernel packages"]
-#    types:
-#      - completed
+
 jobs:
 
-  Merge:
+  x86:
+    uses: armbian/scripts/.github/workflows/build-with-docker.yml@master
 
-    name: "Merge master into nightly"
-    runs-on: [self-hosted, Linux]
-    if: ${{ github.repository_owner == 'Armbian' }}
-
-    steps:
-
-      - name: Checkout Armbian build script
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          repository: armbian/build
-          path: build
-          ref: nightly
-          clean: false
-
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v3
-        with:
-          gpg-private-key: ${{ secrets.GPG_KEY2 }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE2 }}
-          workdir: build
-          git-user-signingkey: true
-          git-commit-gpgsign: true
-
-      - name: Merge master into nightly
-
-        run: |
-          cd build
-          git config --global user.email "info@armbian.com"
-          git config --global user.name "Armbianworker"
-          git checkout master
-          git fetch
-          git merge origin/master
-          git checkout nightly
-          git merge master nightly
-          git push
-
-  Prepare:
-
-    needs: [ Merge ]
-    runs-on: [self-hosted, Linux, images]
-    if: ${{ github.repository_owner == 'armbian' }}
-    outputs:
-      matrix: ${{steps.list_dirs.outputs.matrix}}
-    steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          repository: armbian/build
-          path: build
-          ref: nightly
-          clean: false
-
-      - name: Install SSH key for storage
-
-        uses: shimataro/ssh-key-action@v2
-        with:
-          key: ${{ secrets.KEY_TORRENTS }}
-          known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
-          if_key_exists: replace
-
-      - name: Clean upload folder
-
-        run: |
-          
-          cd build
-          while sudo mountpoint output/images -q
-          do
-              sudo umount output/images || true
-              echo "Trying to unmount ..."
-              sleep 10
-          done
-          sudo apt-get -y -qq install sshfs
-          sudo mkdir -p /root/.ssh/
-          sudo cp ~/.ssh/known_hosts /root/.ssh/
-          sudo rm -rf output/images/*
-          sudo mkdir -p output/images
-          sudo sshfs upload@users.armbian.com:/images output/images -o IdentityFile=~/.ssh/id_rsa
-          sudo find output/images/ -mindepth 1 -type d -exec sudo rm -rf {} \; 2>/dev/null || true
-          while sudo mountpoint output/images -q
-          do
-              sudo umount output/images || true
-              echo "Trying to unmount ..."
-              sleep 10
-          done
-
-      - name: Cut the job into chunks
-        run: |
-
-          mkdir -p temp
-          # clean leftovers
-          sudo rm -rf temp/*
-          cat build/config/targets-cli-beta.conf build/config/targets-desktop-beta.conf | grep -v "^$" | grep -v "^#" | shuf > temp/split.conf
-          PARTS=$(cat temp/split.conf | wc -l)
-          split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 temp/split.conf temp/split-
-          echo $CHUNKS
-
-      - name: Prepare matrix
-        id: list_dirs
-        run:
-          PARTS=$(cat temp/split.conf | wc -l)
-          echo ::set-output name=matrix::$(for x in $(seq -w 001 $(cat temp/split.conf | wc -l)); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)          
-
-      - name: Cache build configurations
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-build
-        with:
-          path: temp
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-
-  Job: # short name because Github will expand with the matrix values
-
-    needs: [ Prepare ]
-    runs-on: [self-hosted, Linux, images]
-    if: ${{ github.repository_owner == 'armbian' }}
-    timeout-minutes: 480
-    strategy:
-      fail-fast: false
-      matrix: 
-        node: ${{fromJson(needs.Prepare.outputs.matrix)}}
-
-    steps:
+    with:
  
-      - name: Cache Gradle packages
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-build
-        with:
-          path: temp
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
+      variant: 'cli:beta'
+      runner: "ubuntu-latest"
+      part: 1
+      of: 1
+      include: 'grep uefi-x86 | '
+      exclude: ''
+      uploading: false
 
-      - name: Fix permissions
-        run: |
+    secrets:
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
 
-          # make sure no temporally dirs are mounted from previous runs
-          while :
-          do
-              sudo pkill compile.sh || true
-              sudo pkill arm-binfmt-P || true
-              sudo pkill aarch64-binfmt-P || true
-              sudo pkill pixz || true
-              if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
-                  sudo mountpoint -q build/cache/rootfs &&  sudo fusermount -u build/cache/rootfs || true
-                  sudo mountpoint -q build/cache/toolchain &&  sudo fusermount -u build/cache/toolchain || true
-              fi
-              sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true
-              [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break
-              echo "Mounted temp directories. Trying to unmount."
-              df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true
-              sleep 10
-          done
-          [[ -d build/output/debug ]] && rm -rf build/output/debug/* || true
-          [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true
-          [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true
+  cli:
+    uses: armbian/scripts/.github/workflows/build-with-docker.yml@master
 
-      - name: Checkout Armbian build script
+    with:
+ 
+      variant: 'cli:beta'
+      runner: "ubuntu-latest"
+      part: 1
+      of: 1
+      include: ''
+      exclude: 'grep -v uefi-x86 | '
+      uploading: false
 
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          repository: armbian/build
-          path: build
-          ref: nightly
-          clean: false
+    secrets:
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
 
-      - name: Checkout Armbian support scripts
 
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          repository: armbian/scripts
-          token: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
-          path: scripts
-          clean: false
+  desktop:
+    uses: armbian/scripts/.github/workflows/build-with-docker.yml@master
 
-      - name: Import GPG key
+    with:
+      variant: 'desktop:beta'
+      runner: "big"
+      part: 1
+      of: 1      
+      include: ''
+      exclude: 'grep -v uefi-x86 | '
+      uploading: false
 
-        uses: crazy-max/ghaction-import-gpg@v3
-        with:
-          gpg-private-key: ${{ secrets.GPG_KEY1 }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE1 }}
-          workdir: scripts
-          git-user-signingkey: true
-          git-commit-gpgsign: true
-
-      - name: Install SSH key for storage
-
-        uses: shimataro/ssh-key-action@v2
-        with:
-          key: ${{ secrets.KEY_TORRENTS }}
-          known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
-          if_key_exists: replace
-
-      - name: Build beta images
-        env:
-          GPG_PASS: ${{ secrets.GPG_PASSPHRASE1 }}
-
-        run: |
-          CHUNK="${{ matrix.node }}"
-          CHUNK=$(echo $CHUNK | sed 's/^0*//') 
-          CHUNK=$(printf "%03d" $CHUNK)
-          cd build
-          # use prepared configs
-          sudo mkdir -p userpatches
-          sudo cp ../scripts/configs/* userpatches/
-          # prepare host
-          [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes
-          sudo rm -rf cache/sources
-          [[ ! -d cache/toolchain ]] && ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host'
-          sudo rm -rf cache/sources
-          if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
-              sudo rm -rf cache/toolchain/*
-              sudo mkdir -p cache/toolchain build/cache/rootfs || true
-              ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
-              ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
-          fi
-
-          # calculate how many images we can build in parallel
-          # PARALLEL_BUILDS=$(awk '{printf("%d",$1/5000)}' <<<$(($(LC_ALL=C free -w 2>/dev/null | grep "^Mem" | awk '{print $2}' || LC_ALL=C free | grep "^Mem"| awk '{print $2}')/1024)))
-          # disable parallel builds since we have lots of runners
-          PARALLEL_BUILDS=0
-          # mount deploy target
-          sudo apt-get -y -qq install sshfs
-          sudo mkdir -p /root/.ssh/
-          sudo cp ~/.ssh/known_hosts /root/.ssh/
-          sudo mkdir -p output/images || true
-          sudo sshfs upload@users.armbian.com:/images output/images -o IdentityFile=~/.ssh/id_rsa
-          # link build targets
-          sudo ln -sf ../../temp/split-${CHUNK}.conf userpatches/targets.conf
-          # we enforce family skip only for kernel building
-          sudo rm -f userpatches/family.skip
-          ./compile.sh all-new-beta-images MULTITHREAD="${PARALLEL_BUILDS}" GPG_PASS="${GPG_PASS}" ARMBIAN_CACHE_TOOLCHAIN_PATH="${ARMBIAN_CACHE_TOOLCHAIN_PATH}" ARMBIAN_CACHE_ROOTFS_PATH="${ARMBIAN_CACHE_ROOTFS_PATH}"
-          # umount
-          while mountpoint output/images -q
-          do
-              sudo umount output/images || true
-              echo "Trying to unmount ..."
-              sleep 10
-          done
-          cp -R output/debug/Armbian ../temp
-
-  Deploy:
-
-    name: Update download infrastructure
-    needs: [Job]
-    runs-on: [self-hosted, Linux, local]
-    if: ${{ github.repository_owner == 'Armbian' }}
-    steps:
-      - name: Cache Gradle packages
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-build
-        with:
-          path: temp
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-
-      - name: upload artefacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: logs
-          path: temp/Armbian/
-
-      - name: Install SSH key for torrent
-        uses: shimataro/ssh-key-action@v2
-        with:
-          key: ${{ secrets.KEY_TORRENTS }}
-          name: id_torrent # optional
-          known_hosts: ${{ secrets.KNOWN_HOSTS_TORRENTS }}
-          if_key_exists: replace
-      - name: Create torrents
-        run: ssh -T -i ~/.ssh/id_torrent ${{ secrets.USER_TORRENTS }}@${{ secrets.HOST_TORRENTS }}
+    secrets:
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}