From 36d0bdcaff288db57b8a92b01af8f6aa299ccfa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Pe=C4=8Dovnik?= Date: Sun, 19 Dec 2021 01:02:16 +0100 Subject: [PATCH] Varius updates to CI scripts (#3359) * RFC u-boot CI to more modern way --- .github/workflows/build-beta-desktop.yml | 9 +- .github/workflows/build-beta-images.yml | 2 +- .github/workflows/build-beta-kernel.yml | 2 +- .github/workflows/build-cache.yml | 2 +- .github/workflows/build-docker.yml | 53 ++++++- .github/workflows/build-test-image-docker.yml | 51 ++---- .github/workflows/build-u-boot.yml | 145 ++++++------------ .../lint-scripts-on-merge-request.yml | 4 +- .github/workflows/maintain.yml | 6 + .github/workflows/update-repository.yml | 2 +- 10 files changed, 122 insertions(+), 154 deletions(-) diff --git a/.github/workflows/build-beta-desktop.yml b/.github/workflows/build-beta-desktop.yml index afd8ad54cd..edb053a55c 100644 --- a/.github/workflows/build-beta-desktop.yml +++ b/.github/workflows/build-beta-desktop.yml @@ -1,9 +1,9 @@ -name: Build Beta Desktop +name: Beta Desktop on: workflow_dispatch: workflow_run: - workflows: ["Build Beta Kernel"] + workflows: ["Beta Kernel"] types: [completed] jobs: @@ -76,7 +76,7 @@ jobs: - name: Pull Docker image run: | - sudo docker pull ghcr.io/armbian/build:$(cat build/VERSION | sed 's/trunk.*/trunk/') + sudo docker pull ghcr.io/armbian/build:$(cat build/VERSION | sed 's/trunk.*/trunk/')"-$(dpkg --print-architecture)" - name: Build package run: | @@ -89,7 +89,8 @@ jobs: DE=$(echo $CHUNK | cut -d":" -f2) echo "RELEASE=${RELEASE}" >> $GITHUB_ENV echo "DE=${DE}" >> $GITHUB_ENV - sed -i "s/-it --rm/-i --rm/" userpatches/config-docker.conf + sed -i "s/-it --rm/-i --rm/" userpatches/config-docker.conf + ./compile.sh docker \ BSP_BUILD="yes" \ BETA="yes" \ diff --git a/.github/workflows/build-beta-images.yml b/.github/workflows/build-beta-images.yml index 7d97d6c913..e805f5dd33 100644 --- a/.github/workflows/build-beta-images.yml +++ b/.github/workflows/build-beta-images.yml @@ -33,7 +33,7 @@ jobs: with: variant: 'cli:beta' - runner: "ubuntu-latest" + runner: "small" part: 1 of: 1 include: '' diff --git a/.github/workflows/build-beta-kernel.yml b/.github/workflows/build-beta-kernel.yml index 4c0c450102..d421fe65b9 100644 --- a/.github/workflows/build-beta-kernel.yml +++ b/.github/workflows/build-beta-kernel.yml @@ -1,4 +1,4 @@ -name: Build Beta Kernel +name: Beta Kernel on: schedule: diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index 94fe6e77b8..eb1c8625f6 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -1,7 +1,7 @@ # This composite actions calls build rootfs action from git@github.com:armbian/scripts.git three times so we # can run all rootfs combinations at once - depending of available runners -name: Build rootfs caches +name: Rootfs Caches on: workflow_dispatch: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 98d8d8268a..4bbd6d1111 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -1,4 +1,4 @@ -name: Build Docker Image +name: Docker Image on: # Trigger the workflow on push but only for the main branch push: @@ -9,14 +9,21 @@ on: jobs: - build: + build-arm64: - name: Build Docker image - runs-on: ubuntu-latest + name: Build for ARM + runs-on: arm64 if: ${{ github.repository_owner == 'Armbian' }} steps: - - uses: actions/checkout@v1 + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: armbian/build + path: build + ref: master + clean: false - name: Login to GitHub Container Registry uses: docker/login-action@v1 @@ -28,10 +35,42 @@ jobs: - name: Build Docker image run: | + + cd build sed -i "s/-it --rm/-i --rm/" config/templates/config-docker.conf touch .ignore_changes ./compile.sh docker BOARD=virtual-qemu BRANCH=current RELEASE=focal BUILD_MINIMAL=yes BUILD_DESKTOP=no KERNEL_ONLY=no KERNEL_CONFIGURE=no COMPRESS_OUTPUTIMAGE=no REPOSITORY_INSTALL="u-boot,kernel,armbian-config,armbian-firmware" JUST_INIT=yes - docker tag armbian:$(cat VERSION) ghcr.io/armbian/build:$(cat VERSION) + docker tag armbian:$(cat VERSION) ghcr.io/armbian/build:$(cat VERSION)-arm64 - name: Push Docker image - run: docker push ghcr.io/armbian/build:$(cat VERSION) + run: docker push ghcr.io/armbian/build:$(cat VERSION)-arm64 + + + build-amd64: + + name: Build for X86 + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'Armbian' }} + steps: + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.CR_PAT }} + + - name: Build Docker image + + run: | + + sed -i "s/-it --rm/-i --rm/" config/templates/config-docker.conf + touch .ignore_changes + ./compile.sh docker BOARD=virtual-qemu BRANCH=current RELEASE=focal BUILD_MINIMAL=yes BUILD_DESKTOP=no KERNEL_ONLY=no KERNEL_CONFIGURE=no COMPRESS_OUTPUTIMAGE=no REPOSITORY_INSTALL="u-boot,kernel,armbian-config,armbian-firmware" JUST_INIT=yes + docker tag armbian:$(cat VERSION) ghcr.io/armbian/build:$(cat VERSION)-amd64 + + - name: Push Docker image + run: docker push ghcr.io/armbian/build:$(cat VERSION)-amd64 diff --git a/.github/workflows/build-test-image-docker.yml b/.github/workflows/build-test-image-docker.yml index bb982fbe9c..e59edc476b 100644 --- a/.github/workflows/build-test-image-docker.yml +++ b/.github/workflows/build-test-image-docker.yml @@ -1,9 +1,9 @@ -name: Build test with Docker +name: Build with Docker on: # Trigger after Docker image was built workflow_run: - workflows: ["Build Docker image"] + workflows: ["Docker Image"] types: - completed @@ -22,6 +22,7 @@ jobs: board: [uefi-arm64,uefi-x86,virtual-qemu] release: [focal,buster,bullseye,hirsute,jammy] desktop: [xfce] + if: ${{ github.repository_owner == 'Armbian' && github.event.workflow_run.conclusion == 'success' }} || github.event_name == 'workflow_dispatch' name: Variant runs-on: ubuntu-latest @@ -35,14 +36,6 @@ jobs: path: build clean: true - - name: Checkout support scripts - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/scripts - path: scripts - clean: true - - name: Sync run: | @@ -52,36 +45,17 @@ jobs: - name: Pull Docker image run: | - sudo docker pull ghcr.io/armbian/build:$(cat build/VERSION) + sudo docker pull ghcr.io/armbian/build:$(cat build/VERSION | sed 's/trunk.*/trunk/')"-$(dpkg --print-architecture)" - name: Build test image run: | cd build - # we need to fix this once but fake toolchain will prevent downloading it each time - mkdir -p cache/toolchain/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu - touch cache/toolchain/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu/.download-complete - mkdir -p cache/toolchain/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabi - touch cache/toolchain/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabi/.download-complete - mkdir -p cache/toolchain/gcc-linaro-aarch64-none-elf-4.8-2013.11_linux - touch cache/toolchain/gcc-linaro-aarch64-none-elf-4.8-2013.11_linux/.download-complete - mkdir -p cache/toolchain/gcc-linaro-arm-linux-gnueabihf-4.8-2014.04_linux - touch cache/toolchain/gcc-linaro-arm-linux-gnueabihf-4.8-2014.04_linux/.download-complete - mkdir -p cache/toolchain/gcc-linaro-arm-none-eabi-4.8-2014.04_linux - touch cache/toolchain/gcc-linaro-arm-none-eabi-4.8-2014.04_linux/.download-complete - mkdir -p cache/toolchain/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu - touch cache/toolchain/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu/.download-complete - mkdir -p cache/toolchain/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf - touch cache/toolchain/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/.download-complete - mkdir -p cache/toolchain/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu - touch cache/toolchain/gcc-arm-9.2-2019.12-x86_64-aarch64-none-linux-gnu/.download-complete - mkdir -p cache/toolchain/gcc-arm-9.2-2019.12-x86_64-arm-none-linux-gnueabihf - touch cache/toolchain/gcc-arm-9.2-2019.12-x86_64-arm-none-linux-gnueabihf/.download-complete - export TERM=dumb sed -i "s/-it --rm/-i --rm/" userpatches/config-docker.conf sed -i "s/COMPRESS_OUTPUTIMAGE=.*/COMPRESS_OUTPUTIMAGE=\"no\"/" userpatches/lib.config + ./compile.sh docker \ BETA=yes \ EXPERT=yes \ @@ -96,12 +70,13 @@ jobs: IGNORE_UPDATES=yes \ DESKTOP_ENVIRONMENT_CONFIG_NAME="config_base" \ DESKTOP_ENVIRONMENT=${{ matrix.desktop }} \ + SKIP_EXTERNAL_TOOLCHAINS=yes \ DESKTOP_APPGROUPS_SELECTED="" -# - name: Upload artefacts -# uses: actions/upload-artifact@v2 -# with: -# name: virtual-qemu -# path: build/output/images/* -# if-no-files-found: ignore -# retention-days: 10 + - name: Upload artefacts + uses: actions/upload-artifact@v2 + with: + name: virtual-qemu + path: build/output/images/* + if-no-files-found: ignore + retention-days: 10 diff --git a/.github/workflows/build-u-boot.yml b/.github/workflows/build-u-boot.yml index ccee3db9ea..9afaf54e72 100644 --- a/.github/workflows/build-u-boot.yml +++ b/.github/workflows/build-u-boot.yml @@ -1,120 +1,67 @@ -name: Build Beta U-boot +name: Beta Bootloader on: workflow_dispatch: + workflow_run: + workflows: ["Beta Kernel"] + types: [completed] jobs: - Merge: - uses: armbian/scripts/.github/workflows/merge-from-branch.yml@master + legacy: + uses: armbian/scripts/.github/workflows/build-u-boot-with-docker.yml@master with: - branch: 'nightly' + + runner: "ubuntu-latest" + include: 'grep legacy | ' + exclude: '' + uploading: false secrets: + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} GPG_KEY2: ${{ secrets.GPG_KEY2 }} GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} - Prepare: - needs: [ Merge ] - runs-on: [self-hosted, Linux, small] - if: ${{ github.repository_owner == 'armbian' }} - outputs: - matrix: ${{steps.list_dirs.outputs.matrix}} - steps: + current: + uses: armbian/scripts/.github/workflows/build-u-boot-with-docker.yml@master - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/build - path: build - ref: nightly - clean: false + with: - - name: Prepare matrix - id: list_dirs - run: - echo ::set-output name=matrix::$(for x in $(cat build/config/target*.conf | grep -v "^$" | grep -v "^#" | awk '{print $1":"$2}' | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) + runner: "small" + include: 'grep current | ' + exclude: '' + uploading: false - U-boot: # short name because Github will expand with the matrix values + secrets: + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} - needs: [ Prepare ] - runs-on: [self-hosted, Linux, small] - if: ${{ github.repository_owner == 'armbian' }} - timeout-minutes: 480 - strategy: - fail-fast: false - matrix: - node: ${{fromJson(needs.Prepare.outputs.matrix)}} - steps: + edge: + uses: armbian/scripts/.github/workflows/build-u-boot-with-docker.yml@master - - name: Install SSH key for storage - uses: shimataro/ssh-key-action@v2 - with: - key: ${{ secrets.KEY_TORRENTS }} - known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }} - if_key_exists: replace + with: - - name: Fix permissions - run: | + runner: "small" + include: 'grep edge | ' + exclude: '' + uploading: false - # make sure no temporally dirs are mounted from previous runs - while : - do - sudo pkill compile.sh || true - sudo pkill arm-binfmt-P || true - sudo pkill aarch64-binfmt-P || true - sudo pkill pixz || true - sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true - [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break - echo "Mounted temp directories. Trying to unmount." - df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true - sleep 10 - done - [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true - [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true - - - - name: Checkout Armbian build script - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/build - path: build - ref: nightly - clean: false - - - name: Build - run: | - CHUNK="${{ matrix.node }}" - BOARD=$(echo $CHUNK | cut -d":" -f1) - echo "BOARD=$BOARD" >> $GITHUB_ENV - BRANCH=$(echo $CHUNK | cut -d":" -f2) - echo "BRANCH=$BRANCH" >> $GITHUB_ENV - cd build - [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes - sudo rm -rf output/debs/* output/debs-beta/* - if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then - sudo mkdir -p cache/toolchain cache/rootfs || true - ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - fi - ./compile.sh ARMBIAN_MIRROR="https://github.com/armbian/mirror/releases/download/" BOARD="$BOARD" PRIVATE_CCACHE="yes" BETA="yes" KERNEL_ONLY="yes" BRANCH="$BRANCH" KERNEL_CONFIGURE="no" OFFLINE="no" REPOSITORY_INSTALL="kernel,bsp,armbian-zsh,armbian-config,armbian-firmware" - - - name: upload artefacts - uses: actions/upload-artifact@v2 - with: - name: u-boot-${{ env.BOARD }}-${{ env.BRANCH }} - path: build/output/debs-beta/*u-boot* - if-no-files-found: ignore - - - name: Deploy to server - if: ${{ success() }} - run: | - while fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do sleep 1; done; - sudo apt-get -y -qq install lftp - sudo chown -R $USER:$USER build/output/debs-beta/ - lftp -u upload, -e "set net:timeout 10;set net:max-retries 16;mirror --Remove-source-files -R --no-empty-dirs --parallel=8 --no-perms $(pwd)/build/output/debs-beta/ debs-beta/ ;bye" sftp://users.armbian.com + secrets: + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} diff --git a/.github/workflows/lint-scripts-on-merge-request.yml b/.github/workflows/lint-scripts-on-merge-request.yml index ef3866502e..cdf3756451 100644 --- a/.github/workflows/lint-scripts-on-merge-request.yml +++ b/.github/workflows/lint-scripts-on-merge-request.yml @@ -1,4 +1,4 @@ -name: Lint BASH scripts +name: Lint scripts on: workflow_dispatch: @@ -34,4 +34,4 @@ jobs: name: Shellcheck path: "*.diff" if-no-files-found: ignore - retention-days: 14 \ No newline at end of file + retention-days: 14 diff --git a/.github/workflows/maintain.yml b/.github/workflows/maintain.yml index f7a45c0c85..eb84154f2a 100644 --- a/.github/workflows/maintain.yml +++ b/.github/workflows/maintain.yml @@ -75,6 +75,12 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get -y autoremove [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true + [[ -d cache ]] && sudo rm -rf cache || true + sudo rm -rf build-* || true + sudo rm -rf temp || true + sudo rm *.conf || true + sudo rm *.txt || true + cd build [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes sudo rm -rf cache/sources diff --git a/.github/workflows/update-repository.yml b/.github/workflows/update-repository.yml index 17fd296ab7..1277d87290 100644 --- a/.github/workflows/update-repository.yml +++ b/.github/workflows/update-repository.yml @@ -3,7 +3,7 @@ name: Update Repository on: workflow_dispatch: workflow_run: - workflows: ["Build Beta Desktop"] + workflows: ["Beta Kernel" , "Beta Desktop" , "Beta Bootloader"] types: [completed] jobs: