From a1facf0f3f5166343f3c4ba2ad4af58142b2ced2 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 18 Nov 2021 21:28:50 +0000 Subject: [PATCH] Update Openwrt process (#900) * First attempt at a openwrt CI * Fix action - helps if I dont forget the syntax half way through writing it * Try building /this/ branch for the openwrt CI * Try a build with openssl configured * Dont enable the n2n package until we have built the environment - makes errors easier to see and clearly related to n2n * Attempt to speed up the openwrt build * Upload any created ipkg packages * Dont test with openssl for the moment * Attempt to speed up openwrt build using a cache of the build dir * The make defconfig run turns all the built binaries stale, so stop caching them. Also use a real ref for the cache key * Minor text name changes * Address yamllint concerns * Attempt to simplify and document missing parts of the openwrt makefile * Attempt to fix mystery openwrt make error * Rename build job name * Avoid nested checkouts, use two separate dirs for the two checkouts in this build * Move the n2n checkout to earler, allowing us to skip one defconfig run * We are going to need working tags from the n2n repo, so ensure we unbreak the github checkout braindamage * Calculate and save the n2n version string * Prepare the way to pass the correct external vars into the openwrt build * Hook calculated build variables into the openwrt package definition * Update artifacts source to match moved checkout dir * Pass env vars in to the make * Allow version script to be influenced by external vars * It will help if I use the same variable names everywhere * Add more version variable calculation overrides * Configure openwrt to use the external git checkout instead of their create-tar-then-extract dance * Using the correct syntax for ifdef will help significantly * Use as many jobs as we have cpus * As the USE_SOURCE_DIR option allows us to use a full git checkout, we do not need to hack the version.sh to allow overrides * Ensure scripts/version.sh works from anywhere * Remove unneeded variables * Update openwrt build documentation to match the new build process * Catch failure to cd as per shellcheck suggestion * Limit lengthy openwrt builds to manual triggers or on a release * Also run on specially named branches * Break list into separate lines for easier future editing --- .github/workflows/openwrt.yml | 112 ++++++++++++++++++++++++++++++++++ packages/openwrt/Makefile | 30 ++++++--- packages/openwrt/README.md | 22 ++++++- scripts/version.sh | 2 + 4 files changed, 156 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/openwrt.yml diff --git a/.github/workflows/openwrt.yml b/.github/workflows/openwrt.yml new file mode 100644 index 0000000..7ed488a --- /dev/null +++ b/.github/workflows/openwrt.yml @@ -0,0 +1,112 @@ +--- +name: Openwrt Build + +# yamllint disable-line rule:truthy +on: + release: + types: + - published + - created + - edited + push: + branches: + - openwrt + + workflow_dispatch: + +jobs: + build: + name: Build ipkg + runs-on: ubuntu-latest + + defaults: + run: + working-directory: openwrt + + steps: + - name: Checkout openwrt + uses: actions/checkout@v2 + with: + path: openwrt + repository: openwrt/openwrt + + - name: Set openwrt ref + id: openwrt_ref + run: | + echo ::set-output name=REF::$(git rev-parse --short HEAD) + git rev-parse --short HEAD + + - name: Checkout n2n + uses: actions/checkout@v2 + with: + path: n2n + fetch-depth: 0 + + - name: Fix Checkout + run: | + git fetch --force --tags + working-directory: n2n + + - name: Set n2n ref + id: n2n_ref + run: | + echo ::set-output name=REF::$(./scripts/version.sh) + ./scripts/version.sh + working-directory: n2n + + - name: Copy n2n package definition into openwrt + run: | + cp -r n2n/packages/openwrt openwrt/package/n2n + working-directory: ./ + + - name: Cache openwrt source downloads + uses: actions/cache@v2 + with: + path: openwrt/dl + key: openwrt-dl-${{ steps.openwrt_ref.outputs.REF }} + + - name: Setup openwrt config and environment + run: | + echo "CONFIG_TARGET_x86=y" >.config + echo "CONFIG_TARGET_x86_64=y" >>.config + + - name: Add n2n package to openwrt config + run: | + echo "CONFIG_PACKAGE_n2n-edge=m" >>.config + echo "CONFIG_PACKAGE_n2n-supernode=m" >>.config + + - name: Build a full config from our stub file + run: | + make defconfig + + - name: Download openwrt sources + run: | + make download + + - name: Build openwrt build environment + run: | + make -j `nproc` tools/install toolchain/install + + - name: Build n2n dependancies + run: | + make -j `nproc` package/libs/libpcap/compile + + - name: Build n2n openwrt packages + env: + N2N_PKG_VERSION: ${{ steps.n2n_ref.outputs.REF }} + run: | + echo "Build for $N2N_PKG_VERSION" + export N2N_PKG_VERSION + make package/n2n/clean V=s + make package/n2n/prepare USE_SOURCE_DIR=$GITHUB_WORKSPACE/n2n V=s + make package/n2n/compile V=s + +# FIXME: add a way to run the test suite! +# - name: Run embedded tests +# run: make test + + - name: Upload built artifacts + uses: actions/upload-artifact@v2 + with: + name: built-ipkgs + path: openwrt/bin/packages/*/base/*.ipk diff --git a/packages/openwrt/Makefile b/packages/openwrt/Makefile index 719a4fa..ce40d1c 100644 --- a/packages/openwrt/Makefile +++ b/packages/openwrt/Makefile @@ -5,18 +5,31 @@ include $(TOPDIR)/rules.mk PKG_NAME:=n2n -PKG_SOURCE_URL:=https://github.com/ntop/n2n.git -PKG_SOURCE_VERSION:=6937640a2bc24832af7fc4ed1658d6aef192f03b -PKG_VERSION:=2.9.0_dev_git$(PKG_SOURCE_VERSION) +PKG_VERSION:=HEAD PKG_RELEASE:=1 -PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz -PKG_SOURCE_SUBDIR:=$(PKG_NAME)-$(PKG_VERSION) PKG_SOURCE_PROTO:=git +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz + +# These are defaults for compiling without any environmental overrides +# (eg, the github action calculates the correct overrides for each build) +PKG_SOURCE_URL:=https://github.com/ntop/n2n +PKG_SOURCE_VERSION:=dev +PKG_MIRROR_HASH:=skip + +# Apply overrides from the build environment +ifdef N2N_PKG_SOURCE_URL + PKG_SOURCE_URL:=$(N2N_PKG_SOURCE_URL) +endif +ifdef N2N_PKG_SOURCE_VERSION + PKG_SOURCE_VERSION:=$(N2N_PKG_SOURCE_VERSION) +endif +ifdef N2N_PKG_VERSION + PKG_VERSION:=$(N2N_PKG_VERSION) +endif PKG_MAINTAINER:=Emanuele Faranda PKG_LICENSE:=GPL3 -PKG_BUILD_PARALLEL:=1 # autogen fix PKG_FIXUP:=autoreconf @@ -52,8 +65,9 @@ The supernode for the N2N infrastructure endef define Build/Configure - ( cd $(PKG_BUILD_DIR); ./autogen.sh ) - $(call Build/Configure/Default) + ( cd $(PKG_BUILD_DIR); \ + ./autogen.sh; \ + ./configure ) endef define Package/n2n-edge/conffiles diff --git a/packages/openwrt/README.md b/packages/openwrt/README.md index 44b229f..bd446e1 100644 --- a/packages/openwrt/README.md +++ b/packages/openwrt/README.md @@ -17,12 +17,27 @@ comes down to the following steps: ## Compilation -From the OpenWRT build directory: +These instructions are for building the current checked out version of the +n2n source (The generally used OpenWRT alternative is to download a tar.gz +file of a specific n2n version, but that is not as suitable for development +or local builds) + +You need both the openwrt repository and the n2n repository checked out +for this. In these instructions, we assume that `openwrt` is the directory +where your openwrt checkout is located and `n2n` is the directory for +the n2n repository. ``` git clone https://github.com/ntop/n2n n2n -cp -r n2n/packages/openwrt package/n2n +N2N_PKG_VERSION=$(n2n/scripts/version.sh) +export N2N_PKG_VERSION + +cp -r n2n/packages/openwrt openwrt/package/n2n + +cd openwrt make menuconfig # select Network -> VPN -> n2n-edge and n2n-supernode +make package/n2n/clean V=s +make package/n2n/prepare USE_SOURCE_DIR=../n2n V=s make package/n2n/compile V=s ``` @@ -30,6 +45,9 @@ If everything went fine, two ipk will be generated, one for the n2n-edge and the other for n2n-supernode. They can be found with `find . -name "n2n*.ipk"`, copied to the target device, and installed with `opkg install`. +The github action described in `.github/workflows/openwrt.yml` implements +an automated version of the above steps. + ## Configuration The edge node can be started with `/etc/init.d/edge start`. diff --git a/scripts/version.sh b/scripts/version.sh index 7c7704c..e551500 100755 --- a/scripts/version.sh +++ b/scripts/version.sh @@ -20,6 +20,8 @@ if [ -d "$TOPDIR/.git" ]; then # If there is a .git directory in our TOPDIR, then this is assumed to be # real git checkout + cd "$TOPDIR" || exit 1 + VER_GIT_SHORT=$(git describe --abbrev=0) if [ "$VER_FILE_SHORT" != "$VER_GIT_SHORT" ]; then