From e22e453d2c163e17d2db6ae6046367e1b200da15 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Fri, 5 Nov 2021 14:32:07 +0000 Subject: [PATCH] Add an integration test and a bunch of framework to support tests on multiple build systems --- CMakeLists.txt | 11 +++- Makefile.in | 9 ++- doc/Scripts.md | 30 +++++++-- scripts/test_harness.sh | 50 +++++++++++++++ scripts/test_integration.sh | 11 ++++ scripts/test_integration_supernode.sh | 33 ++++++++++ scripts/test_units.sh | 21 +++++++ scripts/test_units_harness.sh | 40 ------------ tests/test_integration_supernode.sh.expected | 64 ++++++++++++++++++++ 9 files changed, 221 insertions(+), 48 deletions(-) create mode 100755 scripts/test_harness.sh create mode 100755 scripts/test_integration.sh create mode 100755 scripts/test_integration_supernode.sh create mode 100755 scripts/test_units.sh delete mode 100755 scripts/test_units_harness.sh create mode 100644 tests/test_integration_supernode.sh.expected diff --git a/CMakeLists.txt b/CMakeLists.txt index 45c3984..3a486a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -310,6 +310,15 @@ install(FILES ${PROJECT_BINARY_DIR}/doc/n2n.7.gz # TODO: # - Add the right dependancy so that the tests binaries get built first enable_testing() -add_test(tests ${PROJECT_SOURCE_DIR}/scripts/test_units_harness.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/tests) +add_test(NAME unit + COMMAND ${CMAKE_COMMAND} -E env + TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR} + ${PROJECT_SOURCE_DIR}/scripts/test_units.sh +) +add_test(NAME integration + COMMAND ${CMAKE_COMMAND} -E env + TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR} + ${PROJECT_SOURCE_DIR}/scripts/test_integration.sh +) endif(DEFINED UNIX) diff --git a/Makefile.in b/Makefile.in index ae38339..db245b7 100644 --- a/Makefile.in +++ b/Makefile.in @@ -187,11 +187,14 @@ $(N2N_LIB): $(N2N_OBJS) win32/n2n_win32.a: win32 -.PHONY: test test.units -test: test.units +.PHONY: test test.units test.integration +test: test.units test.integration test.units: tools - scripts/test_units_harness.sh + scripts/test_units.sh + +test.integration: $(APPS) + scripts/test_integration.sh .PHONY: lint lint.python lint.ccode lint.shell lint.yaml lint: lint.python lint.ccode lint.shell lint.yaml diff --git a/doc/Scripts.md b/doc/Scripts.md index 297ea24..7146934 100644 --- a/doc/Scripts.md +++ b/doc/Scripts.md @@ -51,10 +51,6 @@ This shell script is a wrapper for the `uncrustify` C code style checker which checks or applies a set of rules to the code. It is used during the automated lint checks. -### `test_harness.sh` - -This shell script is used to run automated tests during development. - ### `n2n-gateway.sh` A sample script to route all the host traffic towards a remote gateway, @@ -96,3 +92,29 @@ Manually test fetching and config: /etc/munin/plugins/n2n_supernode_pkts /etc/munin/plugins/n2n_supernode_pkts config ``` + +## Testing scripts + +### `test_harness.sh` + +This shell script is used to run automated tests during development. It is +run with the name of one or more other scripts, which are then run with their +output being sent to `*.out` files in the `tests` directory and compared with +the matching `*.expected` file in that same dir. + +### `scripts/test_units.sh` + +This runs all the unit tests via the `test_harness.sh`. Unit tests are those +that are testing a small and isolated bit of code. In this project, the unit +tests are the only ones that contribute towards the code coverage report. + +### `scripts/test_integration.sh` + +This runs all the integration tests via the `test_harness.sh`. These are +tests that interact with multiple features and the test is mainly concerned +about the interactions between them (eg, testing an API interface) + +### `scripts/test_integration_supernode.sh` + +This starts a supernode and runs an integration test on the Json API using +the `n2n-ctl` command. diff --git a/scripts/test_harness.sh b/scripts/test_harness.sh new file mode 100755 index 0000000..c6af90c --- /dev/null +++ b/scripts/test_harness.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# +# This expects to find the tests in the tools dir or scripts dir and the +# expected results in the tests dir. +# +# Run with the name(s) of the tests on the commandline + +# boilerplate so we can support whaky cmake dirs +[ -z "$TOPDIR" ] && TOPDIR="." +[ -z "$BINDIR" ] && BINDIR="." +export TOPDIR +export BINDIR + +if [ -d "$BINDIR/tools" ]; then + TOOLSDIR="$BINDIR/tools" +else + TOOLSDIR="$BINDIR" +fi + +TESTS=$* + +SCRIPTSDIR="$TOPDIR/scripts" +TESTDATA="$TOPDIR/tests" + +# Confirm we have all the tools and data +for i in $TESTS; do + if [ ! -e "$TOOLSDIR/$i" ] && [ ! -e "$SCRIPTSDIR/$i" ]; then + echo "Could not find test $i" + exit 1 + fi + if [ ! -e "$TESTDATA/$i.expected" ]; then + echo "Could not find testdata $TESTDATA/$i.expected" + exit 1 + fi +done + +# Actually run the tests +for i in $TESTS; do + if [ -e "$TOOLSDIR/$i" ]; then + TEST="$TOOLSDIR/$i" + elif [ -e "$SCRIPTSDIR/$i" ]; then + TEST="$SCRIPTSDIR/$i" + fi + + echo "$TEST >$TESTDATA/$i.out" + set -e + "$TEST" >"$TESTDATA/$i.out" + cmp "$TESTDATA/$i.expected" "$TESTDATA/$i.out" + set +e +done diff --git a/scripts/test_integration.sh b/scripts/test_integration.sh new file mode 100755 index 0000000..79355ac --- /dev/null +++ b/scripts/test_integration.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# +# Run all the integration tests via the test harness + +# boilerplate so we can support whaky cmake dirs +[ -z "$TOPDIR" ] && TOPDIR=. +[ -z "$BINDIR" ] && BINDIR=. +export TOPDIR +export BINDIR + +${TOPDIR}/scripts/test_harness.sh test_integration_supernode.sh diff --git a/scripts/test_integration_supernode.sh b/scripts/test_integration_supernode.sh new file mode 100755 index 0000000..57f44bc --- /dev/null +++ b/scripts/test_integration_supernode.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# Do some quick tests via the Json API against the supernode +# + +AUTH=n2n + +# boilerplate so we can support whaky cmake dirs +[ -z "$TOPDIR" ] && TOPDIR=. +[ -z "$BINDIR" ] && BINDIR=. + +docmd() { + echo "###" + "$@" + echo +} + +# start it running in the background +docmd ${BINDIR}/supernode -v + +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 communities +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 packetstats +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 edges --raw + +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 verbose +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 -k $AUTH --write verbose 1 + +# looks strange, but we are querying the state of the "stop" verb +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 stop + +# stop it +docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 -k $AUTH --write stop + diff --git a/scripts/test_units.sh b/scripts/test_units.sh new file mode 100755 index 0000000..b899576 --- /dev/null +++ b/scripts/test_units.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Run all the unit tests via the test harness + +# boilerplate so we can support whaky cmake dirs +[ -z "$TOPDIR" ] && TOPDIR=. +[ -z "$BINDIR" ] && BINDIR=. +export TOPDIR +export BINDIR + +TESTS=" + tests-auth + tests-compress + tests-elliptic + tests-hashing + tests-transform + tests-wire +" + +# shellcheck disable=SC2086 +${TOPDIR}/scripts/test_harness.sh $TESTS diff --git a/scripts/test_units_harness.sh b/scripts/test_units_harness.sh deleted file mode 100755 index 5193297..0000000 --- a/scripts/test_units_harness.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -# This expects to find the tests in the tools dir and the expected results -# in the tests dir. - -TESTS=" - tests-auth - tests-compress - tests-elliptic - tests-hashing - tests-transform - tests-wire -" - -TOOLSDIR=tools -TESTDATA=tests - -# Allow both dirs be overidden -[ -n "$1" ] && TOOLSDIR="$1" -[ -n "$2" ] && TESTDATA="$2" - -# Confirm we have all the tools and data -for i in $TESTS; do - if [ ! -e "$TOOLSDIR/$i" ]; then - echo "Could not find test $TOOLSDIR/$i" - exit 1 - fi - if [ ! -e "$TESTDATA/$i.expected" ]; then - echo "Could not find testdata $TESTDATA/$i.expected" - exit 1 - fi -done - -# Actually run the tests -set -e -for i in $TESTS; do - echo "$TOOLSDIR/$i >$TESTDATA/$i.out" - "$TOOLSDIR/$i" >"$TESTDATA/$i.out" - cmp "$TESTDATA/$i.expected" "$TESTDATA/$i.out" -done diff --git a/tests/test_integration_supernode.sh.expected b/tests/test_integration_supernode.sh.expected new file mode 100644 index 0000000..b1d2a78 --- /dev/null +++ b/tests/test_integration_supernode.sh.expected @@ -0,0 +1,64 @@ +### + +### +[ + { + "community": "-/-", + "ip4addr": "", + "is_federation": 1, + "purgeable": 0 + } +] + +### +[ + { + "tx_pkt": 0, + "type": "forward" + }, + { + "tx_pkt": 0, + "type": "broadcast" + }, + { + "nak": 0, + "rx_pkt": 0, + "type": "reg_super" + }, + { + "tx_pkt": 0, + "type": "errors" + } +] + +### +[] + +### +[ + { + "traceLevel": 3 + } +] + +### +[ + { + "traceLevel": 1 + } +] + +### +[ + { + "keep_running": 1 + } +] + +### +[ + { + "keep_running": 0 + } +] +