Add an integration test and a bunch of framework to support tests on multiple build systems

CMakeLists.txt

@@ -310,6 +310,15 @@ install(FILES ${PROJECT_BINARY_DIR}/doc/n2n.7.gz
 # TODO:
 # - Add the right dependancy so that the tests binaries get built first
 enable_testing()
-add_test(tests ${PROJECT_SOURCE_DIR}/scripts/test_units_harness.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/tests)
+add_test(NAME unit
+    COMMAND ${CMAKE_COMMAND} -E env
+        TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR}
+        ${PROJECT_SOURCE_DIR}/scripts/test_units.sh
+)
+add_test(NAME integration
+    COMMAND ${CMAKE_COMMAND} -E env
+        TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR}
+        ${PROJECT_SOURCE_DIR}/scripts/test_integration.sh
+)
 
 endif(DEFINED UNIX)

Makefile.in

@@ -187,11 +187,14 @@ $(N2N_LIB): $(N2N_OBJS)
 
 win32/n2n_win32.a: win32
 
-.PHONY: test test.units
+.PHONY: test test.units test.integration
-test: test.units
+test: test.units test.integration
 
 test.units: tools
-	scripts/test_units_harness.sh
+	scripts/test_units.sh
+
+test.integration: $(APPS)
+	scripts/test_integration.sh
 
 .PHONY: lint lint.python lint.ccode lint.shell lint.yaml
 lint: lint.python lint.ccode lint.shell lint.yaml

doc/Scripts.md

@@ -51,10 +51,6 @@ This shell script is a wrapper for the `uncrustify` C code style checker
 which checks or applies a set of rules to the code.  It is used during
 the automated lint checks.
 
-### `test_harness.sh`
-
-This shell script is used to run automated tests during development.
-
 ### `n2n-gateway.sh`
 
 A sample script to route all the host traffic towards a remote gateway,
@@ -96,3 +92,29 @@ Manually test fetching and config:
 /etc/munin/plugins/n2n_supernode_pkts
 /etc/munin/plugins/n2n_supernode_pkts config
 ```
+
+## Testing scripts
+
+### `test_harness.sh`
+
+This shell script is used to run automated tests during development.  It is
+run with the name of one or more other scripts, which are then run with their
+output being sent to `*.out` files in the `tests` directory and compared with
+the matching `*.expected` file in that same dir.
+
+### `scripts/test_units.sh`
+
+This runs all the unit tests via the `test_harness.sh`.  Unit tests are those
+that are testing a small and isolated bit of code.  In this project, the unit
+tests are the only ones that contribute towards the code coverage report.
+
+### `scripts/test_integration.sh`
+
+This runs all the integration tests via the `test_harness.sh`.  These are
+tests that interact with multiple features and the test is mainly concerned
+about the interactions between them (eg, testing an API interface)
+
+### `scripts/test_integration_supernode.sh`
+
+This starts a supernode and runs an integration test on the Json API using
+the `n2n-ctl` command.

scripts/test_harness.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+# This expects to find the tests in the tools dir or scripts dir and the
+# expected results in the tests dir.
+#
+# Run with the name(s) of the tests on the commandline
+
+# boilerplate so we can support whaky cmake dirs
+[ -z "$TOPDIR" ] && TOPDIR="."
+[ -z "$BINDIR" ] && BINDIR="."
+export TOPDIR
+export BINDIR
+
+if [ -d "$BINDIR/tools" ]; then
+    TOOLSDIR="$BINDIR/tools"
+else
+    TOOLSDIR="$BINDIR"
+fi
+
+TESTS=$*
+
+SCRIPTSDIR="$TOPDIR/scripts"
+TESTDATA="$TOPDIR/tests"
+
+# Confirm we have all the tools and data
+for i in $TESTS; do
+    if [ ! -e "$TOOLSDIR/$i" ] && [ ! -e "$SCRIPTSDIR/$i" ]; then
+        echo "Could not find test $i"
+        exit 1
+    fi
+    if [ ! -e "$TESTDATA/$i.expected" ]; then
+        echo "Could not find testdata $TESTDATA/$i.expected"
+        exit 1
+    fi
+done
+
+# Actually run the tests
+for i in $TESTS; do
+    if [ -e "$TOOLSDIR/$i" ]; then
+        TEST="$TOOLSDIR/$i"
+    elif [ -e "$SCRIPTSDIR/$i" ]; then
+        TEST="$SCRIPTSDIR/$i"
+    fi
+
+    echo "$TEST >$TESTDATA/$i.out"
+    set -e
+    "$TEST" >"$TESTDATA/$i.out"
+    cmp "$TESTDATA/$i.expected" "$TESTDATA/$i.out"
+    set +e
+done

scripts/test_integration.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+#
+# Run all the integration tests via the test harness
+
+# boilerplate so we can support whaky cmake dirs
+[ -z "$TOPDIR" ] && TOPDIR=.
+[ -z "$BINDIR" ] && BINDIR=.
+export TOPDIR
+export BINDIR
+
+${TOPDIR}/scripts/test_harness.sh test_integration_supernode.sh

scripts/test_integration_supernode.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Do some quick tests via the Json API against the supernode
+#
+
+AUTH=n2n
+
+# boilerplate so we can support whaky cmake dirs
+[ -z "$TOPDIR" ] && TOPDIR=.
+[ -z "$BINDIR" ] && BINDIR=.
+
+docmd() {
+    echo "###"
+    "$@"
+    echo
+}
+
+# start it running in the background
+docmd ${BINDIR}/supernode -v
+
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 communities
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 packetstats
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 edges --raw
+
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 verbose
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 -k $AUTH --write verbose 1
+
+# looks strange, but we are querying the state of the "stop" verb
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 stop
+
+# stop it
+docmd ${TOPDIR}/scripts/n2n-ctl -t 5645 -k $AUTH --write stop
+

scripts/test_units.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Run all the unit tests via the test harness
+
+# boilerplate so we can support whaky cmake dirs
+[ -z "$TOPDIR" ] && TOPDIR=.
+[ -z "$BINDIR" ] && BINDIR=.
+export TOPDIR
+export BINDIR
+
+TESTS="
+    tests-auth
+    tests-compress
+    tests-elliptic
+    tests-hashing
+    tests-transform
+    tests-wire
+"
+
+# shellcheck disable=SC2086
+${TOPDIR}/scripts/test_harness.sh $TESTS

scripts/test_units_harness.sh

@@ -1,40 +0,0 @@
-#!/bin/sh
-#
-# This expects to find the tests in the tools dir and the expected results
-# in the tests dir.
-
-TESTS="
-    tests-auth
-    tests-compress
-    tests-elliptic
-    tests-hashing
-    tests-transform
-    tests-wire
-"
-
-TOOLSDIR=tools
-TESTDATA=tests
-
-# Allow both dirs be overidden
-[ -n "$1" ] && TOOLSDIR="$1"
-[ -n "$2" ] && TESTDATA="$2"
-
-# Confirm we have all the tools and data
-for i in $TESTS; do
-    if [ ! -e "$TOOLSDIR/$i" ]; then
-        echo "Could not find test $TOOLSDIR/$i"
-        exit 1
-    fi
-    if [ ! -e "$TESTDATA/$i.expected" ]; then
-        echo "Could not find testdata $TESTDATA/$i.expected"
-        exit 1
-    fi
-done
-
-# Actually run the tests
-set -e
-for i in $TESTS; do
-    echo "$TOOLSDIR/$i >$TESTDATA/$i.out"
-    "$TOOLSDIR/$i" >"$TESTDATA/$i.out"
-    cmp "$TESTDATA/$i.expected" "$TESTDATA/$i.out"
-done

tests/test_integration_supernode.sh.expected

@@ -0,0 +1,64 @@
+###
+
+###
+[
+    {
+        "community": "-/-",
+        "ip4addr": "",
+        "is_federation": 1,
+        "purgeable": 0
+    }
+]
+
+###
+[
+    {
+        "tx_pkt": 0,
+        "type": "forward"
+    },
+    {
+        "tx_pkt": 0,
+        "type": "broadcast"
+    },
+    {
+        "nak": 0,
+        "rx_pkt": 0,
+        "type": "reg_super"
+    },
+    {
+        "tx_pkt": 0,
+        "type": "errors"
+    }
+]
+
+###
+[]
+
+###
+[
+    {
+        "traceLevel": 3
+    }
+]
+
+###
+[
+    {
+        "traceLevel": 1
+    }
+]
+
+###
+[
+    {
+        "keep_running": 1
+    }
+]
+
+###
+[
+    {
+        "keep_running": 0
+    }
+]
+