Attempt to simplify adding new tests

CMakeLists.txt

@@ -313,12 +313,12 @@ enable_testing()
 add_test(NAME unit
     COMMAND ${CMAKE_COMMAND} -E env
         TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR}
-        ${PROJECT_SOURCE_DIR}/scripts/test_units.sh
+        ${PROJECT_SOURCE_DIR}/scripts/test_harness.sh ${PROJECT_SOURCE_DIR}/tests/tests_units.list
 )
 add_test(NAME integration
     COMMAND ${CMAKE_COMMAND} -E env
         TOPDIR=${PROJECT_SOURCE_DIR} BINDIR=${PROJECT_BINARY_DIR}
-        ${PROJECT_SOURCE_DIR}/scripts/test_integration.sh
+        ${PROJECT_SOURCE_DIR}/scripts/test_harness.sh ${PROJECT_SOURCE_DIR}/tests/tests_integration.list
 )
 
 endif(DEFINED UNIX)

Makefile.in

@@ -205,10 +205,10 @@ win32/n2n_win32.a: win32
 test: test.units test.integration
 
 test.units: tools
-	scripts/test_units.sh
+	scripts/test_harness.sh tests/tests_units.list
 
 test.integration: $(APPS)
-	scripts/test_integration.sh
+	scripts/test_harness.sh tests/tests_integration.list
 
 .PHONY: lint lint.python lint.ccode lint.shell lint.yaml
 lint: lint.python lint.ccode lint.shell lint.yaml

doc/Scripts.md

@@ -98,21 +98,18 @@ Manually test fetching and config:
 ### `test_harness.sh`
 
 This shell script is used to run automated tests during development.  It is
-run with the name of one or more other scripts, which are then run with their
-output being sent to `*.out` files in the `tests` directory and compared with
-the matching `*.expected` file in that same dir.
+run with a testlist filename - pointing at a file containing the list of
+tests to run.
 
-### `scripts/test_units.sh`
+Each test needs a file containing the expected output `${TESTNAME}.expected`
+which is expected to exist in the same directory as the testlist (this dir is
+referred to as `${listdir}` below).
 
-This runs all the unit tests via the `test_harness.sh`.  Unit tests are those
-that are testing a small and isolated bit of code.  In this project, the unit
-tests are the only ones that contribute towards the code coverage report.
+Each test is a program, searched for in several locations, including the
+`${listdir}/../scripts` dir.
 
-### `scripts/test_integration.sh`
-
-This runs all the integration tests via the `test_harness.sh`.  These are
-tests that interact with multiple features and the test is mainly concerned
-about the interactions between them (eg, testing an API interface)
+Each test is run with its output being sent to `*.out` files in the `listdir`
+and compared with the expected output.
 
 ### `scripts/test_integration_supernode.sh`
 

scripts/test_harness.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
 #
+# Run with the name of a test list file.
+#
 # This expects to find the tests in the tools dir or scripts dir and the
 # expected results in the tests dir.
-#
-# Run with the name(s) of the tests on the commandline
 
 # boilerplate so we can support whaky cmake dirs
 [ -z "$TOPDIR" ] && TOPDIR="."
@@ -11,40 +11,37 @@
 export TOPDIR
 export BINDIR
 
-if [ -d "$BINDIR/tools" ]; then
-    TOOLSDIR="$BINDIR/tools"
-else
-    TOOLSDIR="$BINDIR"
+if [ -z "$1" ]; then
+    echo need test list filename
+    exit 1
 fi
+TESTLIST="$1"
+LISTDIR=$(dirname "$TESTLIST")
 
-TESTS=$*
-
-SCRIPTSDIR="$TOPDIR/scripts"
-TESTDATA="$TOPDIR/tests"
-
-# Confirm we have all the tools and data
-for i in $TESTS; do
-    if [ ! -e "$TOOLSDIR/$i" ] && [ ! -e "$SCRIPTSDIR/$i" ]; then
-        echo "Could not find test $i"
-        exit 1
-    fi
-    if [ ! -e "$TESTDATA/$i.expected" ]; then
-        echo "Could not find testdata $TESTDATA/$i.expected"
-        exit 1
-    fi
-done
+TESTS=$(sed -e "s/#.*//" "$TESTLIST")
 
 # Actually run the tests
 for i in $TESTS; do
-    if [ -e "$TOOLSDIR/$i" ]; then
-        TEST="$TOOLSDIR/$i"
-    elif [ -e "$SCRIPTSDIR/$i" ]; then
-        TEST="$SCRIPTSDIR/$i"
+    # Look in several places for the test program
+    if [ -e "$BINDIR/$i" ]; then
+        TEST="$BINDIR/$i"
+    elif [ -e "$BINDIR/tools/$i" ]; then
+        TEST="$BINDIR/tools/$i"
+    elif [ -e "$LISTDIR/../scripts/$i" ]; then
+        TEST="$LISTDIR/../scripts/$i"
+    else
+        echo "Could not find test $i"
+        exit 1
     fi
 
-    echo "$TEST >$TESTDATA/$i.out"
+    if [ ! -e "$LISTDIR/$i.expected" ]; then
+        echo "Could not find testdata $LISTDIR/$i.expected"
+        exit 1
+    fi
+
+    echo "$TEST >$LISTDIR/$i.out"
     set -e
-    "$TEST" >"$TESTDATA/$i.out"
-    cmp "$TESTDATA/$i.expected" "$TESTDATA/$i.out"
+    "$TEST" >"$LISTDIR/$i.out"
+    cmp "$LISTDIR/$i.expected" "$LISTDIR/$i.out"
     set +e
 done

scripts/test_integration.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-#
-# Run all the integration tests via the test harness
-
-# boilerplate so we can support whaky cmake dirs
-[ -z "$TOPDIR" ] && TOPDIR=.
-[ -z "$BINDIR" ] && BINDIR=.
-export TOPDIR
-export BINDIR
-
-${TOPDIR}/scripts/test_harness.sh test_integration_supernode.sh

scripts/test_units.sh

@@ -1,21 +0,0 @@
-#!/bin/sh
-#
-# Run all the unit tests via the test harness
-
-# boilerplate so we can support whaky cmake dirs
-[ -z "$TOPDIR" ] && TOPDIR=.
-[ -z "$BINDIR" ] && BINDIR=.
-export TOPDIR
-export BINDIR
-
-TESTS="
-    tests-auth
-    tests-compress
-    tests-elliptic
-    tests-hashing
-    tests-transform
-    tests-wire
-"
-
-# shellcheck disable=SC2086
-${TOPDIR}/scripts/test_harness.sh $TESTS

tests/tests_integration.list

@@ -0,0 +1,4 @@
+#
+# The integration tests
+
+test_integration_supernode.sh

tests/tests_units.list

@@ -0,0 +1,9 @@
+#
+# The unit tests
+
+tests-auth
+tests-compress
+tests-elliptic
+tests-hashing
+tests-transform
+tests-wire