added JSON interfaces to edge management port and scripts to further process output (#854)

* Add management commands to show data in JSON format

* Add a script to query the JSON management interface

* Suprisingly, the github runner does not have flake8 installed

* Add n2nctl debugging output to show the raw data received from the JSON

* Ensure well known tag wrap-around semantics

* Try to ensure we check every edge case in the protocol handling - only valid packets are allowed

* Add a very simple http to management port gateway

* Fix the lint issue

.github/workflows/tests.yml

@@ -12,11 +12,15 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+      - name: Install essential
+        run: |
+          sudo apt-get install flake8
       - name: Run minimal test set
         run: |
           ./autogen.sh
           ./configure
           make test
+          make lint.python
 
   test_linux:
     needs: smoketest

Makefile.in

@@ -149,6 +149,9 @@ win32/n2n_win32.a: win32
 test: tools
 	tools/test_harness
 
+lint.python:
+	flake8 scripts/n2nctl scripts/n2nhttpd
+
 # To generate coverage information, run configure with
 # CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="--coverage"
 # and run the desired tests.  Ensure that package gcovr is installed
@@ -169,7 +172,7 @@ gcov:
 # It is a convinent target to use during development or from a CI/CD system
 build-dep:
 ifeq ($(CONFIG_TARGET),generic)
-	sudo apt install build-essential autoconf libcap-dev libzstd-dev gcovr
+	sudo apt install build-essential autoconf libcap-dev libzstd-dev gcovr flake8
 else ifeq ($(CONFIG_TARGET),darwin)
 	brew install automake gcovr
 else

scripts/n2nctl

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Licensed under GPLv3
+#
+# Simple script to query the management interface of a running n2n edge node
+
+import argparse
+import socket
+import json
+import collections
+
+next_tag = 0
+
+
+def send_cmd(port, debug, cmd):
+    global next_tag
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+    tagstr = str(next_tag)
+    next_tag = (next_tag + 1) % 1000
+
+    message = "{} {}".format(cmd, tagstr).encode('utf8')
+    sock.sendto(message, ("127.0.0.1", 5644))
+
+    # FIXME:
+    # - there is no timeout for any of the socket handling
+
+    begin, _ = sock.recvfrom(1024)
+    begin = json.loads(begin.decode('utf8'))
+    assert(begin['_tag'] == tagstr)
+    assert(begin['_type'] == 'begin')
+    assert(begin['_cmd'] == cmd)
+
+    result = list()
+
+    while True:
+        data, _ = sock.recvfrom(1024)
+        data = json.loads(data.decode('utf8'))
+        assert(data['_tag'] == tagstr)
+
+        if data['_type'] == 'unknowncmd':
+            raise ValueError('Unknown command {}'.format(cmd))
+
+        if data['_type'] == 'end':
+            return result
+
+        if data['_type'] != 'row':
+            raise ValueError('Unknown data type {} from '
+                             'edge'.format(data['_type']))
+
+        # remove our boring metadata
+        del data['_tag']
+        del data['_type']
+
+        if debug:
+            print(data)
+
+        result.append(data)
+
+
+def str_table(rows, columns):
+    """Given an array of dicts, do a simple table print"""
+    result = list()
+    widths = collections.defaultdict(lambda: 0)
+    for row in rows:
+        for col in columns:
+            if col in row:
+                widths[col] = max(widths[col], len(str(row[col])))
+
+    for col in columns:
+        if widths[col] == 0:
+            widths[col] = 1
+        result += "{:{}.{}} ".format(col, widths[col], widths[col])
+    result += "\n"
+
+    for row in rows:
+        for col in columns:
+            if col in row:
+                data = row[col]
+            else:
+                data = ''
+            result += "{:{}} ".format(data, widths[col])
+        result += "\n"
+
+    return ''.join(result)
+
+
+def subcmd_show_supernodes(args):
+    rows = send_cmd(args.port, args.debug, 'j.super')
+    columns = [
+        'version',
+        'current',
+        'macaddr',
+        'sockaddr',
+        'uptime',
+    ]
+
+    return str_table(rows, columns)
+
+
+def subcmd_show_peers(args):
+    rows = send_cmd(args.port, args.debug, 'j.peer')
+    columns = [
+        'mode',
+        'ip4addr',
+        'macaddr',
+        'sockaddr',
+        'desc',
+    ]
+
+    return str_table(rows, columns)
+
+
+subcmds = {
+    'supernodes': {
+        'func': subcmd_show_supernodes,
+        'help': 'Show the list of supernodes',
+    },
+    'peers': {
+        'func': subcmd_show_peers,
+        'help': 'Show the list of peers',
+    },
+}
+
+
+def main():
+    ap = argparse.ArgumentParser(
+            description='Query the running local n2n edge')
+    ap.add_argument('-t', '--port', action='store', default=5644,
+                    help='Management Port (default=5644)')
+    ap.add_argument('-d', '--debug', action='store_true',
+                    help='Also show raw internal data')
+
+    subcmd = ap.add_subparsers(help='Subcommand', dest='cmd')
+    subcmd.required = True
+
+    for key, value in subcmds.items():
+        value['parser'] = subcmd.add_parser(key, help=value['help'])
+        value['parser'].set_defaults(func=value['func'])
+
+    args = ap.parse_args()
+
+    result = args.func(args)
+    print(result)
+
+
+if __name__ == '__main__':
+    main()

scripts/n2nhttpd

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Licensed under GPLv3
+#
+# Simple http server to allow user control of n2n edge nodes
+#
+# Currently only for demonstration - needs javascript written to render the
+# results properly.
+#
+# Try it out with
+#   http://localhost:8080/edge/peer
+#   http://localhost:8080/edge/super
+
+import argparse
+import socket
+import json
+import socketserver
+import http.server
+
+from http import HTTPStatus
+
+next_tag = 0
+
+
+def send_cmd(port, debug, cmd):
+    """Send a text command to the edge and process the JSON reply packets"""
+    global next_tag
+
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+    tagstr = str(next_tag)
+    next_tag = (next_tag + 1) % 1000
+
+    message = "{} {}".format(cmd, tagstr).encode('utf8')
+    sock.sendto(message, ("127.0.0.1", 5644))
+
+    # FIXME:
+    # - there is no timeout for any of the socket handling
+
+    begin, _ = sock.recvfrom(1024)
+    begin = json.loads(begin.decode('utf8'))
+    assert(begin['_tag'] == tagstr)
+    assert(begin['_type'] == 'begin')
+    assert(begin['_cmd'] == cmd)
+
+    result = list()
+
+    while True:
+        data, _ = sock.recvfrom(1024)
+        data = json.loads(data.decode('utf8'))
+        assert(data['_tag'] == tagstr)
+
+        if data['_type'] == 'unknowncmd':
+            raise ValueError('Unknown command {}'.format(cmd))
+
+        if data['_type'] == 'end':
+            return result
+
+        if data['_type'] != 'row':
+            raise ValueError('Unknown data type {} from '
+                             'edge'.format(data['_type']))
+
+        # remove our boring metadata
+        del data['_tag']
+        del data['_type']
+
+        if debug:
+            print(data)
+
+        result.append(data)
+
+
+class SimpleHandler(http.server.BaseHTTPRequestHandler):
+
+    def log_request(self, code='-', size='-'):
+        # Dont spam the output
+        pass
+
+    def do_GET(self):
+        url_tail = self.path
+        if url_tail.startswith("/edge/"):
+            tail = url_tail.split('/')
+            cmd = 'j.' + tail[2]
+
+            try:
+                data = send_cmd(5644, False, cmd)
+            except ValueError:
+                self.send_response(HTTPStatus.BAD_REQUEST)
+                self.end_headers()
+                self.wfile.write(b'Bad Command')
+                return
+
+            self.send_response(HTTPStatus.OK)
+            self.send_header('Content-type', 'application/json')
+            self.end_headers()
+            self.wfile.write(json.dumps(data).encode('utf8'))
+
+
+def main():
+    ap = argparse.ArgumentParser(
+            description='Control the running local n2n edge via http')
+    #  TODO - this needs to pass into the handler object
+    # ap.add_argument('-t', '--mgmtport', action='store', default=5644,
+    #                 help='Management Port (default=5644)')
+    # ap.add_argument('-d', '--debug', action='store_true',
+    #                 help='Also show raw internal data')
+    ap.add_argument('port', action='store',
+                    default=8080, type=int, nargs='?',
+                    help='Serve requests on TCP port (default 8080)')
+
+    args = ap.parse_args()
+
+    with socketserver.TCPServer(("", args.port), SimpleHandler) as httpd:
+        httpd.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        httpd.serve_forever()
+
+
+if __name__ == '__main__':
+    main()

src/edge_utils.c

@@ -1806,6 +1806,160 @@ static char *get_ip_from_arp (dec_ip_str_t buf, const n2n_mac_t req_mac) {
 #endif
 #endif
 
+static void handleMgmtJson_super (n2n_edge_t *eee, char *tag, char *udp_buf, struct sockaddr_in sender_sock) {
+    size_t msg_len;
+    struct peer_info *peer, *tmpPeer;
+    macstr_t mac_buf;
+    n2n_sock_str_t sockbuf;
+    selection_criterion_str_t sel_buf;
+
+    traceEvent(TRACE_DEBUG, "mgmt j.super");
+
+    HASH_ITER(hh, eee->conf.supernodes, peer, tmpPeer) {
+
+        /*
+         * TODO:
+         * The version string provided by the remote supernode could contain
+         * chars that make our JSON invalid.
+         * - do we care?
+         */
+
+        msg_len = snprintf(udp_buf, N2N_PKT_BUF_SIZE,
+                            "{"
+                            "\"_tag\":\"%s\","
+                            "\"_type\":\"row\","
+                            "\"version\":\"%s\","
+                            "\"purgeable\":%i,"
+                            "\"current\":%i,"
+                            "\"macaddr\":\"%s\","
+                            "\"sockaddr\":\"%s\","
+                            "\"selection\":\"%s\","
+                            "\"lastseen\":%li,"
+                            "\"uptime\":%li}\n",
+                            tag,
+                            peer->version,
+                            peer->purgeable,
+                            (peer == eee->curr_sn) ? (eee->sn_wait ? 2 : 1 ) : 0,
+                            is_null_mac(peer->mac_addr) ? "" : macaddr_str(mac_buf, peer->mac_addr),
+                            sock_to_cstr(sockbuf, &(peer->sock)),
+                            sn_selection_criterion_str(sel_buf, peer),
+                            peer->last_seen,
+                            peer->uptime);
+
+        sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0,
+               (struct sockaddr *) &sender_sock, sizeof(struct sockaddr_in));
+    }
+}
+
+static void handleMgmtJson_peer (n2n_edge_t *eee, char *tag, char *udp_buf, struct sockaddr_in sender_sock) {
+    size_t msg_len;
+    struct peer_info *peer, *tmpPeer;
+    macstr_t mac_buf;
+    n2n_sock_str_t sockbuf;
+    in_addr_t net;
+
+    traceEvent(TRACE_DEBUG, "mgmt j.peer");
+
+    /* FIXME:
+     * dont repeat yourself - the body of these two loops is identical
+     */
+
+    // dump nodes with forwarding through supernodes
+    HASH_ITER(hh, eee->pending_peers, peer, tmpPeer) {
+        net = htonl(peer->dev_addr.net_addr);
+        msg_len = snprintf(udp_buf, N2N_PKT_BUF_SIZE,
+                            "{"
+                            "\"_tag\":\"%s\","
+                            "\"_type\":\"row\","
+                            "\"mode\":\"pSp\","
+                            "\"ip4addr\":\"%s\","
+                            "\"macaddr\":\"%s\","
+                            "\"sockaddr\":\"%s\","
+                            "\"desc\":\"%s\","
+                            "\"lastseen\":%li}\n",
+                            tag,
+                            (peer->dev_addr.net_addr == 0) ? "" : inet_ntoa(*(struct in_addr *) &net),
+                            (is_null_mac(peer->mac_addr)) ? "" : macaddr_str(mac_buf, peer->mac_addr),
+                            sock_to_cstr(sockbuf, &(peer->sock)),
+                            peer->dev_desc,
+                            peer->last_seen);
+
+        sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0/*flags*/,
+               (struct sockaddr *) &sender_sock, sizeof(struct sockaddr_in));
+    }
+
+    // dump peer-to-peer nodes
+    HASH_ITER(hh, eee->known_peers, peer, tmpPeer) {
+        net = htonl(peer->dev_addr.net_addr);
+        msg_len = snprintf(udp_buf, N2N_PKT_BUF_SIZE,
+                            "{"
+                            "\"_tag\":\"%s\","
+                            "\"_type\":\"row\","
+                            "\"mode\":\"p2p\","
+                            "\"ip4addr\":\"%s\","
+                            "\"macaddr\":\"%s\","
+                            "\"sockaddr\":\"%s\","
+                            "\"desc\":\"%s\","
+                            "\"lastseen\":%li}\n",
+                            tag,
+                            (peer->dev_addr.net_addr == 0) ? "" : inet_ntoa(*(struct in_addr *) &net),
+                            (is_null_mac(peer->mac_addr)) ? "" : macaddr_str(mac_buf, peer->mac_addr),
+                            sock_to_cstr(sockbuf, &(peer->sock)),
+                            peer->dev_desc,
+                            peer->last_seen);
+        sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0/*flags*/,
+               (struct sockaddr *) &sender_sock, sizeof(struct sockaddr_in));
+    }
+}
+
+static void handleMgmtJson (n2n_edge_t *eee, char *cmdp, char *udp_buf, struct sockaddr_in sender_sock) {
+    size_t msg_len;
+
+    char cmd[10];
+    char tag[10];
+
+    /* save the command name before we reuse the buffer */
+    strncpy(cmd, cmdp, sizeof(cmd)-1);
+    cmd[sizeof(cmd)-1] = 0;
+
+    /* Extract the tag to use in all reply packets */
+    char *tagp = strtok(NULL, " \r\n");
+    if(tagp) {
+        strncpy(tag, tagp, sizeof(tag)-1);
+        tag[sizeof(tag)-1] = 0;
+    } else {
+        tag[0] = '0';
+        tag[1] = 0;
+    }
+
+    /*
+     * TODO:
+     * The tag provided by the requester could contain chars
+     * that make our JSON invalid.
+     * - do we care?
+     */
+    msg_len = snprintf(udp_buf, N2N_PKT_BUF_SIZE,
+                        "{\"_tag\":\"%s\",\"_type\":\"begin\",\"_cmd\":\"%s\"}\n", tag, cmd);
+    sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0,
+           (struct sockaddr *) &sender_sock, sizeof(struct sockaddr_in));
+
+    if(0 == strcmp(cmd, "j.super")) {
+        handleMgmtJson_super(eee, tag, udp_buf, sender_sock);
+    } else if(0 == strcmp(cmd, "j.peer")) {
+        handleMgmtJson_peer(eee, tag, udp_buf, sender_sock);
+    } else {
+        msg_len = snprintf(udp_buf, N2N_PKT_BUF_SIZE,
+                            "{\"_tag\":\"%s\",\"_type\":\"unknowncmd\"}\n", tag);
+        sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0,
+               (struct sockaddr *) &sender_sock, sizeof(struct sockaddr_in));
+    }
+
+    msg_len = snprintf(udp_buf, N2N_PKT_BUF_SIZE,
+                        "{\"_tag\":\"%s\",\"_type\":\"end\"}\n", tag);
+    sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0,
+           (struct sockaddr *) &sender_sock, sizeof(struct sockaddr_in));
+    return;
+}
 
 /** Read a datagram from the management UDP socket and take appropriate
  *    action. */
@@ -1842,6 +1996,9 @@ static void readFromMgmtSocket (n2n_edge_t *eee, int *keep_running) {
         return; /* failed to receive data from UDP */
     }
 
+    /* avoid parsing any uninitialized junk from the stack */
+    udp_buf[recvlen] = 0;
+
     if((0 == memcmp(udp_buf, "help", 4)) || (0 == memcmp(udp_buf, "?", 1))) {
         msg_len = 0;
 
@@ -1851,6 +2008,8 @@ static void readFromMgmtSocket (n2n_edge_t *eee, int *keep_running) {
                             "\thelp    | This help message\n"
                             "\t+verb   | Increase verbosity of logging\n"
                             "\t-verb   | Decrease verbosity of logging\n"
+                            "\tj.super | JSON supernode info\n"
+                            "\tj.peer  | JSON peer info\n"
                             "\t<enter> | Display statistics\n\n");
 
         sendto(eee->udp_mgmt_sock, udp_buf, msg_len, 0/*flags*/,
@@ -1898,6 +2057,13 @@ static void readFromMgmtSocket (n2n_edge_t *eee, int *keep_running) {
         return;
     }
 
+    char * cmdp = strtok( (char *)udp_buf, " \r\n");
+    if(cmdp && (0 == memcmp(cmdp, "j.", 2))) {
+        /* We think this is a JSON request */
+        handleMgmtJson(eee, cmdp, (char *)udp_buf, sender_sock);
+        return;
+    }
+
     traceEvent(TRACE_DEBUG, "mgmt status requested");
 
     msg_len = 0;