WIP: request and connection machine

Just need a simple request logging phase to wrap this up

Signed-off-by: Ava Affine <ava@sunnypup.io>

.clangd

@@ -0,0 +1,3 @@
+CompileFlags:
+  Compiler: gcc
+  Add: ["-xc", "-std=c2x"]

.gitignore

@@ -1,3 +1,5 @@
 libmuninn.so
 libmuninn.dylib
 nginx/.cache
+.cache
+mun_pfx

Makefile

@@ -10,23 +10,22 @@ ifneq ($(MUNINN_NGINX_DEBUG),)
 MUNINN_NGINX_DEBUG="--with-debug"
 endif
 
-CURDIR = $(.CURDIR)
+.PHONY: all clean
-
-.PHONY: all clean build
 
 all: nginx/objs/nginx
 
-nginx/objs/Makefile:
+nginx/objs/Makefile: config Makefile
+	make clean
 	cd $(NGINX_SOURCE_DIR) && \
 		./auto/configure \
 			$(MUNINN_NGINX_DEBUG) \
 			--with-http_ssl_module \
 			--with-compat \
-			--add-module=$(.CURDIR)
+			--add-module=$(shell pwd)
 
-nginx/objs/nginx: nginx/objs/Makefile
+nginx/objs/nginx: nginx/objs/Makefile muninn_core.c muninn_daemon.c muninn.h
 	$(MUNIN_BEAR) make -C $(NGINX_SOURCE_DIR)
 
 clean:
-	make -C $(NGINX_SOURCE_DIR) clean
+	-make -C $(NGINX_SOURCE_DIR) clean
 

README.md

@@ -22,13 +22,65 @@ The loose plan to implement Muninn is as follows:
 9. Muninn can identify as authoritatively owning a certain DN (and Peers abide).
 10. Something other than logging is done for conflicts over who owns what DN
 
-Currently Muninn is on phase 2.
+Currently Muninn is working on phase 3.
 
 ## Building Muninn
-simply run `make`.
+Muninn is implemented as a statically linked NGINX Module. The output of the
+provided build process is an NGINX binary that contains Muninn functionality. To
+build Muninn simply run `make`. The resulting binary will be in the build tree
+at `$(pwd)/nginx/objs/nginx`.
 
 ## Running Muninn
-TODO
+Muninn may be ran in any way which NGINX is currently run. See the configuration
+section for more details. To add Muninn to an existing running NGINX simply
+compile Muninn, copy the output binary over your NGINX executable, and follow
+the existing NGINX binary upgrade process.
 
 ## Configuring Muninn
-TODO
+Muninn uses an NGINX global configuration block similar to the existing NGINX
+HTTP module. To configure Muninn begin a standard NGINX configuration and open
+a Muninn block:
+
+```nginx
+error_log /dev/stdout info;
+pid       /tmp/munin_pid;
+daemon    off;
+
+events {}
+
+muninn {
+
+}
+```
+
+To serve DNS over UDP on port 53 add a `dns_listener` directive like below.
+Currently Muninn only supports UDP, but plans to provide for TCP and DoH as 
+development progresses.
+
+```nginx
+error_log /dev/stdout info;
+pid       /tmp/munin_pid;
+daemon    off;
+
+events {}
+
+muninn {
+    dns_listen 0.0.0.0:53;
+}
+```
+
+The default Muninn DNS over UDP server provides for 1024 simultaneous 
+connections. To change this add the `dns_connection_pool_count` like so:
+
+```nginx
+error_log /dev/stdout info;
+pid       /tmp/munin_pid;
+daemon    off;
+
+events {}
+
+muninn {
+    dns_listen 0.0.0.0:53;
+    dns_connection_pool_count 24;
+}
+```

config

@@ -1,5 +1,15 @@
-ngx_module_name=muninn
+ngx_module_type=CORE
-ngx_module_srcs="$ngx_addon_dir/muninn.c"
+ngx_module_name=ngx_muninn_core_module
-ngx_module_link=YES
+ngx_module_srcs="$ngx_addon_dir/muninn_core.c"
+ngx_module_link=ADDON
 
 . auto/module
+
+ngx_module_type=CORE
+ngx_module_name=ngx_muninn_daemon_module
+ngx_module_srcs="$ngx_addon_dir/muninn_daemon.c"
+ngx_module_link=ADDON
+
+. auto/module
+
+ngx_addon_name="muninn"

muninn.c

@@ -1,27 +0,0 @@
-#include <ngx_config.h>
-#include <ngx_core.h>
-
-static ngx_command_t  ngx_muninn_commands[] = {};
-
-static ngx_core_module_t  ngx_muninn_module_ctx = {
-    ngx_string("muninn"),
-    NULL, // create conf
-    NULL  // init conf
-};
-
-ngx_module_t  ngx_muninn_core_module = {
-    NGX_MODULE_V1,
-    &ngx_muninn_module_ctx,                /* module context */
-    ngx_muninn_commands,                   /* module directives */
-    NGX_CORE_MODULE,                       /* module type */
-    NULL,                                  /* init master */
-    NULL,                                  /* init module */
-    NULL,                                  /* init process */
-    NULL,                                  /* init thread */
-    NULL,                                  /* exit thread */
-    NULL,                                  /* exit process */
-    NULL,                                  /* exit master */
-    NGX_MODULE_V1_PADDING
-};
-
-

muninn.h

@@ -0,0 +1,50 @@
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+#define NGX_MUNINN_MODULE        0x42000000
+#define NGX_MUNINN_CONF          0x00000086
+
+#define MUN_DNS_LEGACY_UDP_SZ    512
+#define MUN_DEFAULT_CONN_POOL_SZ 1024
+
+
+/* TODO: (in muninn.h)
+ * need a global tree structure for storing records.
+ * should store both nameservers, empty non-leaves, and leaf node DNS records.
+ *
+ * if possible some text hashing function for fast lookup.
+ *
+ * cache lookup should return closest matching record in tree
+ *
+ * data structure needs to be atomic
+ *
+ * STORE IN SHARED MEMORY
+ */
+
+typedef struct mun_dns_request {
+    ngx_chain_t *in, *out;
+    ngx_pool_t  *pool;
+    ngx_log_t   *log;
+    ngx_uint_t   err_ct;
+    //TODO: record cache lookup closest result
+
+    ngx_event_t *ev;
+
+    struct dep_list {
+        struct mun_dns_request *dependency;
+        struct dep_list        *next;
+    } *dependencies;
+
+    struct mun_dns_request *dependent;
+} mun_dns_request;
+
+
+// request phase handlers
+void mun_dns_cache_lookup(ngx_event_t *ev);
+void mun_dns_prep_record_fetch(ngx_event_t *ev);
+void mun_dns_record_fetch(ngx_event_t *ev);
+void mun_postpone_request(ngx_event_t *ev);
+void mun_dns_record_response(ngx_event_t *ev);
+void mun_handle_request_error(ngx_event_t *ev);
+void mun_request_teardown(ngx_event_t *ev);

muninn_core.c

@@ -0,0 +1,55 @@
+#include "muninn.h"
+
+
+static char *mun_conf_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+static void *mun_core_create_conf(ngx_cycle_t *cycle);
+
+
+static ngx_command_t  ngx_muninn_core_commands[] = {
+    { ngx_string("muninn"),
+      NGX_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS,
+      mun_conf_server,
+      0, 0, NULL },
+
+    ngx_null_command
+};
+
+static ngx_core_module_t ngx_muninn_core_module_ctx = {
+    ngx_string("muninn_core"),
+    mun_core_create_conf,
+    NULL,
+};
+
+ngx_module_t ngx_muninn_core_module = {
+    NGX_MODULE_V1,
+    &ngx_muninn_core_module_ctx,
+    ngx_muninn_core_commands,
+    NGX_CORE_MODULE,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NGX_MODULE_V1_PADDING
+};
+
+
+static void *mun_core_create_conf(ngx_cycle_t *cycle) {
+    ngx_int_t *conf;
+    if (!(conf = ngx_pcalloc(cycle->pool, sizeof(ngx_int_t))))
+        return NULL;
+
+    return conf;
+}
+
+static char *mun_conf_server(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) {
+    if (!conf) return "no conf";
+    if (**(ngx_int_t **) conf) return "duplicate block";
+
+    **(ngx_int_t **)conf = 1;
+    cf->module_type = NGX_MUNINN_MODULE;
+    cf->cmd_type = NGX_MUNINN_CONF;
+    return ngx_conf_parse(cf, NULL);
+}

muninn_daemon.c

@@ -0,0 +1,273 @@
+#include "ngx_event.h"
+#include "ngx_event_posted.h"
+#include "ngx_palloc.h"
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wdangling-else"
+#else
+#pragma GCC   diagnostic ignored "-Wdangling-else"
+#pragma GCC   diagnostic ignored "-Wmultistatement-macros"
+#endif
+
+#include "muninn.h"
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+
+// Configuration functions
+static char *mun_conf_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+static char *mun_conf_connection_ct(ngx_conf_t *cf, ngx_command_t *c, void *v);
+
+// request state machine
+static void mun_request_parse(ngx_connection_t *c);
+
+
+static ngx_int_t mun_daemon_dns_connection_pool_ct = MUN_DEFAULT_CONN_POOL_SZ;
+
+
+// Module definitions
+static ngx_command_t ngx_muninn_daemon_commands[] = {
+    { ngx_string("dns_listen"),
+      NGX_MUNINN_CONF|NGX_DIRECT_CONF|NGX_CONF_1MORE,
+      mun_conf_listen,
+      0, 0, NULL },
+
+    { ngx_string("dns_connection_pool_count"),
+      NGX_MUNINN_CONF|NGX_DIRECT_CONF|NGX_CONF_1MORE,
+      mun_conf_connection_ct,
+      0, 0, NULL },
+
+    // TODO: shared memory size cap
+    // TODO: dns resource retry limit
+
+    ngx_null_command
+};
+
+static ngx_core_module_t ngx_muninn_daemon_module_ctx = {
+    ngx_string("muninn_daemon"), NULL, NULL,
+    /* TODO:
+     * 1. init shm somewhere in here
+     */
+};
+
+ngx_module_t ngx_muninn_daemon_module = {
+    NGX_MODULE_V1,
+    &ngx_muninn_daemon_module_ctx,
+    ngx_muninn_daemon_commands,
+    NGX_MUNINN_MODULE,
+    NULL,
+    NULL,
+    NULL, // TODO: make bootstrapping request on a single worker here
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NGX_MODULE_V1_PADDING
+};
+
+
+static char *mun_conf_connection_ct(
+    ngx_conf_t *cf,
+    ngx_command_t *cmd,
+    void *conf
+) {
+    ngx_int_t  argc = cf->args->nelts;
+    ngx_str_t *argv = cf->args->elts;
+    ngx_int_t  num;
+
+    if (argc < 1) {
+        ngx_conf_log_error(NGX_LOG_ERR, cf, 0,
+                "dns_connection_pool_count requires at least one argument");
+        return NGX_CONF_ERROR;
+    }
+
+    if ((num = ngx_atoi(argv[1].data, argv[1].len)) == NGX_ERROR) {
+        ngx_conf_log_error(NGX_LOG_ERR, cf, 0,
+                "invalid integer passed to dns_connection_pool_count");
+        return NGX_CONF_ERROR;
+    }
+
+    if (num < 1) {
+        ngx_conf_log_error(NGX_LOG_ERR, cf, 0,
+                "dns_connection_pool_count must be set to at least 1");
+        return NGX_CONF_ERROR;
+    }
+
+    mun_daemon_dns_connection_pool_ct = num;
+    return NGX_CONF_OK;
+}
+
+static char *mun_conf_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *in) {
+    ngx_url_t        *u;
+    ngx_int_t         argc;
+    ngx_str_t        *argv;
+    ngx_listening_t  *listening;
+
+    argc = cf->args->nelts;
+    argv = cf->args->elts;
+
+    if (argc < 1) {
+        ngx_conf_log_error(NGX_LOG_ERR, cf, 0,
+                "dns_listen needs at least one argument");
+        return NGX_CONF_ERROR;
+    }
+
+    if (!(u = ngx_pcalloc(cf->pool, sizeof(ngx_url_t)))) return NGX_CONF_ERROR;
+
+    u->url = argv[1];
+    u->listen = 1;
+    u->default_port = 53;
+    if (ngx_parse_url(cf->pool, u) != NGX_OK) {
+        if (u->err) {
+            ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+                    "%s in \"%V\" of the \"dns_listen\" directive",
+                    u->err, &u->url);
+        }
+
+        return NGX_CONF_ERROR;
+    }
+
+    listening = ngx_create_listening(cf, (struct sockaddr *) &u->sockaddr,
+            u->socklen);
+    if (!listening) return NGX_CONF_ERROR;
+
+    listening->addr_ntop = 1;
+    listening->pool_size = mun_daemon_dns_connection_pool_ct;
+    listening->handler = mun_request_parse;
+    listening->logp = cf->log;
+    listening->log.data = &listening->addr_text;
+    listening->log.handler = ngx_accept_log_error;
+    listening->sndbuf = 1;
+    listening->rcvbuf = 1;
+    listening->backlog = mun_daemon_dns_connection_pool_ct * 2;
+    listening->protocol = IPPROTO_UDP;
+    listening->type = SOCK_DGRAM;
+
+    // listener added to cycle at creation
+    ngx_conf_log_error(NGX_LOG_NOTICE, cf, 0, "muninn DNS server enabled");
+
+    return NGX_CONF_OK;
+}
+
+static void mun_request_parse(ngx_connection_t *c) {
+    ngx_pool_t      *pool;
+    mun_dns_request *r;
+
+    if (!(pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, c->log))) return;
+    if (!(r = ngx_pcalloc(pool, sizeof(mun_dns_request)))) {
+        ngx_destroy_pool(pool);
+        return;
+    }
+
+    if (!(r->ev = ngx_pcalloc(pool, sizeof(ngx_event_t)))) {
+        ngx_destroy_pool(pool);
+        return;
+    }
+
+    r->pool = pool;
+    r->log = c->log;
+    r->ev->data = r;
+    r->ev->handler = mun_dns_cache_lookup;
+    ngx_post_event(r->ev, &ngx_posted_events);
+
+    /* TODO:
+     * 3. track request source in request object here
+     * 4. parse requested DNS resource
+     * 8. log accepted request
+     */
+}
+
+void mun_dns_cache_lookup(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: request cache lookup handler
+     * if hit, clear dependencies and set request next handler to postpone phase
+     * if miss next handler is request records fetch phase AND attach cache returned closest node to tree
+     * reschedule our request event.
+     */
+}
+
+void mun_dns_prep_record_fetch(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: request record(s) prep fetch
+     * find closest neighbor in request struct and figure out all resources we are missing
+     * initialize linked requests for all dependencies.
+     * each linked request should store a reference to this one as a dependent.
+     * their handlers should all be set to fetch record (next)
+     *
+     * set this request next phase to postpone phase
+     * dont reschedule.
+     */
+}
+
+void mun_dns_record_fetch(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: fetch record phase
+     * open a UDP connection or whatever the fuck to some godawful NS and get
+     * the actual record.
+     * finally store the record in the cache.
+     *
+     * next phase is postpone phase.
+     * do reschedule.
+     */
+}
+
+void mun_postpone_request(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: POSTPONE PHASE FOR LINKED REQUESTS
+     * IFF there are still linked requests we depend on then return.
+     * if there are requests depending on us then post their events now.
+     *     remove ourselves from their dependency linkedlist.
+     *     next phase is teardown, reschedule.
+     *
+     * if we have a request source (client wants to know) then set next to output phase
+     * call output phase without rescheduling.
+     */
+}
+
+void mun_dns_record_response(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: output phase
+     * write the records to client.
+     * next phase is teardown.
+     * reschedule.
+     */
+}
+
+void mun_handle_request_error(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: error handler phase
+     * if dependent request then try to restart request processing for this.
+     *   (increment error counter in mun_dns_request)
+     *   if error counter >= max then set dependent request error counter the same
+     *       AND set dependent request phase handler to error handler
+     *       AND schedule dependent request
+     *       return without reschedule
+     *
+     * otherwise attempt retry
+     * if not retriable then send some DNS error IDK
+     * loop over dependencies and set their handlers to teardown.
+     *     MUST REMOVE THEM FROM QUEUE IF POSTED
+     *     re-schedule them all with teardown handlers.
+     * set next phase to teardown. DONT reschedule.
+     */
+}
+
+void mun_request_teardown(ngx_event_t *ev) {
+    mun_dns_request *r = (mun_dns_request *) ev->data;
+
+    /* TODO: teardown phase
+     * if dependencies then return awaiting full teardown
+     *
+     * if dependent request then remove ourselves from their dependencies
+     *     and schedule dependent request event
+     *
+     * log teardown
+     * delete our pool
+     */
+}

snippets/muninn-test.conf

@@ -0,0 +1,9 @@
+error_log info;
+pid       /tmp/munin_pid;
+
+events {}
+
+muninn {
+    dns_connection_pool_count 5;
+    dns_listen 0.0.0.0:8053;
+}

snippets/test-muninn.sh

@@ -0,0 +1,20 @@
+#!/bin/zsh
+
+set -ex
+alias muninn="nginx/objs/nginx -p $(pwd)/mun_pfx -c muninn-test.conf"
+
+# step 1: build
+make all -j24
+
+# step 2: set up test pfx
+mkdir -p $(pwd)/mun_pfx/logs/
+cp $(pwd)/snippets/muninn-test.conf mun_pfx/
+
+# step 3: test configuration
+muninn -T
+
+#step 4: test runtime
+muninn
+sleep 1
+muninn -s quit
+