Merge commit '484a904fa7' as 'nginx'

nginx/src/event/modules/ngx_devpoll_module.c

@@ -0,0 +1,561 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#if (NGX_TEST_BUILD_DEVPOLL)
+
+/* Solaris declarations */
+
+#ifndef POLLREMOVE
+#define POLLREMOVE   0x0800
+#endif
+#define DP_POLL      0xD001
+#define DP_ISPOLLED  0xD002
+
+struct dvpoll {
+    struct pollfd  *dp_fds;
+    int             dp_nfds;
+    int             dp_timeout;
+};
+
+#endif
+
+
+typedef struct {
+    ngx_uint_t      changes;
+    ngx_uint_t      events;
+} ngx_devpoll_conf_t;
+
+
+static ngx_int_t ngx_devpoll_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static void ngx_devpoll_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_devpoll_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_devpoll_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_devpoll_set_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_devpoll_process_events(ngx_cycle_t *cycle,
+    ngx_msec_t timer, ngx_uint_t flags);
+
+static void *ngx_devpoll_create_conf(ngx_cycle_t *cycle);
+static char *ngx_devpoll_init_conf(ngx_cycle_t *cycle, void *conf);
+
+static int              dp = -1;
+static struct pollfd   *change_list, *event_list;
+static ngx_uint_t       nchanges, max_changes, nevents;
+
+static ngx_event_t    **change_index;
+
+
+static ngx_str_t      devpoll_name = ngx_string("/dev/poll");
+
+static ngx_command_t  ngx_devpoll_commands[] = {
+
+    { ngx_string("devpoll_changes"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_devpoll_conf_t, changes),
+      NULL },
+
+    { ngx_string("devpoll_events"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_devpoll_conf_t, events),
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_event_module_t  ngx_devpoll_module_ctx = {
+    &devpoll_name,
+    ngx_devpoll_create_conf,               /* create configuration */
+    ngx_devpoll_init_conf,                 /* init configuration */
+
+    {
+        ngx_devpoll_add_event,             /* add an event */
+        ngx_devpoll_del_event,             /* delete an event */
+        ngx_devpoll_add_event,             /* enable an event */
+        ngx_devpoll_del_event,             /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+        NULL,                              /* trigger a notify */
+        ngx_devpoll_process_events,        /* process the events */
+        ngx_devpoll_init,                  /* init the events */
+        ngx_devpoll_done,                  /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_devpoll_module = {
+    NGX_MODULE_V1,
+    &ngx_devpoll_module_ctx,               /* module context */
+    ngx_devpoll_commands,                  /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_int_t
+ngx_devpoll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    size_t               n;
+    ngx_devpoll_conf_t  *dpcf;
+
+    dpcf = ngx_event_get_conf(cycle->conf_ctx, ngx_devpoll_module);
+
+    if (dp == -1) {
+        dp = open("/dev/poll", O_RDWR);
+
+        if (dp == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "open(/dev/poll) failed");
+            return NGX_ERROR;
+        }
+    }
+
+    if (max_changes < dpcf->changes) {
+        if (nchanges) {
+            n = nchanges * sizeof(struct pollfd);
+            if (write(dp, change_list, n) != (ssize_t) n) {
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                              "write(/dev/poll) failed");
+                return NGX_ERROR;
+            }
+
+            nchanges = 0;
+        }
+
+        if (change_list) {
+            ngx_free(change_list);
+        }
+
+        change_list = ngx_alloc(sizeof(struct pollfd) * dpcf->changes,
+                                cycle->log);
+        if (change_list == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (change_index) {
+            ngx_free(change_index);
+        }
+
+        change_index = ngx_alloc(sizeof(ngx_event_t *) * dpcf->changes,
+                                 cycle->log);
+        if (change_index == NULL) {
+            return NGX_ERROR;
+        }
+    }
+
+    max_changes = dpcf->changes;
+
+    if (nevents < dpcf->events) {
+        if (event_list) {
+            ngx_free(event_list);
+        }
+
+        event_list = ngx_alloc(sizeof(struct pollfd) * dpcf->events,
+                               cycle->log);
+        if (event_list == NULL) {
+            return NGX_ERROR;
+        }
+    }
+
+    nevents = dpcf->events;
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_devpoll_module_ctx.actions;
+
+    ngx_event_flags = NGX_USE_LEVEL_EVENT|NGX_USE_FD_EVENT;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_devpoll_done(ngx_cycle_t *cycle)
+{
+    if (close(dp) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                      "close(/dev/poll) failed");
+    }
+
+    dp = -1;
+
+    ngx_free(change_list);
+    ngx_free(event_list);
+    ngx_free(change_index);
+
+    change_list = NULL;
+    event_list = NULL;
+    change_index = NULL;
+    max_changes = 0;
+    nchanges = 0;
+    nevents = 0;
+}
+
+
+static ngx_int_t
+ngx_devpoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+#if (NGX_DEBUG)
+    ngx_connection_t *c;
+#endif
+
+#if (NGX_READ_EVENT != POLLIN)
+    event = (event == NGX_READ_EVENT) ? POLLIN : POLLOUT;
+#endif
+
+#if (NGX_DEBUG)
+    c = ev->data;
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "devpoll add event: fd:%d ev:%04Xi", c->fd, event);
+#endif
+
+    ev->active = 1;
+
+    return ngx_devpoll_set_event(ev, event, 0);
+}
+
+
+static ngx_int_t
+ngx_devpoll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+#if (NGX_READ_EVENT != POLLIN)
+    event = (event == NGX_READ_EVENT) ? POLLIN : POLLOUT;
+#endif
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "devpoll del event: fd:%d ev:%04Xi", c->fd, event);
+
+    if (ngx_devpoll_set_event(ev, POLLREMOVE, flags) == NGX_ERROR) {
+        return NGX_ERROR;
+    }
+
+    ev->active = 0;
+
+    if (flags & NGX_CLOSE_EVENT) {
+        e = (event == POLLIN) ? c->write : c->read;
+
+        if (e) {
+            e->active = 0;
+        }
+
+        return NGX_OK;
+    }
+
+    /* restore the pair event if it exists */
+
+    if (event == POLLIN) {
+        e = c->write;
+        event = POLLOUT;
+
+    } else {
+        e = c->read;
+        event = POLLIN;
+    }
+
+    if (e && e->active) {
+        return ngx_devpoll_set_event(e, event, 0);
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_devpoll_set_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    size_t             n;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "devpoll fd:%d ev:%04Xi fl:%04Xi", c->fd, event, flags);
+
+    if (nchanges >= max_changes) {
+        ngx_log_error(NGX_LOG_WARN, ev->log, 0,
+                      "/dev/pool change list is filled up");
+
+        n = nchanges * sizeof(struct pollfd);
+        if (write(dp, change_list, n) != (ssize_t) n) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
+                          "write(/dev/poll) failed");
+            return NGX_ERROR;
+        }
+
+        nchanges = 0;
+    }
+
+    change_list[nchanges].fd = c->fd;
+    change_list[nchanges].events = (short) event;
+    change_list[nchanges].revents = 0;
+
+    change_index[nchanges] = ev;
+    ev->index = nchanges;
+
+    nchanges++;
+
+    if (flags & NGX_CLOSE_EVENT) {
+        n = nchanges * sizeof(struct pollfd);
+        if (write(dp, change_list, n) != (ssize_t) n) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
+                          "write(/dev/poll) failed");
+            return NGX_ERROR;
+        }
+
+        nchanges = 0;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_devpoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags)
+{
+    int                 events, revents, rc;
+    size_t              n;
+    ngx_fd_t            fd;
+    ngx_err_t           err;
+    ngx_int_t           i;
+    ngx_uint_t          level, instance;
+    ngx_event_t        *rev, *wev;
+    ngx_queue_t        *queue;
+    ngx_connection_t   *c;
+    struct pollfd       pfd;
+    struct dvpoll       dvp;
+
+    /* NGX_TIMER_INFINITE == INFTIM */
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "devpoll timer: %M", timer);
+
+    if (nchanges) {
+        n = nchanges * sizeof(struct pollfd);
+        if (write(dp, change_list, n) != (ssize_t) n) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                          "write(/dev/poll) failed");
+            return NGX_ERROR;
+        }
+
+        nchanges = 0;
+    }
+
+    dvp.dp_fds = event_list;
+    dvp.dp_nfds = (int) nevents;
+    dvp.dp_timeout = timer;
+    events = ioctl(dp, DP_POLL, &dvp);
+
+    err = (events == -1) ? ngx_errno : 0;
+
+    if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
+        ngx_time_update();
+    }
+
+    if (err) {
+        if (err == NGX_EINTR) {
+
+            if (ngx_event_timer_alarm) {
+                ngx_event_timer_alarm = 0;
+                return NGX_OK;
+            }
+
+            level = NGX_LOG_INFO;
+
+        } else {
+            level = NGX_LOG_ALERT;
+        }
+
+        ngx_log_error(level, cycle->log, err, "ioctl(DP_POLL) failed");
+        return NGX_ERROR;
+    }
+
+    if (events == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "ioctl(DP_POLL) returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    for (i = 0; i < events; i++) {
+
+        fd = event_list[i].fd;
+        revents = event_list[i].revents;
+
+        c = ngx_cycle->files[fd];
+
+        if (c == NULL || c->fd == -1) {
+
+            pfd.fd = fd;
+            pfd.events = 0;
+            pfd.revents = 0;
+
+            rc = ioctl(dp, DP_ISPOLLED, &pfd);
+
+            switch (rc) {
+
+            case -1:
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                    "ioctl(DP_ISPOLLED) failed for socket %d, event %04Xd",
+                    fd, revents);
+                break;
+
+            case 0:
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                    "phantom event %04Xd for closed and removed socket %d",
+                    revents, fd);
+                break;
+
+            default:
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                    "unexpected event %04Xd for closed and removed socket %d, "
+                    "ioctl(DP_ISPOLLED) returned rc:%d, fd:%d, event %04Xd",
+                    revents, fd, rc, pfd.fd, pfd.revents);
+
+                pfd.fd = fd;
+                pfd.events = POLLREMOVE;
+                pfd.revents = 0;
+
+                if (write(dp, &pfd, sizeof(struct pollfd))
+                    != (ssize_t) sizeof(struct pollfd))
+                {
+                    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                                  "write(/dev/poll) for %d failed", fd);
+                }
+
+                if (close(fd) == -1) {
+                    ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                                  "close(%d) failed", fd);
+                }
+
+                break;
+            }
+
+            continue;
+        }
+
+        ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "devpoll: fd:%d, ev:%04Xd, rev:%04Xd",
+                       fd, event_list[i].events, revents);
+
+        if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                          "ioctl(DP_POLL) error fd:%d ev:%04Xd rev:%04Xd",
+                          fd, event_list[i].events, revents);
+        }
+
+        if (revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "strange ioctl(DP_POLL) events "
+                          "fd:%d ev:%04Xd rev:%04Xd",
+                          fd, event_list[i].events, revents);
+        }
+
+        if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
+
+            /*
+             * if the error events were returned, add POLLIN and POLLOUT
+             * to handle the events at least in one active handler
+             */
+
+            revents |= POLLIN|POLLOUT;
+        }
+
+        rev = c->read;
+
+        if ((revents & POLLIN) && rev->active) {
+            rev->ready = 1;
+            rev->available = -1;
+
+            if (flags & NGX_POST_EVENTS) {
+                queue = rev->accept ? &ngx_posted_accept_events
+                                    : &ngx_posted_events;
+
+                ngx_post_event(rev, queue);
+
+            } else {
+                instance = rev->instance;
+
+                rev->handler(rev);
+
+                if (c->fd == -1 || rev->instance != instance) {
+                    continue;
+                }
+            }
+        }
+
+        wev = c->write;
+
+        if ((revents & POLLOUT) && wev->active) {
+            wev->ready = 1;
+
+            if (flags & NGX_POST_EVENTS) {
+                ngx_post_event(wev, &ngx_posted_events);
+
+            } else {
+                wev->handler(wev);
+            }
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+static void *
+ngx_devpoll_create_conf(ngx_cycle_t *cycle)
+{
+    ngx_devpoll_conf_t  *dpcf;
+
+    dpcf = ngx_palloc(cycle->pool, sizeof(ngx_devpoll_conf_t));
+    if (dpcf == NULL) {
+        return NULL;
+    }
+
+    dpcf->changes = NGX_CONF_UNSET;
+    dpcf->events = NGX_CONF_UNSET;
+
+    return dpcf;
+}
+
+
+static char *
+ngx_devpoll_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_devpoll_conf_t *dpcf = conf;
+
+    ngx_conf_init_uint_value(dpcf->changes, 32);
+    ngx_conf_init_uint_value(dpcf->events, 32);
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_eventport_module.c

@@ -0,0 +1,650 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#if (NGX_TEST_BUILD_EVENTPORT)
+
+#define ushort_t  u_short
+#define uint_t    u_int
+
+#ifndef CLOCK_REALTIME
+#define CLOCK_REALTIME          0
+typedef int     clockid_t;
+typedef void *  timer_t;
+#elif (NGX_DARWIN)
+typedef void *  timer_t;
+#endif
+
+/* Solaris declarations */
+
+#define PORT_SOURCE_AIO         1
+#define PORT_SOURCE_TIMER       2
+#define PORT_SOURCE_USER        3
+#define PORT_SOURCE_FD          4
+#define PORT_SOURCE_ALERT       5
+#define PORT_SOURCE_MQ          6
+
+#ifndef ETIME
+#define ETIME                   64
+#endif
+
+#define SIGEV_PORT              4
+
+typedef struct {
+    int         portev_events;  /* event data is source specific */
+    ushort_t    portev_source;  /* event source */
+    ushort_t    portev_pad;     /* port internal use */
+    uintptr_t   portev_object;  /* source specific object */
+    void       *portev_user;    /* user cookie */
+} port_event_t;
+
+typedef struct  port_notify {
+    int         portnfy_port;   /* bind request(s) to port */
+    void       *portnfy_user;   /* user defined */
+} port_notify_t;
+
+#if (__FreeBSD__ && __FreeBSD_version < 700005) || (NGX_DARWIN)
+
+typedef struct itimerspec {     /* definition per POSIX.4 */
+    struct timespec it_interval;/* timer period */
+    struct timespec it_value;   /* timer expiration */
+} itimerspec_t;
+
+#endif
+
+int port_create(void);
+
+int port_create(void)
+{
+    return -1;
+}
+
+
+int port_associate(int port, int source, uintptr_t object, int events,
+    void *user);
+
+int port_associate(int port, int source, uintptr_t object, int events,
+    void *user)
+{
+    return -1;
+}
+
+
+int port_dissociate(int port, int source, uintptr_t object);
+
+int port_dissociate(int port, int source, uintptr_t object)
+{
+    return -1;
+}
+
+
+int port_getn(int port, port_event_t list[], uint_t max, uint_t *nget,
+    struct timespec *timeout);
+
+int port_getn(int port, port_event_t list[], uint_t max, uint_t *nget,
+    struct timespec *timeout)
+{
+    return -1;
+}
+
+int port_send(int port, int events, void *user);
+
+int port_send(int port, int events, void *user)
+{
+    return -1;
+}
+
+
+int timer_create(clockid_t clock_id, struct sigevent *evp, timer_t *timerid);
+
+int timer_create(clockid_t clock_id, struct sigevent *evp, timer_t *timerid)
+{
+    return -1;
+}
+
+
+int timer_settime(timer_t timerid, int flags, const struct itimerspec *value,
+    struct itimerspec *ovalue);
+
+int timer_settime(timer_t timerid, int flags, const struct itimerspec *value,
+    struct itimerspec *ovalue)
+{
+    return -1;
+}
+
+
+int timer_delete(timer_t timerid);
+
+int timer_delete(timer_t timerid)
+{
+    return -1;
+}
+
+#endif
+
+
+typedef struct {
+    ngx_uint_t  events;
+} ngx_eventport_conf_t;
+
+
+static ngx_int_t ngx_eventport_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static void ngx_eventport_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_eventport_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_eventport_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_eventport_notify(ngx_event_handler_pt handler);
+static ngx_int_t ngx_eventport_process_events(ngx_cycle_t *cycle,
+    ngx_msec_t timer, ngx_uint_t flags);
+
+static void *ngx_eventport_create_conf(ngx_cycle_t *cycle);
+static char *ngx_eventport_init_conf(ngx_cycle_t *cycle, void *conf);
+
+static int            ep = -1;
+static port_event_t  *event_list;
+static ngx_uint_t     nevents;
+static timer_t        event_timer = (timer_t) -1;
+static ngx_event_t    notify_event;
+
+static ngx_str_t      eventport_name = ngx_string("eventport");
+
+
+static ngx_command_t  ngx_eventport_commands[] = {
+
+    { ngx_string("eventport_events"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_eventport_conf_t, events),
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_event_module_t  ngx_eventport_module_ctx = {
+    &eventport_name,
+    ngx_eventport_create_conf,             /* create configuration */
+    ngx_eventport_init_conf,               /* init configuration */
+
+    {
+        ngx_eventport_add_event,           /* add an event */
+        ngx_eventport_del_event,           /* delete an event */
+        ngx_eventport_add_event,           /* enable an event */
+        ngx_eventport_del_event,           /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+        ngx_eventport_notify,              /* trigger a notify */
+        ngx_eventport_process_events,      /* process the events */
+        ngx_eventport_init,                /* init the events */
+        ngx_eventport_done,                /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_eventport_module = {
+    NGX_MODULE_V1,
+    &ngx_eventport_module_ctx,             /* module context */
+    ngx_eventport_commands,                /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_int_t
+ngx_eventport_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    port_notify_t          pn;
+    struct itimerspec      its;
+    struct sigevent        sev;
+    ngx_eventport_conf_t  *epcf;
+
+    epcf = ngx_event_get_conf(cycle->conf_ctx, ngx_eventport_module);
+
+    if (ep == -1) {
+        ep = port_create();
+
+        if (ep == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "port_create() failed");
+            return NGX_ERROR;
+        }
+
+        notify_event.active = 1;
+        notify_event.log = cycle->log;
+    }
+
+    if (nevents < epcf->events) {
+        if (event_list) {
+            ngx_free(event_list);
+        }
+
+        event_list = ngx_alloc(sizeof(port_event_t) * epcf->events,
+                               cycle->log);
+        if (event_list == NULL) {
+            return NGX_ERROR;
+        }
+    }
+
+    ngx_event_flags = NGX_USE_EVENTPORT_EVENT;
+
+    if (timer) {
+        ngx_memzero(&pn, sizeof(port_notify_t));
+        pn.portnfy_port = ep;
+
+        ngx_memzero(&sev, sizeof(struct sigevent));
+        sev.sigev_notify = SIGEV_PORT;
+        sev.sigev_value.sival_ptr = &pn;
+
+        if (timer_create(CLOCK_REALTIME, &sev, &event_timer) == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "timer_create() failed");
+            return NGX_ERROR;
+        }
+
+        its.it_interval.tv_sec = timer / 1000;
+        its.it_interval.tv_nsec = (timer % 1000) * 1000000;
+        its.it_value.tv_sec = timer / 1000;
+        its.it_value.tv_nsec = (timer % 1000) * 1000000;
+
+        if (timer_settime(event_timer, 0, &its, NULL) == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "timer_settime() failed");
+            return NGX_ERROR;
+        }
+
+        ngx_event_flags |= NGX_USE_TIMER_EVENT;
+    }
+
+    nevents = epcf->events;
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_eventport_module_ctx.actions;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_eventport_done(ngx_cycle_t *cycle)
+{
+    if (event_timer != (timer_t) -1) {
+        if (timer_delete(event_timer) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                          "timer_delete() failed");
+        }
+
+        event_timer = (timer_t) -1;
+    }
+
+    if (close(ep) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                      "close() event port failed");
+    }
+
+    ep = -1;
+
+    ngx_free(event_list);
+
+    event_list = NULL;
+    nevents = 0;
+}
+
+
+static ngx_int_t
+ngx_eventport_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_int_t          events, prev;
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    events = event;
+
+    if (event == NGX_READ_EVENT) {
+        e = c->write;
+        prev = POLLOUT;
+#if (NGX_READ_EVENT != POLLIN)
+        events = POLLIN;
+#endif
+
+    } else {
+        e = c->read;
+        prev = POLLIN;
+#if (NGX_WRITE_EVENT != POLLOUT)
+        events = POLLOUT;
+#endif
+    }
+
+    if (e->oneshot) {
+        events |= prev;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "eventport add event: fd:%d ev:%04Xi", c->fd, events);
+
+    if (port_associate(ep, PORT_SOURCE_FD, c->fd, events,
+                       (void *) ((uintptr_t) ev | ev->instance))
+        == -1)
+    {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
+                      "port_associate() failed");
+        return NGX_ERROR;
+    }
+
+    ev->active = 1;
+    ev->oneshot = 1;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_eventport_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    /*
+     * when the file descriptor is closed, the event port automatically
+     * dissociates it from the port, so we do not need to dissociate explicitly
+     * the event before the closing the file descriptor
+     */
+
+    if (flags & NGX_CLOSE_EVENT) {
+        ev->active = 0;
+        ev->oneshot = 0;
+        return NGX_OK;
+    }
+
+    c = ev->data;
+
+    if (event == NGX_READ_EVENT) {
+        e = c->write;
+        event = POLLOUT;
+
+    } else {
+        e = c->read;
+        event = POLLIN;
+    }
+
+    if (e->oneshot) {
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "eventport change event: fd:%d ev:%04Xi", c->fd, event);
+
+        if (port_associate(ep, PORT_SOURCE_FD, c->fd, event,
+                           (void *) ((uintptr_t) ev | ev->instance))
+            == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
+                          "port_associate() failed");
+            return NGX_ERROR;
+        }
+
+    } else if (ev->active) {
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "eventport del event: fd:%d", c->fd);
+
+        if (port_dissociate(ep, PORT_SOURCE_FD, c->fd) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
+                          "port_dissociate() failed");
+            return NGX_ERROR;
+        }
+    }
+
+    ev->active = 0;
+    ev->oneshot = 0;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_eventport_notify(ngx_event_handler_pt handler)
+{
+    notify_event.handler = handler;
+
+    if (port_send(ep, 0, &notify_event) != 0) {
+        ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno,
+                      "port_send() failed");
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_eventport_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags)
+{
+    int                 n, revents;
+    u_int               events;
+    ngx_err_t           err;
+    ngx_int_t           instance;
+    ngx_uint_t          i, level;
+    ngx_event_t        *ev, *rev, *wev;
+    ngx_queue_t        *queue;
+    ngx_connection_t   *c;
+    struct timespec     ts, *tp;
+
+    if (timer == NGX_TIMER_INFINITE) {
+        tp = NULL;
+
+    } else {
+        ts.tv_sec = timer / 1000;
+        ts.tv_nsec = (timer % 1000) * 1000000;
+        tp = &ts;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "eventport timer: %M", timer);
+
+    events = 1;
+
+    n = port_getn(ep, event_list, (u_int) nevents, &events, tp);
+
+    err = ngx_errno;
+
+    if (flags & NGX_UPDATE_TIME) {
+        ngx_time_update();
+    }
+
+    if (n == -1) {
+        if (err == ETIME) {
+            if (timer != NGX_TIMER_INFINITE) {
+                return NGX_OK;
+            }
+
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "port_getn() returned no events without timeout");
+            return NGX_ERROR;
+        }
+
+        level = (err == NGX_EINTR) ? NGX_LOG_INFO : NGX_LOG_ALERT;
+        ngx_log_error(level, cycle->log, err, "port_getn() failed");
+        return NGX_ERROR;
+    }
+
+    if (events == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "port_getn() returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    for (i = 0; i < events; i++) {
+
+        if (event_list[i].portev_source == PORT_SOURCE_TIMER) {
+            ngx_time_update();
+            continue;
+        }
+
+        ev = event_list[i].portev_user;
+
+        switch (event_list[i].portev_source) {
+
+        case PORT_SOURCE_FD:
+
+            instance = (uintptr_t) ev & 1;
+            ev = (ngx_event_t *) ((uintptr_t) ev & (uintptr_t) ~1);
+
+            if (ev->closed || ev->instance != instance) {
+
+                /*
+                 * the stale event from a file descriptor
+                 * that was just closed in this iteration
+                 */
+
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "eventport: stale event %p", ev);
+                continue;
+            }
+
+            revents = event_list[i].portev_events;
+
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "eventport: fd:%d, ev:%04Xd",
+                           (int) event_list[i].portev_object, revents);
+
+            if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "port_getn() error fd:%d ev:%04Xd",
+                               (int) event_list[i].portev_object, revents);
+            }
+
+            if (revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                              "strange port_getn() events fd:%d ev:%04Xd",
+                              (int) event_list[i].portev_object, revents);
+            }
+
+            if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
+
+                /*
+                 * if the error events were returned, add POLLIN and POLLOUT
+                 * to handle the events at least in one active handler
+                 */
+
+                revents |= POLLIN|POLLOUT;
+            }
+
+            c = ev->data;
+            rev = c->read;
+            wev = c->write;
+
+            rev->active = 0;
+            wev->active = 0;
+
+            if (revents & POLLIN) {
+                rev->ready = 1;
+                rev->available = -1;
+
+                if (flags & NGX_POST_EVENTS) {
+                    queue = rev->accept ? &ngx_posted_accept_events
+                                        : &ngx_posted_events;
+
+                    ngx_post_event(rev, queue);
+
+                } else {
+                    rev->handler(rev);
+
+                    if (ev->closed || ev->instance != instance) {
+                        continue;
+                    }
+                }
+
+                if (rev->accept) {
+                    if (ngx_use_accept_mutex) {
+                        ngx_accept_events = 1;
+                        continue;
+                    }
+
+                    if (port_associate(ep, PORT_SOURCE_FD, c->fd, POLLIN,
+                                       (void *) ((uintptr_t) ev | ev->instance))
+                        == -1)
+                    {
+                        ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
+                                      "port_associate() failed");
+                        return NGX_ERROR;
+                    }
+                }
+            }
+
+            if (revents & POLLOUT) {
+                wev->ready = 1;
+
+                if (flags & NGX_POST_EVENTS) {
+                    ngx_post_event(wev, &ngx_posted_events);
+
+                } else {
+                    wev->handler(wev);
+                }
+            }
+
+            continue;
+
+        case PORT_SOURCE_USER:
+
+            ev->handler(ev);
+
+            continue;
+
+        default:
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "unexpected eventport object %d",
+                          (int) event_list[i].portev_object);
+            continue;
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+static void *
+ngx_eventport_create_conf(ngx_cycle_t *cycle)
+{
+    ngx_eventport_conf_t  *epcf;
+
+    epcf = ngx_palloc(cycle->pool, sizeof(ngx_eventport_conf_t));
+    if (epcf == NULL) {
+        return NULL;
+    }
+
+    epcf->events = NGX_CONF_UNSET;
+
+    return epcf;
+}
+
+
+static char *
+ngx_eventport_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_eventport_conf_t *epcf = conf;
+
+    ngx_conf_init_uint_value(epcf->events, 32);
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_iocp_module.c

@@ -0,0 +1,379 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_iocp_module.h>
+
+
+static ngx_int_t ngx_iocp_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static ngx_thread_value_t __stdcall ngx_iocp_timer(void *data);
+static void ngx_iocp_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_iocp_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t key);
+static ngx_int_t ngx_iocp_del_connection(ngx_connection_t *c, ngx_uint_t flags);
+static ngx_int_t ngx_iocp_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags);
+static void *ngx_iocp_create_conf(ngx_cycle_t *cycle);
+static char *ngx_iocp_init_conf(ngx_cycle_t *cycle, void *conf);
+
+
+static ngx_str_t      iocp_name = ngx_string("iocp");
+
+static ngx_command_t  ngx_iocp_commands[] = {
+
+    { ngx_string("iocp_threads"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_iocp_conf_t, threads),
+      NULL },
+
+    { ngx_string("post_acceptex"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_iocp_conf_t, post_acceptex),
+      NULL },
+
+    { ngx_string("acceptex_read"),
+      NGX_EVENT_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      0,
+      offsetof(ngx_iocp_conf_t, acceptex_read),
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_event_module_t  ngx_iocp_module_ctx = {
+    &iocp_name,
+    ngx_iocp_create_conf,                  /* create configuration */
+    ngx_iocp_init_conf,                    /* init configuration */
+
+    {
+        ngx_iocp_add_event,                /* add an event */
+        NULL,                              /* delete an event */
+        NULL,                              /* enable an event */
+        NULL,                              /* disable an event */
+        NULL,                              /* add an connection */
+        ngx_iocp_del_connection,           /* delete an connection */
+        NULL,                              /* trigger a notify */
+        ngx_iocp_process_events,           /* process the events */
+        ngx_iocp_init,                     /* init the events */
+        ngx_iocp_done                      /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_iocp_module = {
+    NGX_MODULE_V1,
+    &ngx_iocp_module_ctx,                  /* module context */
+    ngx_iocp_commands,                     /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+ngx_os_io_t ngx_iocp_io = {
+    ngx_overlapped_wsarecv,
+    NULL,
+    ngx_udp_overlapped_wsarecv,
+    NULL,
+    NULL,
+    NULL,
+    ngx_overlapped_wsasend_chain,
+    0
+};
+
+
+static HANDLE      iocp;
+static ngx_tid_t   timer_thread;
+static ngx_msec_t  msec;
+
+
+static ngx_int_t
+ngx_iocp_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    ngx_iocp_conf_t  *cf;
+
+    cf = ngx_event_get_conf(cycle->conf_ctx, ngx_iocp_module);
+
+    if (iocp == NULL) {
+        iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0,
+                                      cf->threads);
+    }
+
+    if (iocp == NULL) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                      "CreateIoCompletionPort() failed");
+        return NGX_ERROR;
+    }
+
+    ngx_io = ngx_iocp_io;
+
+    ngx_event_actions = ngx_iocp_module_ctx.actions;
+
+    ngx_event_flags = NGX_USE_IOCP_EVENT;
+
+    if (timer == 0) {
+        return NGX_OK;
+    }
+
+    /*
+     * The waitable timer could not be used, because
+     * GetQueuedCompletionStatus() does not set a thread to alertable state
+     */
+
+    if (timer_thread == NULL) {
+
+        msec = timer;
+
+        if (ngx_create_thread(&timer_thread, ngx_iocp_timer, &msec, cycle->log)
+            != 0)
+        {
+            return NGX_ERROR;
+        }
+    }
+
+    ngx_event_flags |= NGX_USE_TIMER_EVENT;
+
+    return NGX_OK;
+}
+
+
+static ngx_thread_value_t __stdcall
+ngx_iocp_timer(void *data)
+{
+    ngx_msec_t  timer = *(ngx_msec_t *) data;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0,
+                   "THREAD %p %p", &msec, data);
+
+    for ( ;; ) {
+        Sleep(timer);
+
+        ngx_time_update();
+#if 1
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0, "timer");
+#endif
+    }
+
+#if defined(__WATCOMC__) || defined(__GNUC__)
+    return 0;
+#endif
+}
+
+
+static void
+ngx_iocp_done(ngx_cycle_t *cycle)
+{
+    if (CloseHandle(iocp) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                      "iocp CloseHandle() failed");
+    }
+
+    iocp = NULL;
+}
+
+
+static ngx_int_t
+ngx_iocp_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t key)
+{
+    ngx_connection_t  *c;
+
+    c = (ngx_connection_t *) ev->data;
+
+    c->read->active = 1;
+    c->write->active = 1;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "iocp add: fd:%d k:%ui ov:%p", c->fd, key, &ev->ovlp);
+
+    if (CreateIoCompletionPort((HANDLE) c->fd, iocp, key, 0) == NULL) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      "CreateIoCompletionPort() failed");
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_iocp_del_connection(ngx_connection_t *c, ngx_uint_t flags)
+{
+#if 0
+    if (flags & NGX_CLOSE_EVENT) {
+        return NGX_OK;
+    }
+
+    if (CancelIo((HANDLE) c->fd) == 0) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, "CancelIo() failed");
+        return NGX_ERROR;
+    }
+#endif
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_iocp_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
+{
+    int                rc;
+    u_int              key;
+    u_long             bytes;
+    ngx_err_t          err;
+    ngx_msec_t         delta;
+    ngx_event_t       *ev;
+    ngx_event_ovlp_t  *ovlp;
+
+    if (timer == NGX_TIMER_INFINITE) {
+        timer = INFINITE;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "iocp timer: %M", timer);
+
+    rc = GetQueuedCompletionStatus(iocp, &bytes, (PULONG_PTR) &key,
+                                   (LPOVERLAPPED *) &ovlp, (u_long) timer);
+
+    if (rc == 0) {
+        err = ngx_errno;
+    } else {
+        err = 0;
+    }
+
+    delta = ngx_current_msec;
+
+    if (flags & NGX_UPDATE_TIME) {
+        ngx_time_update();
+    }
+
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "iocp: %d b:%d k:%d ov:%p", rc, bytes, key, ovlp);
+
+    if (timer != INFINITE) {
+        delta = ngx_current_msec - delta;
+
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "iocp timer: %M, delta: %M", timer, delta);
+    }
+
+    if (err) {
+        if (ovlp == NULL) {
+            if (err != WAIT_TIMEOUT) {
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, err,
+                              "GetQueuedCompletionStatus() failed");
+
+                return NGX_ERROR;
+            }
+
+            return NGX_OK;
+        }
+
+        ovlp->error = err;
+    }
+
+    if (ovlp == NULL) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "GetQueuedCompletionStatus() returned no operation");
+        return NGX_ERROR;
+    }
+
+
+    ev = ovlp->event;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, err, "iocp event:%p", ev);
+
+
+    if (err == ERROR_NETNAME_DELETED /* the socket was closed */
+        || err == ERROR_OPERATION_ABORTED /* the operation was canceled */)
+    {
+
+        /*
+         * the WSA_OPERATION_ABORTED completion notification
+         * for a file descriptor that was closed
+         */
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, err,
+                       "iocp: aborted event %p", ev);
+
+        return NGX_OK;
+    }
+
+    if (err) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, err,
+                      "GetQueuedCompletionStatus() returned operation error");
+    }
+
+    switch (key) {
+
+    case NGX_IOCP_ACCEPT:
+        if (bytes) {
+            ev->ready = 1;
+        }
+        break;
+
+    case NGX_IOCP_IO:
+        ev->complete = 1;
+        ev->ready = 1;
+        break;
+
+    case NGX_IOCP_CONNECT:
+        ev->ready = 1;
+    }
+
+    ev->available = bytes;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "iocp event handler: %p", ev->handler);
+
+    ev->handler(ev);
+
+    return NGX_OK;
+}
+
+
+static void *
+ngx_iocp_create_conf(ngx_cycle_t *cycle)
+{
+    ngx_iocp_conf_t  *cf;
+
+    cf = ngx_palloc(cycle->pool, sizeof(ngx_iocp_conf_t));
+    if (cf == NULL) {
+        return NULL;
+    }
+
+    cf->threads = NGX_CONF_UNSET;
+    cf->post_acceptex = NGX_CONF_UNSET;
+    cf->acceptex_read = NGX_CONF_UNSET;
+
+    return cf;
+}
+
+
+static char *
+ngx_iocp_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_iocp_conf_t *cf = conf;
+
+    ngx_conf_init_value(cf->threads, 0);
+    ngx_conf_init_value(cf->post_acceptex, 10);
+    ngx_conf_init_value(cf->acceptex_read, 1);
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_iocp_module.h

@@ -0,0 +1,22 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_IOCP_MODULE_H_INCLUDED_
+#define _NGX_IOCP_MODULE_H_INCLUDED_
+
+
+typedef struct {
+    int  threads;
+    int  post_acceptex;
+    int  acceptex_read;
+} ngx_iocp_conf_t;
+
+
+extern ngx_module_t  ngx_iocp_module;
+
+
+#endif /* _NGX_IOCP_MODULE_H_INCLUDED_ */

nginx/src/event/modules/ngx_kqueue_module.c

@@ -0,0 +1,731 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+/* NetBSD up to 10.0 incompatibly defines kevent.udata as "intptr_t" */
+
+#if (__NetBSD__ && __NetBSD_Version__ < 1000000000)
+#define NGX_KQUEUE_UDATA_T
+#else
+#define NGX_KQUEUE_UDATA_T  (void *)
+#endif
+
+
+typedef struct {
+    ngx_uint_t  changes;
+    ngx_uint_t  events;
+} ngx_kqueue_conf_t;
+
+
+static ngx_int_t ngx_kqueue_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+#ifdef EVFILT_USER
+static ngx_int_t ngx_kqueue_notify_init(ngx_log_t *log);
+#endif
+static void ngx_kqueue_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_kqueue_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_kqueue_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_kqueue_set_event(ngx_event_t *ev, ngx_int_t filter,
+    ngx_uint_t flags);
+#ifdef EVFILT_USER
+static ngx_int_t ngx_kqueue_notify(ngx_event_handler_pt handler);
+#endif
+static ngx_int_t ngx_kqueue_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags);
+static ngx_inline void ngx_kqueue_dump_event(ngx_log_t *log,
+    struct kevent *kev);
+
+static void *ngx_kqueue_create_conf(ngx_cycle_t *cycle);
+static char *ngx_kqueue_init_conf(ngx_cycle_t *cycle, void *conf);
+
+
+int                    ngx_kqueue = -1;
+
+static struct kevent  *change_list;
+static struct kevent  *event_list;
+static ngx_uint_t      max_changes, nchanges, nevents;
+
+#ifdef EVFILT_USER
+static ngx_event_t     notify_event;
+static struct kevent   notify_kev;
+#endif
+
+
+static ngx_str_t      kqueue_name = ngx_string("kqueue");
+
+static ngx_command_t  ngx_kqueue_commands[] = {
+
+    { ngx_string("kqueue_changes"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_kqueue_conf_t, changes),
+      NULL },
+
+    { ngx_string("kqueue_events"),
+      NGX_EVENT_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      0,
+      offsetof(ngx_kqueue_conf_t, events),
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_event_module_t  ngx_kqueue_module_ctx = {
+    &kqueue_name,
+    ngx_kqueue_create_conf,                /* create configuration */
+    ngx_kqueue_init_conf,                  /* init configuration */
+
+    {
+        ngx_kqueue_add_event,              /* add an event */
+        ngx_kqueue_del_event,              /* delete an event */
+        ngx_kqueue_add_event,              /* enable an event */
+        ngx_kqueue_del_event,              /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+#ifdef EVFILT_USER
+        ngx_kqueue_notify,                 /* trigger a notify */
+#else
+        NULL,                              /* trigger a notify */
+#endif
+        ngx_kqueue_process_events,         /* process the events */
+        ngx_kqueue_init,                   /* init the events */
+        ngx_kqueue_done                    /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_kqueue_module = {
+    NGX_MODULE_V1,
+    &ngx_kqueue_module_ctx,                /* module context */
+    ngx_kqueue_commands,                   /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_int_t
+ngx_kqueue_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    ngx_kqueue_conf_t  *kcf;
+    struct timespec     ts;
+#if (NGX_HAVE_TIMER_EVENT)
+    struct kevent       kev;
+#endif
+
+    kcf = ngx_event_get_conf(cycle->conf_ctx, ngx_kqueue_module);
+
+    if (ngx_kqueue == -1) {
+        ngx_kqueue = kqueue();
+
+        if (ngx_kqueue == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                          "kqueue() failed");
+            return NGX_ERROR;
+        }
+
+#ifdef EVFILT_USER
+        if (ngx_kqueue_notify_init(cycle->log) != NGX_OK) {
+            return NGX_ERROR;
+        }
+#endif
+    }
+
+    if (max_changes < kcf->changes) {
+        if (nchanges) {
+            ts.tv_sec = 0;
+            ts.tv_nsec = 0;
+
+            if (kevent(ngx_kqueue, change_list, (int) nchanges, NULL, 0, &ts)
+                == -1)
+            {
+                ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                              "kevent() failed");
+                return NGX_ERROR;
+            }
+            nchanges = 0;
+        }
+
+        if (change_list) {
+            ngx_free(change_list);
+        }
+
+        change_list = ngx_alloc(kcf->changes * sizeof(struct kevent),
+                                cycle->log);
+        if (change_list == NULL) {
+            return NGX_ERROR;
+        }
+    }
+
+    max_changes = kcf->changes;
+
+    if (nevents < kcf->events) {
+        if (event_list) {
+            ngx_free(event_list);
+        }
+
+        event_list = ngx_alloc(kcf->events * sizeof(struct kevent), cycle->log);
+        if (event_list == NULL) {
+            return NGX_ERROR;
+        }
+    }
+
+    ngx_event_flags = NGX_USE_ONESHOT_EVENT
+                      |NGX_USE_KQUEUE_EVENT
+                      |NGX_USE_VNODE_EVENT;
+
+#if (NGX_HAVE_TIMER_EVENT)
+
+    if (timer) {
+        kev.ident = 0;
+        kev.filter = EVFILT_TIMER;
+        kev.flags = EV_ADD|EV_ENABLE;
+        kev.fflags = 0;
+        kev.data = timer;
+        kev.udata = NGX_KQUEUE_UDATA_T (uintptr_t) 0;
+
+        ts.tv_sec = 0;
+        ts.tv_nsec = 0;
+
+        if (kevent(ngx_kqueue, &kev, 1, NULL, 0, &ts) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                          "kevent(EVFILT_TIMER) failed");
+            return NGX_ERROR;
+        }
+
+        ngx_event_flags |= NGX_USE_TIMER_EVENT;
+    }
+
+#endif
+
+#if (NGX_HAVE_CLEAR_EVENT)
+    ngx_event_flags |= NGX_USE_CLEAR_EVENT;
+#else
+    ngx_event_flags |= NGX_USE_LEVEL_EVENT;
+#endif
+
+#if (NGX_HAVE_LOWAT_EVENT)
+    ngx_event_flags |= NGX_USE_LOWAT_EVENT;
+#endif
+
+    nevents = kcf->events;
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_kqueue_module_ctx.actions;
+
+    return NGX_OK;
+}
+
+
+#ifdef EVFILT_USER
+
+static ngx_int_t
+ngx_kqueue_notify_init(ngx_log_t *log)
+{
+    notify_kev.ident = 0;
+    notify_kev.filter = EVFILT_USER;
+    notify_kev.data = 0;
+    notify_kev.flags = EV_ADD|EV_CLEAR;
+    notify_kev.fflags = 0;
+    notify_kev.udata = NGX_KQUEUE_UDATA_T (uintptr_t) 0;
+
+    if (kevent(ngx_kqueue, &notify_kev, 1, NULL, 0, NULL) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+                      "kevent(EVFILT_USER, EV_ADD) failed");
+        return NGX_ERROR;
+    }
+
+    notify_event.active = 1;
+    notify_event.log = log;
+
+    notify_kev.flags = 0;
+    notify_kev.fflags = NOTE_TRIGGER;
+    notify_kev.udata = NGX_KQUEUE_UDATA_T ((uintptr_t) &notify_event);
+
+    return NGX_OK;
+}
+
+#endif
+
+
+static void
+ngx_kqueue_done(ngx_cycle_t *cycle)
+{
+    if (close(ngx_kqueue) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
+                      "kqueue close() failed");
+    }
+
+    ngx_kqueue = -1;
+
+    ngx_free(change_list);
+    ngx_free(event_list);
+
+    change_list = NULL;
+    event_list = NULL;
+    max_changes = 0;
+    nchanges = 0;
+    nevents = 0;
+}
+
+
+static ngx_int_t
+ngx_kqueue_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_int_t          rc;
+#if 0
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+#endif
+
+    ev->active = 1;
+    ev->disabled = 0;
+    ev->oneshot = (flags & NGX_ONESHOT_EVENT) ? 1 : 0;
+
+#if 0
+
+    if (ev->index < nchanges
+        && ((uintptr_t) change_list[ev->index].udata & (uintptr_t) ~1)
+            == (uintptr_t) ev)
+    {
+        if (change_list[ev->index].flags == EV_DISABLE) {
+
+            /*
+             * if the EV_DISABLE is still not passed to a kernel
+             * we will not pass it
+             */
+
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                           "kevent activated: %d: ft:%i",
+                           ngx_event_ident(ev->data), event);
+
+            if (ev->index < --nchanges) {
+                e = (ngx_event_t *)
+                    ((uintptr_t) change_list[nchanges].udata & (uintptr_t) ~1);
+                change_list[ev->index] = change_list[nchanges];
+                e->index = ev->index;
+            }
+
+            return NGX_OK;
+        }
+
+        c = ev->data;
+
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "previous event on #%d were not passed in kernel", c->fd);
+
+        return NGX_ERROR;
+    }
+
+#endif
+
+    rc = ngx_kqueue_set_event(ev, event, EV_ADD|EV_ENABLE|flags);
+
+    return rc;
+}
+
+
+static ngx_int_t
+ngx_kqueue_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_int_t     rc;
+    ngx_event_t  *e;
+
+    ev->active = 0;
+    ev->disabled = 0;
+
+    if (ev->index < nchanges
+        && ((uintptr_t) change_list[ev->index].udata & (uintptr_t) ~1)
+            == (uintptr_t) ev)
+    {
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "kevent deleted: %d: ft:%i",
+                       ngx_event_ident(ev->data), event);
+
+        /* if the event is still not passed to a kernel we will not pass it */
+
+        nchanges--;
+
+        if (ev->index < nchanges) {
+            e = (ngx_event_t *)
+                    ((uintptr_t) change_list[nchanges].udata & (uintptr_t) ~1);
+            change_list[ev->index] = change_list[nchanges];
+            e->index = ev->index;
+        }
+
+        return NGX_OK;
+    }
+
+    /*
+     * when the file descriptor is closed the kqueue automatically deletes
+     * its filters so we do not need to delete explicitly the event
+     * before the closing the file descriptor.
+     */
+
+    if (flags & NGX_CLOSE_EVENT) {
+        return NGX_OK;
+    }
+
+    if (flags & NGX_DISABLE_EVENT) {
+        ev->disabled = 1;
+
+    } else {
+        flags |= EV_DELETE;
+    }
+
+    rc = ngx_kqueue_set_event(ev, event, flags);
+
+    return rc;
+}
+
+
+static ngx_int_t
+ngx_kqueue_set_event(ngx_event_t *ev, ngx_int_t filter, ngx_uint_t flags)
+{
+    struct kevent     *kev;
+    struct timespec    ts;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "kevent set event: %d: ft:%i fl:%04Xi",
+                   c->fd, filter, flags);
+
+    if (nchanges >= max_changes) {
+        ngx_log_error(NGX_LOG_WARN, ev->log, 0,
+                      "kqueue change list is filled up");
+
+        ts.tv_sec = 0;
+        ts.tv_nsec = 0;
+
+        if (kevent(ngx_kqueue, change_list, (int) nchanges, NULL, 0, &ts)
+            == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, "kevent() failed");
+            return NGX_ERROR;
+        }
+
+        nchanges = 0;
+    }
+
+    kev = &change_list[nchanges];
+
+    kev->ident = c->fd;
+    kev->filter = (short) filter;
+    kev->flags = (u_short) flags;
+    kev->udata = NGX_KQUEUE_UDATA_T ((uintptr_t) ev | ev->instance);
+
+    if (filter == EVFILT_VNODE) {
+        kev->fflags = NOTE_DELETE|NOTE_WRITE|NOTE_EXTEND
+                                 |NOTE_ATTRIB|NOTE_RENAME
+#if (__FreeBSD__ == 4 && __FreeBSD_version >= 430000) \
+    || __FreeBSD_version >= 500018
+                                 |NOTE_REVOKE
+#endif
+                      ;
+        kev->data = 0;
+
+    } else {
+#if (NGX_HAVE_LOWAT_EVENT)
+        if (flags & NGX_LOWAT_EVENT) {
+            kev->fflags = NOTE_LOWAT;
+            kev->data = ev->available;
+
+        } else {
+            kev->fflags = 0;
+            kev->data = 0;
+        }
+#else
+        kev->fflags = 0;
+        kev->data = 0;
+#endif
+    }
+
+    ev->index = nchanges;
+    nchanges++;
+
+    if (flags & NGX_FLUSH_EVENT) {
+        ts.tv_sec = 0;
+        ts.tv_nsec = 0;
+
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "kevent flush");
+
+        if (kevent(ngx_kqueue, change_list, (int) nchanges, NULL, 0, &ts)
+            == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, "kevent() failed");
+            return NGX_ERROR;
+        }
+
+        nchanges = 0;
+    }
+
+    return NGX_OK;
+}
+
+
+#ifdef EVFILT_USER
+
+static ngx_int_t
+ngx_kqueue_notify(ngx_event_handler_pt handler)
+{
+    notify_event.handler = handler;
+
+    if (kevent(ngx_kqueue, &notify_kev, 1, NULL, 0, NULL) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno,
+                      "kevent(EVFILT_USER, NOTE_TRIGGER) failed");
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+#endif
+
+
+static ngx_int_t
+ngx_kqueue_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags)
+{
+    int               events, n;
+    ngx_int_t         i, instance;
+    ngx_uint_t        level;
+    ngx_err_t         err;
+    ngx_event_t      *ev;
+    ngx_queue_t      *queue;
+    struct timespec   ts, *tp;
+
+    n = (int) nchanges;
+    nchanges = 0;
+
+    if (timer == NGX_TIMER_INFINITE) {
+        tp = NULL;
+
+    } else {
+
+        ts.tv_sec = timer / 1000;
+        ts.tv_nsec = (timer % 1000) * 1000000;
+
+        /*
+         * 64-bit Darwin kernel has the bug: kernel level ts.tv_nsec is
+         * the int32_t while user level ts.tv_nsec is the long (64-bit),
+         * so on the big endian PowerPC all nanoseconds are lost.
+         */
+
+#if (NGX_DARWIN_KEVENT_BUG)
+        ts.tv_nsec <<= 32;
+#endif
+
+        tp = &ts;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "kevent timer: %M, changes: %d", timer, n);
+
+    events = kevent(ngx_kqueue, change_list, n, event_list, (int) nevents, tp);
+
+    err = (events == -1) ? ngx_errno : 0;
+
+    if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
+        ngx_time_update();
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "kevent events: %d", events);
+
+    if (err) {
+        if (err == NGX_EINTR) {
+
+            if (ngx_event_timer_alarm) {
+                ngx_event_timer_alarm = 0;
+                return NGX_OK;
+            }
+
+            level = NGX_LOG_INFO;
+
+        } else {
+            level = NGX_LOG_ALERT;
+        }
+
+        ngx_log_error(level, cycle->log, err, "kevent() failed");
+        return NGX_ERROR;
+    }
+
+    if (events == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "kevent() returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    for (i = 0; i < events; i++) {
+
+        ngx_kqueue_dump_event(cycle->log, &event_list[i]);
+
+        if (event_list[i].flags & EV_ERROR) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, event_list[i].data,
+                          "kevent() error on %d filter:%d flags:%04Xd",
+                          (int) event_list[i].ident, event_list[i].filter,
+                          event_list[i].flags);
+            continue;
+        }
+
+#if (NGX_HAVE_TIMER_EVENT)
+
+        if (event_list[i].filter == EVFILT_TIMER) {
+            ngx_time_update();
+            continue;
+        }
+
+#endif
+
+        ev = (ngx_event_t *) event_list[i].udata;
+
+        switch (event_list[i].filter) {
+
+        case EVFILT_READ:
+        case EVFILT_WRITE:
+
+            instance = (uintptr_t) ev & 1;
+            ev = (ngx_event_t *) ((uintptr_t) ev & (uintptr_t) ~1);
+
+            if (ev->closed || ev->instance != instance) {
+
+                /*
+                 * the stale event from a file descriptor
+                 * that was just closed in this iteration
+                 */
+
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "kevent: stale event %p", ev);
+                continue;
+            }
+
+            if (ev->log && (ev->log->log_level & NGX_LOG_DEBUG_CONNECTION)) {
+                ngx_kqueue_dump_event(ev->log, &event_list[i]);
+            }
+
+            if (ev->oneshot) {
+                ev->active = 0;
+            }
+
+            ev->available = event_list[i].data;
+
+            if (event_list[i].flags & EV_EOF) {
+                ev->pending_eof = 1;
+                ev->kq_errno = event_list[i].fflags;
+            }
+
+            ev->ready = 1;
+
+            break;
+
+        case EVFILT_VNODE:
+            ev->kq_vnode = 1;
+
+            break;
+
+        case EVFILT_AIO:
+            ev->complete = 1;
+            ev->ready = 1;
+
+            break;
+
+#ifdef EVFILT_USER
+        case EVFILT_USER:
+            break;
+#endif
+
+        default:
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "unexpected kevent() filter %d",
+                          event_list[i].filter);
+            continue;
+        }
+
+        if (flags & NGX_POST_EVENTS) {
+            queue = ev->accept ? &ngx_posted_accept_events
+                               : &ngx_posted_events;
+
+            ngx_post_event(ev, queue);
+
+            continue;
+        }
+
+        ev->handler(ev);
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_inline void
+ngx_kqueue_dump_event(ngx_log_t *log, struct kevent *kev)
+{
+    if (kev->ident > 0x8000000 && kev->ident != (unsigned) -1) {
+        ngx_log_debug6(NGX_LOG_DEBUG_EVENT, log, 0,
+                       "kevent: %p: ft:%d fl:%04Xd ff:%08Xd d:%d ud:%p",
+                       (void *) kev->ident, kev->filter,
+                       kev->flags, kev->fflags,
+                       (int) kev->data, kev->udata);
+
+    } else {
+        ngx_log_debug6(NGX_LOG_DEBUG_EVENT, log, 0,
+                       "kevent: %d: ft:%d fl:%04Xd ff:%08Xd d:%d ud:%p",
+                       (int) kev->ident, kev->filter,
+                       kev->flags, kev->fflags,
+                       (int) kev->data, kev->udata);
+    }
+}
+
+
+static void *
+ngx_kqueue_create_conf(ngx_cycle_t *cycle)
+{
+    ngx_kqueue_conf_t  *kcf;
+
+    kcf = ngx_palloc(cycle->pool, sizeof(ngx_kqueue_conf_t));
+    if (kcf == NULL) {
+        return NULL;
+    }
+
+    kcf->changes = NGX_CONF_UNSET;
+    kcf->events = NGX_CONF_UNSET;
+
+    return kcf;
+}
+
+
+static char *
+ngx_kqueue_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_kqueue_conf_t *kcf = conf;
+
+    ngx_conf_init_uint_value(kcf->changes, 512);
+    ngx_conf_init_uint_value(kcf->events, 512);
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_poll_module.c

@@ -0,0 +1,416 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+static ngx_int_t ngx_poll_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static void ngx_poll_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_poll_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_poll_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_poll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags);
+static char *ngx_poll_init_conf(ngx_cycle_t *cycle, void *conf);
+
+
+static struct pollfd  *event_list;
+static ngx_uint_t      nevents;
+
+
+static ngx_str_t           poll_name = ngx_string("poll");
+
+static ngx_event_module_t  ngx_poll_module_ctx = {
+    &poll_name,
+    NULL,                                  /* create configuration */
+    ngx_poll_init_conf,                    /* init configuration */
+
+    {
+        ngx_poll_add_event,                /* add an event */
+        ngx_poll_del_event,                /* delete an event */
+        ngx_poll_add_event,                /* enable an event */
+        ngx_poll_del_event,                /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+        NULL,                              /* trigger a notify */
+        ngx_poll_process_events,           /* process the events */
+        ngx_poll_init,                     /* init the events */
+        ngx_poll_done                      /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_poll_module = {
+    NGX_MODULE_V1,
+    &ngx_poll_module_ctx,                  /* module context */
+    NULL,                                  /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+
+static ngx_int_t
+ngx_poll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    struct pollfd   *list;
+
+    if (event_list == NULL) {
+        nevents = 0;
+    }
+
+    if (ngx_process >= NGX_PROCESS_WORKER
+        || cycle->old_cycle == NULL
+        || cycle->old_cycle->connection_n < cycle->connection_n)
+    {
+        list = ngx_alloc(sizeof(struct pollfd) * cycle->connection_n,
+                         cycle->log);
+        if (list == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (event_list) {
+            ngx_memcpy(list, event_list, sizeof(struct pollfd) * nevents);
+            ngx_free(event_list);
+        }
+
+        event_list = list;
+    }
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_poll_module_ctx.actions;
+
+    ngx_event_flags = NGX_USE_LEVEL_EVENT|NGX_USE_FD_EVENT;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_poll_done(ngx_cycle_t *cycle)
+{
+    ngx_free(event_list);
+
+    event_list = NULL;
+}
+
+
+static ngx_int_t
+ngx_poll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ev->active = 1;
+
+    if (ev->index != NGX_INVALID_INDEX) {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "poll event fd:%d ev:%i is already set", c->fd, event);
+        return NGX_OK;
+    }
+
+    if (event == NGX_READ_EVENT) {
+        e = c->write;
+#if (NGX_READ_EVENT != POLLIN)
+        event = POLLIN;
+#endif
+
+    } else {
+        e = c->read;
+#if (NGX_WRITE_EVENT != POLLOUT)
+        event = POLLOUT;
+#endif
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "poll add event: fd:%d ev:%i", c->fd, event);
+
+    if (e == NULL || e->index == NGX_INVALID_INDEX) {
+        event_list[nevents].fd = c->fd;
+        event_list[nevents].events = (short) event;
+        event_list[nevents].revents = 0;
+
+        ev->index = nevents;
+        nevents++;
+
+    } else {
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "poll add index: %i", e->index);
+
+        event_list[e->index].events |= (short) event;
+        ev->index = e->index;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_poll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ev->active = 0;
+
+    if (ev->index == NGX_INVALID_INDEX) {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "poll event fd:%d ev:%i is already deleted",
+                      c->fd, event);
+        return NGX_OK;
+    }
+
+    if (event == NGX_READ_EVENT) {
+        e = c->write;
+#if (NGX_READ_EVENT != POLLIN)
+        event = POLLIN;
+#endif
+
+    } else {
+        e = c->read;
+#if (NGX_WRITE_EVENT != POLLOUT)
+        event = POLLOUT;
+#endif
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "poll del event: fd:%d ev:%i", c->fd, event);
+
+    if (e == NULL || e->index == NGX_INVALID_INDEX) {
+        nevents--;
+
+        if (ev->index < nevents) {
+
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                           "index: copy event %ui to %i", nevents, ev->index);
+
+            event_list[ev->index] = event_list[nevents];
+
+            c = ngx_cycle->files[event_list[nevents].fd];
+
+            if (c->fd == -1) {
+                ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                              "unexpected last event");
+
+            } else {
+                if (c->read->index == nevents) {
+                    c->read->index = ev->index;
+                }
+
+                if (c->write->index == nevents) {
+                    c->write->index = ev->index;
+                }
+            }
+        }
+
+    } else {
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "poll del index: %i", e->index);
+
+        event_list[e->index].events &= (short) ~event;
+    }
+
+    ev->index = NGX_INVALID_INDEX;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_poll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
+{
+    int                 ready, revents;
+    ngx_err_t           err;
+    ngx_uint_t          i, found, level;
+    ngx_event_t        *ev;
+    ngx_queue_t        *queue;
+    ngx_connection_t   *c;
+
+    /* NGX_TIMER_INFINITE == INFTIM */
+
+#if (NGX_DEBUG0)
+    if (cycle->log->log_level & NGX_LOG_DEBUG_ALL) {
+        for (i = 0; i < nevents; i++) {
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "poll: %ui: fd:%d ev:%04Xd",
+                           i, event_list[i].fd, event_list[i].events);
+        }
+    }
+#endif
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "poll timer: %M", timer);
+
+    ready = poll(event_list, (u_int) nevents, (int) timer);
+
+    err = (ready == -1) ? ngx_errno : 0;
+
+    if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
+        ngx_time_update();
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "poll ready %d of %ui", ready, nevents);
+
+    if (err) {
+        if (err == NGX_EINTR) {
+
+            if (ngx_event_timer_alarm) {
+                ngx_event_timer_alarm = 0;
+                return NGX_OK;
+            }
+
+            level = NGX_LOG_INFO;
+
+        } else {
+            level = NGX_LOG_ALERT;
+        }
+
+        ngx_log_error(level, cycle->log, err, "poll() failed");
+        return NGX_ERROR;
+    }
+
+    if (ready == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "poll() returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    for (i = 0; i < nevents && ready; i++) {
+
+        revents = event_list[i].revents;
+
+#if 1
+        ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "poll: %ui: fd:%d ev:%04Xd rev:%04Xd",
+                       i, event_list[i].fd, event_list[i].events, revents);
+#else
+        if (revents) {
+            ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "poll: %ui: fd:%d ev:%04Xd rev:%04Xd",
+                           i, event_list[i].fd, event_list[i].events, revents);
+        }
+#endif
+
+        if (revents & POLLNVAL) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "poll() error fd:%d ev:%04Xd rev:%04Xd",
+                          event_list[i].fd, event_list[i].events, revents);
+        }
+
+        if (revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "strange poll() events fd:%d ev:%04Xd rev:%04Xd",
+                          event_list[i].fd, event_list[i].events, revents);
+        }
+
+        if (event_list[i].fd == -1) {
+            /*
+             * the disabled event, a workaround for our possible bug,
+             * see the comment below
+             */
+            continue;
+        }
+
+        c = ngx_cycle->files[event_list[i].fd];
+
+        if (c->fd == -1) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "unexpected event");
+
+            /*
+             * it is certainly our fault and it should be investigated,
+             * in the meantime we disable this event to avoid a CPU spinning
+             */
+
+            if (i == nevents - 1) {
+                nevents--;
+            } else {
+                event_list[i].fd = -1;
+            }
+
+            continue;
+        }
+
+        if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
+
+            /*
+             * if the error events were returned, add POLLIN and POLLOUT
+             * to handle the events at least in one active handler
+             */
+
+            revents |= POLLIN|POLLOUT;
+        }
+
+        found = 0;
+
+        if ((revents & POLLIN) && c->read->active) {
+            found = 1;
+
+            ev = c->read;
+            ev->ready = 1;
+            ev->available = -1;
+
+            queue = ev->accept ? &ngx_posted_accept_events
+                               : &ngx_posted_events;
+
+            ngx_post_event(ev, queue);
+        }
+
+        if ((revents & POLLOUT) && c->write->active) {
+            found = 1;
+
+            ev = c->write;
+            ev->ready = 1;
+
+            ngx_post_event(ev, &ngx_posted_events);
+        }
+
+        if (found) {
+            ready--;
+            continue;
+        }
+    }
+
+    if (ready != 0) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "poll ready != events");
+    }
+
+    return NGX_OK;
+}
+
+
+static char *
+ngx_poll_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_event_conf_t  *ecf;
+
+    ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module);
+
+    if (ecf->use != ngx_poll_module.ctx_index) {
+        return NGX_CONF_OK;
+    }
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_select_module.c

@@ -0,0 +1,424 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+static ngx_int_t ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static void ngx_select_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_select_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_select_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags);
+static void ngx_select_repair_fd_sets(ngx_cycle_t *cycle);
+static char *ngx_select_init_conf(ngx_cycle_t *cycle, void *conf);
+
+
+static fd_set         master_read_fd_set;
+static fd_set         master_write_fd_set;
+static fd_set         work_read_fd_set;
+static fd_set         work_write_fd_set;
+
+static ngx_int_t      max_fd;
+static ngx_uint_t     nevents;
+
+static ngx_event_t  **event_index;
+
+
+static ngx_str_t           select_name = ngx_string("select");
+
+static ngx_event_module_t  ngx_select_module_ctx = {
+    &select_name,
+    NULL,                                  /* create configuration */
+    ngx_select_init_conf,                  /* init configuration */
+
+    {
+        ngx_select_add_event,              /* add an event */
+        ngx_select_del_event,              /* delete an event */
+        ngx_select_add_event,              /* enable an event */
+        ngx_select_del_event,              /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+        NULL,                              /* trigger a notify */
+        ngx_select_process_events,         /* process the events */
+        ngx_select_init,                   /* init the events */
+        ngx_select_done                    /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_select_module = {
+    NGX_MODULE_V1,
+    &ngx_select_module_ctx,                /* module context */
+    NULL,                                  /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_int_t
+ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    ngx_event_t  **index;
+
+    if (event_index == NULL) {
+        FD_ZERO(&master_read_fd_set);
+        FD_ZERO(&master_write_fd_set);
+        nevents = 0;
+    }
+
+    if (ngx_process >= NGX_PROCESS_WORKER
+        || cycle->old_cycle == NULL
+        || cycle->old_cycle->connection_n < cycle->connection_n)
+    {
+        index = ngx_alloc(sizeof(ngx_event_t *) * 2 * cycle->connection_n,
+                          cycle->log);
+        if (index == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (event_index) {
+            ngx_memcpy(index, event_index, sizeof(ngx_event_t *) * nevents);
+            ngx_free(event_index);
+        }
+
+        event_index = index;
+    }
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_select_module_ctx.actions;
+
+    ngx_event_flags = NGX_USE_LEVEL_EVENT;
+
+    max_fd = -1;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_select_done(ngx_cycle_t *cycle)
+{
+    ngx_free(event_index);
+
+    event_index = NULL;
+}
+
+
+static ngx_int_t
+ngx_select_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "select add event fd:%d ev:%i", c->fd, event);
+
+    if (ev->index != NGX_INVALID_INDEX) {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "select event fd:%d ev:%i is already set", c->fd, event);
+        return NGX_OK;
+    }
+
+    if ((event == NGX_READ_EVENT && ev->write)
+        || (event == NGX_WRITE_EVENT && !ev->write))
+    {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "invalid select %s event fd:%d ev:%i",
+                      ev->write ? "write" : "read", c->fd, event);
+        return NGX_ERROR;
+    }
+
+    if (event == NGX_READ_EVENT) {
+        FD_SET(c->fd, &master_read_fd_set);
+
+    } else if (event == NGX_WRITE_EVENT) {
+        FD_SET(c->fd, &master_write_fd_set);
+    }
+
+    if (max_fd != -1 && max_fd < c->fd) {
+        max_fd = c->fd;
+    }
+
+    ev->active = 1;
+
+    event_index[nevents] = ev;
+    ev->index = nevents;
+    nevents++;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_select_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ev->active = 0;
+
+    if (ev->index == NGX_INVALID_INDEX) {
+        return NGX_OK;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "select del event fd:%d ev:%i", c->fd, event);
+
+    if (event == NGX_READ_EVENT) {
+        FD_CLR(c->fd, &master_read_fd_set);
+
+    } else if (event == NGX_WRITE_EVENT) {
+        FD_CLR(c->fd, &master_write_fd_set);
+    }
+
+    if (max_fd == c->fd) {
+        max_fd = -1;
+    }
+
+    if (ev->index < --nevents) {
+        e = event_index[nevents];
+        event_index[ev->index] = e;
+        e->index = ev->index;
+    }
+
+    ev->index = NGX_INVALID_INDEX;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags)
+{
+    int                ready, nready;
+    ngx_err_t          err;
+    ngx_uint_t         i, found;
+    ngx_event_t       *ev;
+    ngx_queue_t       *queue;
+    struct timeval     tv, *tp;
+    ngx_connection_t  *c;
+
+    if (max_fd == -1) {
+        for (i = 0; i < nevents; i++) {
+            c = event_index[i]->data;
+            if (max_fd < c->fd) {
+                max_fd = c->fd;
+            }
+        }
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "change max_fd: %i", max_fd);
+    }
+
+#if (NGX_DEBUG)
+    if (cycle->log->log_level & NGX_LOG_DEBUG_ALL) {
+        for (i = 0; i < nevents; i++) {
+            ev = event_index[i];
+            c = ev->data;
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "select event: fd:%d wr:%d", c->fd, ev->write);
+        }
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "max_fd: %i", max_fd);
+    }
+#endif
+
+    if (timer == NGX_TIMER_INFINITE) {
+        tp = NULL;
+
+    } else {
+        tv.tv_sec = (long) (timer / 1000);
+        tv.tv_usec = (long) ((timer % 1000) * 1000);
+        tp = &tv;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "select timer: %M", timer);
+
+    work_read_fd_set = master_read_fd_set;
+    work_write_fd_set = master_write_fd_set;
+
+    ready = select(max_fd + 1, &work_read_fd_set, &work_write_fd_set, NULL, tp);
+
+    err = (ready == -1) ? ngx_errno : 0;
+
+    if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
+        ngx_time_update();
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "select ready %d", ready);
+
+    if (err) {
+        ngx_uint_t  level;
+
+        if (err == NGX_EINTR) {
+
+            if (ngx_event_timer_alarm) {
+                ngx_event_timer_alarm = 0;
+                return NGX_OK;
+            }
+
+            level = NGX_LOG_INFO;
+
+        } else {
+            level = NGX_LOG_ALERT;
+        }
+
+        ngx_log_error(level, cycle->log, err, "select() failed");
+
+        if (err == NGX_EBADF) {
+            ngx_select_repair_fd_sets(cycle);
+        }
+
+        return NGX_ERROR;
+    }
+
+    if (ready == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "select() returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    nready = 0;
+
+    for (i = 0; i < nevents; i++) {
+        ev = event_index[i];
+        c = ev->data;
+        found = 0;
+
+        if (ev->write) {
+            if (FD_ISSET(c->fd, &work_write_fd_set)) {
+                found = 1;
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "select write %d", c->fd);
+            }
+
+        } else {
+            if (FD_ISSET(c->fd, &work_read_fd_set)) {
+                found = 1;
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "select read %d", c->fd);
+            }
+        }
+
+        if (found) {
+            ev->ready = 1;
+            ev->available = -1;
+
+            queue = ev->accept ? &ngx_posted_accept_events
+                               : &ngx_posted_events;
+
+            ngx_post_event(ev, queue);
+
+            nready++;
+        }
+    }
+
+    if (ready != nready) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "select ready != events: %d:%d", ready, nready);
+
+        ngx_select_repair_fd_sets(cycle);
+    }
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_select_repair_fd_sets(ngx_cycle_t *cycle)
+{
+    int           n;
+    socklen_t     len;
+    ngx_err_t     err;
+    ngx_socket_t  s;
+
+    for (s = 0; s <= max_fd; s++) {
+
+        if (FD_ISSET(s, &master_read_fd_set) == 0) {
+            continue;
+        }
+
+        len = sizeof(int);
+
+        if (getsockopt(s, SOL_SOCKET, SO_TYPE, &n, &len) == -1) {
+            err = ngx_socket_errno;
+
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, err,
+                          "invalid descriptor #%d in read fd_set", s);
+
+            FD_CLR(s, &master_read_fd_set);
+        }
+    }
+
+    for (s = 0; s <= max_fd; s++) {
+
+        if (FD_ISSET(s, &master_write_fd_set) == 0) {
+            continue;
+        }
+
+        len = sizeof(int);
+
+        if (getsockopt(s, SOL_SOCKET, SO_TYPE, &n, &len) == -1) {
+            err = ngx_socket_errno;
+
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, err,
+                          "invalid descriptor #%d in write fd_set", s);
+
+            FD_CLR(s, &master_write_fd_set);
+        }
+    }
+
+    max_fd = -1;
+}
+
+
+static char *
+ngx_select_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_event_conf_t  *ecf;
+
+    ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module);
+
+    if (ecf->use != ngx_select_module.ctx_index) {
+        return NGX_CONF_OK;
+    }
+
+    /* disable warning: the default FD_SETSIZE is 1024U in FreeBSD 5.x */
+
+    if (cycle->connection_n > FD_SETSIZE) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                      "the maximum number of files "
+                      "supported by select() is %ud", FD_SETSIZE);
+        return NGX_CONF_ERROR;
+    }
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_win32_poll_module.c

@@ -0,0 +1,436 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Maxim Dounin
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+static ngx_int_t ngx_poll_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static void ngx_poll_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_poll_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_poll_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_poll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags);
+static char *ngx_poll_init_conf(ngx_cycle_t *cycle, void *conf);
+
+
+static struct pollfd      *event_list;
+static ngx_connection_t  **event_index;
+static ngx_uint_t          nevents;
+
+
+static ngx_str_t           poll_name = ngx_string("poll");
+
+static ngx_event_module_t  ngx_poll_module_ctx = {
+    &poll_name,
+    NULL,                                  /* create configuration */
+    ngx_poll_init_conf,                    /* init configuration */
+
+    {
+        ngx_poll_add_event,                /* add an event */
+        ngx_poll_del_event,                /* delete an event */
+        ngx_poll_add_event,                /* enable an event */
+        ngx_poll_del_event,                /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+        NULL,                              /* trigger a notify */
+        ngx_poll_process_events,           /* process the events */
+        ngx_poll_init,                     /* init the events */
+        ngx_poll_done                      /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_poll_module = {
+    NGX_MODULE_V1,
+    &ngx_poll_module_ctx,                  /* module context */
+    NULL,                                  /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+
+static ngx_int_t
+ngx_poll_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    struct pollfd      *list;
+    ngx_connection_t  **index;
+
+    if (event_list == NULL) {
+        nevents = 0;
+    }
+
+    if (ngx_process >= NGX_PROCESS_WORKER
+        || cycle->old_cycle == NULL
+        || cycle->old_cycle->connection_n < cycle->connection_n)
+    {
+        list = ngx_alloc(sizeof(struct pollfd) * cycle->connection_n,
+                         cycle->log);
+        if (list == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (event_list) {
+            ngx_memcpy(list, event_list, sizeof(struct pollfd) * nevents);
+            ngx_free(event_list);
+        }
+
+        event_list = list;
+
+        index = ngx_alloc(sizeof(ngx_connection_t *) * cycle->connection_n,
+                          cycle->log);
+        if (index == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (event_index) {
+            ngx_memcpy(index, event_index,
+                       sizeof(ngx_connection_t *) * nevents);
+            ngx_free(event_index);
+        }
+
+        event_index = index;
+    }
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_poll_module_ctx.actions;
+
+    ngx_event_flags = NGX_USE_LEVEL_EVENT;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_poll_done(ngx_cycle_t *cycle)
+{
+    ngx_free(event_list);
+    ngx_free(event_index);
+
+    event_list = NULL;
+    event_index = NULL;
+}
+
+
+static ngx_int_t
+ngx_poll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ev->active = 1;
+
+    if (ev->index != NGX_INVALID_INDEX) {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "poll event fd:%d ev:%i is already set", c->fd, event);
+        return NGX_OK;
+    }
+
+    if (event == NGX_READ_EVENT) {
+        e = c->write;
+#if (NGX_READ_EVENT != POLLIN)
+        event = POLLIN;
+#endif
+
+    } else {
+        e = c->read;
+#if (NGX_WRITE_EVENT != POLLOUT)
+        event = POLLOUT;
+#endif
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "poll add event: fd:%d ev:%i", c->fd, event);
+
+    if (e == NULL || e->index == NGX_INVALID_INDEX) {
+
+        event_list[nevents].fd = c->fd;
+        event_list[nevents].events = (short) event;
+        event_list[nevents].revents = 0;
+
+        event_index[nevents] = c;
+
+        ev->index = nevents;
+        nevents++;
+
+    } else {
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "poll add index: %i", e->index);
+
+        event_list[e->index].events |= (short) event;
+        ev->index = e->index;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_poll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ev->active = 0;
+
+    if (ev->index == NGX_INVALID_INDEX) {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "poll event fd:%d ev:%i is already deleted",
+                      c->fd, event);
+        return NGX_OK;
+    }
+
+    if (event == NGX_READ_EVENT) {
+        e = c->write;
+#if (NGX_READ_EVENT != POLLIN)
+        event = POLLIN;
+#endif
+
+    } else {
+        e = c->read;
+#if (NGX_WRITE_EVENT != POLLOUT)
+        event = POLLOUT;
+#endif
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "poll del event: fd:%d ev:%i", c->fd, event);
+
+    if (e == NULL || e->index == NGX_INVALID_INDEX) {
+        nevents--;
+
+        if (ev->index < nevents) {
+
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                           "index: copy event %ui to %i", nevents, ev->index);
+
+            event_list[ev->index] = event_list[nevents];
+            event_index[ev->index] = event_index[nevents];
+
+            c = event_index[ev->index];
+
+            if (c->fd == (ngx_socket_t) -1) {
+                ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                              "unexpected last event");
+
+            } else {
+                if (c->read->index == nevents) {
+                    c->read->index = ev->index;
+                }
+
+                if (c->write->index == nevents) {
+                    c->write->index = ev->index;
+                }
+            }
+        }
+
+    } else {
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "poll del index: %i", e->index);
+
+        event_list[e->index].events &= (short) ~event;
+    }
+
+    ev->index = NGX_INVALID_INDEX;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_poll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
+{
+    int                 ready, revents;
+    ngx_err_t           err;
+    ngx_uint_t          i, found;
+    ngx_event_t        *ev;
+    ngx_queue_t        *queue;
+    ngx_connection_t   *c;
+
+    /* NGX_TIMER_INFINITE == INFTIM */
+
+#if (NGX_DEBUG0)
+    if (cycle->log->log_level & NGX_LOG_DEBUG_ALL) {
+        for (i = 0; i < nevents; i++) {
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "poll: %ui: fd:%d ev:%04Xd",
+                           i, event_list[i].fd, event_list[i].events);
+        }
+    }
+#endif
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "poll timer: %M", timer);
+
+    ready = WSAPoll(event_list, (u_int) nevents, (int) timer);
+
+    err = (ready == -1) ? ngx_errno : 0;
+
+    if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) {
+        ngx_time_update();
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "poll ready %d of %ui", ready, nevents);
+
+    if (err) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, err, "WSAPoll() failed");
+        return NGX_ERROR;
+    }
+
+    if (ready == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "WSAPoll() returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    for (i = 0; i < nevents && ready; i++) {
+
+        revents = event_list[i].revents;
+
+#if 1
+        ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "poll: %ui: fd:%d ev:%04Xd rev:%04Xd",
+                       i, event_list[i].fd, event_list[i].events, revents);
+#else
+        if (revents) {
+            ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "poll: %ui: fd:%d ev:%04Xd rev:%04Xd",
+                           i, event_list[i].fd, event_list[i].events, revents);
+        }
+#endif
+
+        if (revents & POLLNVAL) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "poll() error fd:%d ev:%04Xd rev:%04Xd",
+                          event_list[i].fd, event_list[i].events, revents);
+        }
+
+        if (revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                          "strange poll() events fd:%d ev:%04Xd rev:%04Xd",
+                          event_list[i].fd, event_list[i].events, revents);
+        }
+
+        if (event_list[i].fd == (ngx_socket_t) -1) {
+            /*
+             * the disabled event, a workaround for our possible bug,
+             * see the comment below
+             */
+            continue;
+        }
+
+        c = event_index[i];
+
+        if (c->fd == (ngx_socket_t) -1) {
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "unexpected event");
+
+            /*
+             * it is certainly our fault and it should be investigated,
+             * in the meantime we disable this event to avoid a CPU spinning
+             */
+
+            if (i == nevents - 1) {
+                nevents--;
+            } else {
+                event_list[i].fd = (ngx_socket_t) -1;
+            }
+
+            continue;
+        }
+
+        if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
+
+            /*
+             * if the error events were returned, add POLLIN and POLLOUT
+             * to handle the events at least in one active handler
+             */
+
+            revents |= POLLIN|POLLOUT;
+        }
+
+        found = 0;
+
+        if ((revents & POLLIN) && c->read->active) {
+            found = 1;
+
+            ev = c->read;
+            ev->ready = 1;
+            ev->available = -1;
+
+            queue = ev->accept ? &ngx_posted_accept_events
+                               : &ngx_posted_events;
+
+            ngx_post_event(ev, queue);
+        }
+
+        if ((revents & POLLOUT) && c->write->active) {
+            found = 1;
+
+            ev = c->write;
+            ev->ready = 1;
+
+            ngx_post_event(ev, &ngx_posted_events);
+        }
+
+        if (found) {
+            ready--;
+            continue;
+        }
+    }
+
+    if (ready != 0) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "poll ready != events");
+    }
+
+    return NGX_OK;
+}
+
+
+static char *
+ngx_poll_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_event_conf_t  *ecf;
+
+    ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module);
+
+    if (ecf->use != ngx_poll_module.ctx_index) {
+        return NGX_CONF_OK;
+    }
+
+#if (NGX_LOAD_WSAPOLL)
+
+    if (!ngx_have_wsapoll) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                      "poll is not available on this platform");
+        return NGX_CONF_ERROR;
+    }
+
+#endif
+
+    return NGX_CONF_OK;
+}

nginx/src/event/modules/ngx_win32_select_module.c

@@ -0,0 +1,408 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+static ngx_int_t ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer);
+static void ngx_select_done(ngx_cycle_t *cycle);
+static ngx_int_t ngx_select_add_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_select_del_event(ngx_event_t *ev, ngx_int_t event,
+    ngx_uint_t flags);
+static ngx_int_t ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags);
+static void ngx_select_repair_fd_sets(ngx_cycle_t *cycle);
+static char *ngx_select_init_conf(ngx_cycle_t *cycle, void *conf);
+
+
+static fd_set         master_read_fd_set;
+static fd_set         master_write_fd_set;
+static fd_set         work_read_fd_set;
+static fd_set         work_write_fd_set;
+static fd_set         work_except_fd_set;
+
+static ngx_uint_t     max_read;
+static ngx_uint_t     max_write;
+static ngx_uint_t     nevents;
+
+static ngx_event_t  **event_index;
+
+
+static ngx_str_t           select_name = ngx_string("select");
+
+static ngx_event_module_t  ngx_select_module_ctx = {
+    &select_name,
+    NULL,                                  /* create configuration */
+    ngx_select_init_conf,                  /* init configuration */
+
+    {
+        ngx_select_add_event,              /* add an event */
+        ngx_select_del_event,              /* delete an event */
+        ngx_select_add_event,              /* enable an event */
+        ngx_select_del_event,              /* disable an event */
+        NULL,                              /* add an connection */
+        NULL,                              /* delete an connection */
+        NULL,                              /* trigger a notify */
+        ngx_select_process_events,         /* process the events */
+        ngx_select_init,                   /* init the events */
+        ngx_select_done                    /* done the events */
+    }
+
+};
+
+ngx_module_t  ngx_select_module = {
+    NGX_MODULE_V1,
+    &ngx_select_module_ctx,                /* module context */
+    NULL,                                  /* module directives */
+    NGX_EVENT_MODULE,                      /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_int_t
+ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer)
+{
+    ngx_event_t  **index;
+
+    if (event_index == NULL) {
+        FD_ZERO(&master_read_fd_set);
+        FD_ZERO(&master_write_fd_set);
+        nevents = 0;
+    }
+
+    if (ngx_process >= NGX_PROCESS_WORKER
+        || cycle->old_cycle == NULL
+        || cycle->old_cycle->connection_n < cycle->connection_n)
+    {
+        index = ngx_alloc(sizeof(ngx_event_t *) * 2 * cycle->connection_n,
+                          cycle->log);
+        if (index == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (event_index) {
+            ngx_memcpy(index, event_index, sizeof(ngx_event_t *) * nevents);
+            ngx_free(event_index);
+        }
+
+        event_index = index;
+    }
+
+    ngx_io = ngx_os_io;
+
+    ngx_event_actions = ngx_select_module_ctx.actions;
+
+    ngx_event_flags = NGX_USE_LEVEL_EVENT;
+
+    max_read = 0;
+    max_write = 0;
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_select_done(ngx_cycle_t *cycle)
+{
+    ngx_free(event_index);
+
+    event_index = NULL;
+}
+
+
+static ngx_int_t
+ngx_select_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "select add event fd:%d ev:%i", c->fd, event);
+
+    if (ev->index != NGX_INVALID_INDEX) {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "select event fd:%d ev:%i is already set", c->fd, event);
+        return NGX_OK;
+    }
+
+    if ((event == NGX_READ_EVENT && ev->write)
+        || (event == NGX_WRITE_EVENT && !ev->write))
+    {
+        ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                      "invalid select %s event fd:%d ev:%i",
+                      ev->write ? "write" : "read", c->fd, event);
+        return NGX_ERROR;
+    }
+
+    if ((event == NGX_READ_EVENT && max_read >= FD_SETSIZE)
+        || (event == NGX_WRITE_EVENT && max_write >= FD_SETSIZE))
+    {
+        ngx_log_error(NGX_LOG_ERR, ev->log, 0,
+                      "maximum number of descriptors "
+                      "supported by select() is %d", FD_SETSIZE);
+        return NGX_ERROR;
+    }
+
+    if (event == NGX_READ_EVENT) {
+        FD_SET(c->fd, &master_read_fd_set);
+        max_read++;
+
+    } else if (event == NGX_WRITE_EVENT) {
+        FD_SET(c->fd, &master_write_fd_set);
+        max_write++;
+    }
+
+    ev->active = 1;
+
+    event_index[nevents] = ev;
+    ev->index = nevents;
+    nevents++;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_select_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
+{
+    ngx_event_t       *e;
+    ngx_connection_t  *c;
+
+    c = ev->data;
+
+    ev->active = 0;
+
+    if (ev->index == NGX_INVALID_INDEX) {
+        return NGX_OK;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "select del event fd:%d ev:%i", c->fd, event);
+
+    if (event == NGX_READ_EVENT) {
+        FD_CLR(c->fd, &master_read_fd_set);
+        max_read--;
+
+    } else if (event == NGX_WRITE_EVENT) {
+        FD_CLR(c->fd, &master_write_fd_set);
+        max_write--;
+    }
+
+    if (ev->index < --nevents) {
+        e = event_index[nevents];
+        event_index[ev->index] = e;
+        e->index = ev->index;
+    }
+
+    ev->index = NGX_INVALID_INDEX;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
+    ngx_uint_t flags)
+{
+    int                ready, nready;
+    ngx_err_t          err;
+    ngx_uint_t         i, found;
+    ngx_event_t       *ev;
+    ngx_queue_t       *queue;
+    struct timeval     tv, *tp;
+    ngx_connection_t  *c;
+
+#if (NGX_DEBUG)
+    if (cycle->log->log_level & NGX_LOG_DEBUG_ALL) {
+        for (i = 0; i < nevents; i++) {
+            ev = event_index[i];
+            c = ev->data;
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "select event: fd:%d wr:%d", c->fd, ev->write);
+        }
+    }
+#endif
+
+    if (timer == NGX_TIMER_INFINITE) {
+        tp = NULL;
+
+    } else {
+        tv.tv_sec = (long) (timer / 1000);
+        tv.tv_usec = (long) ((timer % 1000) * 1000);
+        tp = &tv;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "select timer: %M", timer);
+
+    work_read_fd_set = master_read_fd_set;
+    work_write_fd_set = master_write_fd_set;
+    work_except_fd_set = master_write_fd_set;
+
+    if (max_read || max_write) {
+        ready = select(0, &work_read_fd_set, &work_write_fd_set,
+                       &work_except_fd_set, tp);
+
+    } else {
+
+        /*
+         * Winsock select() requires that at least one descriptor set must be
+         * be non-null, and any non-null descriptor set must contain at least
+         * one handle to a socket.  Otherwise select() returns WSAEINVAL.
+         */
+
+        ngx_msleep(timer);
+
+        ready = 0;
+    }
+
+    err = (ready == -1) ? ngx_socket_errno : 0;
+
+    if (flags & NGX_UPDATE_TIME) {
+        ngx_time_update();
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "select ready %d", ready);
+
+    if (err) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, err, "select() failed");
+
+        if (err == WSAENOTSOCK) {
+            ngx_select_repair_fd_sets(cycle);
+        }
+
+        return NGX_ERROR;
+    }
+
+    if (ready == 0) {
+        if (timer != NGX_TIMER_INFINITE) {
+            return NGX_OK;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "select() returned no events without timeout");
+        return NGX_ERROR;
+    }
+
+    nready = 0;
+
+    for (i = 0; i < nevents; i++) {
+        ev = event_index[i];
+        c = ev->data;
+        found = 0;
+
+        if (ev->write) {
+            if (FD_ISSET(c->fd, &work_write_fd_set)) {
+                found++;
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "select write %d", c->fd);
+            }
+
+            if (FD_ISSET(c->fd, &work_except_fd_set)) {
+                found++;
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "select except %d", c->fd);
+            }
+
+        } else {
+            if (FD_ISSET(c->fd, &work_read_fd_set)) {
+                found++;
+                ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                               "select read %d", c->fd);
+            }
+        }
+
+        if (found) {
+            ev->ready = 1;
+            ev->available = -1;
+
+            queue = ev->accept ? &ngx_posted_accept_events
+                               : &ngx_posted_events;
+
+            ngx_post_event(ev, queue);
+
+            nready += found;
+        }
+    }
+
+    if (ready != nready) {
+        ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
+                      "select ready != events: %d:%d", ready, nready);
+
+        ngx_select_repair_fd_sets(cycle);
+    }
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_select_repair_fd_sets(ngx_cycle_t *cycle)
+{
+    int           n;
+    u_int         i;
+    socklen_t     len;
+    ngx_err_t     err;
+    ngx_socket_t  s;
+
+    for (i = 0; i < master_read_fd_set.fd_count; i++) {
+
+        s = master_read_fd_set.fd_array[i];
+        len = sizeof(int);
+
+        if (getsockopt(s, SOL_SOCKET, SO_TYPE, (char *) &n, &len) == -1) {
+            err = ngx_socket_errno;
+
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, err,
+                          "invalid descriptor #%d in read fd_set", s);
+
+            FD_CLR(s, &master_read_fd_set);
+        }
+    }
+
+    for (i = 0; i < master_write_fd_set.fd_count; i++) {
+
+        s = master_write_fd_set.fd_array[i];
+        len = sizeof(int);
+
+        if (getsockopt(s, SOL_SOCKET, SO_TYPE, (char *) &n, &len) == -1) {
+            err = ngx_socket_errno;
+
+            ngx_log_error(NGX_LOG_ALERT, cycle->log, err,
+                          "invalid descriptor #%d in write fd_set", s);
+
+            FD_CLR(s, &master_write_fd_set);
+        }
+    }
+}
+
+
+static char *
+ngx_select_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+    ngx_event_conf_t  *ecf;
+
+    ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module);
+
+    if (ecf->use != ngx_select_module.ctx_index) {
+        return NGX_CONF_OK;
+    }
+
+    return NGX_CONF_OK;
+}

nginx/src/event/ngx_event.h

@@ -0,0 +1,533 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_H_INCLUDED_
+#define _NGX_EVENT_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+#define NGX_INVALID_INDEX  0xd0d0d0d0
+
+
+#if (NGX_HAVE_IOCP)
+
+typedef struct {
+    WSAOVERLAPPED    ovlp;
+    ngx_event_t     *event;
+    int              error;
+} ngx_event_ovlp_t;
+
+#endif
+
+
+struct ngx_event_s {
+    void            *data;
+
+    unsigned         write:1;
+
+    unsigned         accept:1;
+
+    /* used to detect the stale events in kqueue and epoll */
+    unsigned         instance:1;
+
+    /*
+     * the event was passed or would be passed to a kernel;
+     * in aio mode - operation was posted.
+     */
+    unsigned         active:1;
+
+    unsigned         disabled:1;
+
+    /* the ready event; in aio mode 0 means that no operation can be posted */
+    unsigned         ready:1;
+
+    unsigned         oneshot:1;
+
+    /* aio operation is complete */
+    unsigned         complete:1;
+
+    unsigned         eof:1;
+    unsigned         error:1;
+
+    unsigned         timedout:1;
+    unsigned         timer_set:1;
+
+    unsigned         delayed:1;
+
+    unsigned         deferred_accept:1;
+
+    /* the pending eof reported by kqueue, epoll or in aio chain operation */
+    unsigned         pending_eof:1;
+
+    unsigned         posted:1;
+
+    unsigned         closed:1;
+
+    /* to test on worker exit */
+    unsigned         channel:1;
+    unsigned         resolver:1;
+
+    unsigned         cancelable:1;
+
+#if (NGX_HAVE_KQUEUE)
+    unsigned         kq_vnode:1;
+
+    /* the pending errno reported by kqueue */
+    int              kq_errno;
+#endif
+
+    /*
+     * kqueue only:
+     *   accept:     number of sockets that wait to be accepted
+     *   read:       bytes to read when event is ready
+     *               or lowat when event is set with NGX_LOWAT_EVENT flag
+     *   write:      available space in buffer when event is ready
+     *               or lowat when event is set with NGX_LOWAT_EVENT flag
+     *
+     * iocp: TODO
+     *
+     * otherwise:
+     *   accept:     1 if accept many, 0 otherwise
+     *   read:       bytes to read when event is ready, -1 if not known
+     */
+
+    int              available;
+
+    ngx_event_handler_pt  handler;
+
+
+#if (NGX_HAVE_IOCP)
+    ngx_event_ovlp_t ovlp;
+#endif
+
+    ngx_uint_t       index;
+
+    ngx_log_t       *log;
+
+    ngx_rbtree_node_t   timer;
+
+    /* the posted queue */
+    ngx_queue_t      queue;
+
+#if 0
+
+    /* the threads support */
+
+    /*
+     * the event thread context, we store it here
+     * if $(CC) does not understand __thread declaration
+     * and pthread_getspecific() is too costly
+     */
+
+    void            *thr_ctx;
+
+#if (NGX_EVENT_T_PADDING)
+
+    /* event should not cross cache line in SMP */
+
+    uint32_t         padding[NGX_EVENT_T_PADDING];
+#endif
+#endif
+};
+
+
+#if (NGX_HAVE_FILE_AIO)
+
+struct ngx_event_aio_s {
+    void                      *data;
+    ngx_event_handler_pt       handler;
+    ngx_file_t                *file;
+
+    ngx_fd_t                   fd;
+
+#if (NGX_HAVE_EVENTFD)
+    int64_t                    res;
+#endif
+
+#if !(NGX_HAVE_EVENTFD) || (NGX_TEST_BUILD_EPOLL)
+    ngx_err_t                  err;
+    size_t                     nbytes;
+#endif
+
+    ngx_aiocb_t                aiocb;
+    ngx_event_t                event;
+};
+
+#endif
+
+
+typedef struct {
+    ngx_int_t  (*add)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
+    ngx_int_t  (*del)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
+
+    ngx_int_t  (*enable)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
+    ngx_int_t  (*disable)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags);
+
+    ngx_int_t  (*add_conn)(ngx_connection_t *c);
+    ngx_int_t  (*del_conn)(ngx_connection_t *c, ngx_uint_t flags);
+
+    ngx_int_t  (*notify)(ngx_event_handler_pt handler);
+
+    ngx_int_t  (*process_events)(ngx_cycle_t *cycle, ngx_msec_t timer,
+                                 ngx_uint_t flags);
+
+    ngx_int_t  (*init)(ngx_cycle_t *cycle, ngx_msec_t timer);
+    void       (*done)(ngx_cycle_t *cycle);
+} ngx_event_actions_t;
+
+
+extern ngx_event_actions_t   ngx_event_actions;
+#if (NGX_HAVE_EPOLLRDHUP)
+extern ngx_uint_t            ngx_use_epoll_rdhup;
+#endif
+
+
+/*
+ * The event filter requires to read/write the whole data:
+ * select, poll, /dev/poll, kqueue, epoll.
+ */
+#define NGX_USE_LEVEL_EVENT      0x00000001
+
+/*
+ * The event filter is deleted after a notification without an additional
+ * syscall: kqueue, epoll.
+ */
+#define NGX_USE_ONESHOT_EVENT    0x00000002
+
+/*
+ * The event filter notifies only the changes and an initial level:
+ * kqueue, epoll.
+ */
+#define NGX_USE_CLEAR_EVENT      0x00000004
+
+/*
+ * The event filter has kqueue features: the eof flag, errno,
+ * available data, etc.
+ */
+#define NGX_USE_KQUEUE_EVENT     0x00000008
+
+/*
+ * The event filter supports low water mark: kqueue's NOTE_LOWAT.
+ * kqueue in FreeBSD 4.1-4.2 has no NOTE_LOWAT so we need a separate flag.
+ */
+#define NGX_USE_LOWAT_EVENT      0x00000010
+
+/*
+ * The event filter requires to do i/o operation until EAGAIN: epoll.
+ */
+#define NGX_USE_GREEDY_EVENT     0x00000020
+
+/*
+ * The event filter is epoll.
+ */
+#define NGX_USE_EPOLL_EVENT      0x00000040
+
+/*
+ * Obsolete.
+ */
+#define NGX_USE_RTSIG_EVENT      0x00000080
+
+/*
+ * Obsolete.
+ */
+#define NGX_USE_AIO_EVENT        0x00000100
+
+/*
+ * Need to add socket or handle only once: i/o completion port.
+ */
+#define NGX_USE_IOCP_EVENT       0x00000200
+
+/*
+ * The event filter has no opaque data and requires file descriptors table:
+ * poll, /dev/poll.
+ */
+#define NGX_USE_FD_EVENT         0x00000400
+
+/*
+ * The event module handles periodic or absolute timer event by itself:
+ * kqueue in FreeBSD 4.4, NetBSD 2.0, and MacOSX 10.4, Solaris 10's event ports.
+ */
+#define NGX_USE_TIMER_EVENT      0x00000800
+
+/*
+ * All event filters on file descriptor are deleted after a notification:
+ * Solaris 10's event ports.
+ */
+#define NGX_USE_EVENTPORT_EVENT  0x00001000
+
+/*
+ * The event filter support vnode notifications: kqueue.
+ */
+#define NGX_USE_VNODE_EVENT      0x00002000
+
+
+/*
+ * The event filter is deleted just before the closing file.
+ * Has no meaning for select and poll.
+ * kqueue, epoll, eventport:         allows to avoid explicit delete,
+ *                                   because filter automatically is deleted
+ *                                   on file close,
+ *
+ * /dev/poll:                        we need to flush POLLREMOVE event
+ *                                   before closing file.
+ */
+#define NGX_CLOSE_EVENT    1
+
+/*
+ * disable temporarily event filter, this may avoid locks
+ * in kernel malloc()/free(): kqueue.
+ */
+#define NGX_DISABLE_EVENT  2
+
+/*
+ * event must be passed to kernel right now, do not wait until batch processing.
+ */
+#define NGX_FLUSH_EVENT    4
+
+
+/* these flags have a meaning only for kqueue */
+#define NGX_LOWAT_EVENT    0
+#define NGX_VNODE_EVENT    0
+
+
+#if (NGX_HAVE_EPOLL) && !(NGX_HAVE_EPOLLRDHUP)
+#define EPOLLRDHUP         0
+#endif
+
+
+#if (NGX_HAVE_KQUEUE)
+
+#define NGX_READ_EVENT     EVFILT_READ
+#define NGX_WRITE_EVENT    EVFILT_WRITE
+
+#undef  NGX_VNODE_EVENT
+#define NGX_VNODE_EVENT    EVFILT_VNODE
+
+/*
+ * NGX_CLOSE_EVENT, NGX_LOWAT_EVENT, and NGX_FLUSH_EVENT are the module flags
+ * and they must not go into a kernel so we need to choose the value
+ * that must not interfere with any existent and future kqueue flags.
+ * kqueue has such values - EV_FLAG1, EV_EOF, and EV_ERROR:
+ * they are reserved and cleared on a kernel entrance.
+ */
+#undef  NGX_CLOSE_EVENT
+#define NGX_CLOSE_EVENT    EV_EOF
+
+#undef  NGX_LOWAT_EVENT
+#define NGX_LOWAT_EVENT    EV_FLAG1
+
+#undef  NGX_FLUSH_EVENT
+#define NGX_FLUSH_EVENT    EV_ERROR
+
+#define NGX_LEVEL_EVENT    0
+#define NGX_ONESHOT_EVENT  EV_ONESHOT
+#define NGX_CLEAR_EVENT    EV_CLEAR
+
+#undef  NGX_DISABLE_EVENT
+#define NGX_DISABLE_EVENT  EV_DISABLE
+
+
+#elif (NGX_HAVE_DEVPOLL && !(NGX_TEST_BUILD_DEVPOLL)) \
+      || (NGX_HAVE_EVENTPORT && !(NGX_TEST_BUILD_EVENTPORT))
+
+#define NGX_READ_EVENT     POLLIN
+#define NGX_WRITE_EVENT    POLLOUT
+
+#define NGX_LEVEL_EVENT    0
+#define NGX_ONESHOT_EVENT  1
+
+
+#elif (NGX_HAVE_EPOLL) && !(NGX_TEST_BUILD_EPOLL)
+
+#define NGX_READ_EVENT     (EPOLLIN|EPOLLRDHUP)
+#define NGX_WRITE_EVENT    EPOLLOUT
+
+#define NGX_LEVEL_EVENT    0
+#define NGX_CLEAR_EVENT    EPOLLET
+#define NGX_ONESHOT_EVENT  0x70000000
+#if 0
+#define NGX_ONESHOT_EVENT  EPOLLONESHOT
+#endif
+
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+#define NGX_EXCLUSIVE_EVENT  EPOLLEXCLUSIVE
+#endif
+
+#elif (NGX_HAVE_POLL)
+
+#define NGX_READ_EVENT     POLLIN
+#define NGX_WRITE_EVENT    POLLOUT
+
+#define NGX_LEVEL_EVENT    0
+#define NGX_ONESHOT_EVENT  1
+
+
+#else /* select */
+
+#define NGX_READ_EVENT     0
+#define NGX_WRITE_EVENT    1
+
+#define NGX_LEVEL_EVENT    0
+#define NGX_ONESHOT_EVENT  1
+
+#endif /* NGX_HAVE_KQUEUE */
+
+
+#if (NGX_HAVE_IOCP)
+#define NGX_IOCP_ACCEPT      0
+#define NGX_IOCP_IO          1
+#define NGX_IOCP_CONNECT     2
+#endif
+
+
+#if (NGX_TEST_BUILD_EPOLL)
+#define NGX_EXCLUSIVE_EVENT  0
+#endif
+
+
+#ifndef NGX_CLEAR_EVENT
+#define NGX_CLEAR_EVENT    0    /* dummy declaration */
+#endif
+
+
+#define ngx_process_events   ngx_event_actions.process_events
+#define ngx_done_events      ngx_event_actions.done
+
+#define ngx_add_event        ngx_event_actions.add
+#define ngx_del_event        ngx_event_actions.del
+#define ngx_add_conn         ngx_event_actions.add_conn
+#define ngx_del_conn         ngx_event_actions.del_conn
+
+#define ngx_notify           ngx_event_actions.notify
+
+#define ngx_add_timer        ngx_event_add_timer
+#define ngx_del_timer        ngx_event_del_timer
+
+
+extern ngx_os_io_t  ngx_io;
+
+#define ngx_recv             ngx_io.recv
+#define ngx_recv_chain       ngx_io.recv_chain
+#define ngx_udp_recv         ngx_io.udp_recv
+#define ngx_send             ngx_io.send
+#define ngx_send_chain       ngx_io.send_chain
+#define ngx_udp_send         ngx_io.udp_send
+#define ngx_udp_send_chain   ngx_io.udp_send_chain
+
+
+#define NGX_EVENT_MODULE      0x544E5645  /* "EVNT" */
+#define NGX_EVENT_CONF        0x02000000
+
+
+typedef struct {
+    ngx_uint_t    connections;
+    ngx_uint_t    use;
+
+    ngx_flag_t    multi_accept;
+    ngx_flag_t    accept_mutex;
+
+    ngx_msec_t    accept_mutex_delay;
+
+    u_char       *name;
+
+#if (NGX_DEBUG)
+    ngx_array_t   debug_connection;
+#endif
+} ngx_event_conf_t;
+
+
+typedef struct {
+    ngx_str_t              *name;
+
+    void                 *(*create_conf)(ngx_cycle_t *cycle);
+    char                 *(*init_conf)(ngx_cycle_t *cycle, void *conf);
+
+    ngx_event_actions_t     actions;
+} ngx_event_module_t;
+
+
+extern ngx_atomic_t          *ngx_connection_counter;
+
+extern ngx_atomic_t          *ngx_accept_mutex_ptr;
+extern ngx_shmtx_t            ngx_accept_mutex;
+extern ngx_uint_t             ngx_use_accept_mutex;
+extern ngx_uint_t             ngx_accept_events;
+extern ngx_uint_t             ngx_accept_mutex_held;
+extern ngx_msec_t             ngx_accept_mutex_delay;
+extern ngx_int_t              ngx_accept_disabled;
+extern ngx_uint_t             ngx_use_exclusive_accept;
+
+
+#if (NGX_STAT_STUB)
+
+extern ngx_atomic_t  *ngx_stat_accepted;
+extern ngx_atomic_t  *ngx_stat_handled;
+extern ngx_atomic_t  *ngx_stat_requests;
+extern ngx_atomic_t  *ngx_stat_active;
+extern ngx_atomic_t  *ngx_stat_reading;
+extern ngx_atomic_t  *ngx_stat_writing;
+extern ngx_atomic_t  *ngx_stat_waiting;
+
+#endif
+
+
+#define NGX_UPDATE_TIME         1
+#define NGX_POST_EVENTS         2
+
+
+extern sig_atomic_t           ngx_event_timer_alarm;
+extern ngx_uint_t             ngx_event_flags;
+extern ngx_module_t           ngx_events_module;
+extern ngx_module_t           ngx_event_core_module;
+
+
+#define ngx_event_get_conf(conf_ctx, module)                                  \
+             (*(ngx_get_conf(conf_ctx, ngx_events_module))) [module.ctx_index]
+
+
+
+void ngx_event_accept(ngx_event_t *ev);
+ngx_int_t ngx_trylock_accept_mutex(ngx_cycle_t *cycle);
+ngx_int_t ngx_enable_accept_events(ngx_cycle_t *cycle);
+u_char *ngx_accept_log_error(ngx_log_t *log, u_char *buf, size_t len);
+#if (NGX_DEBUG)
+void ngx_debug_accepted_connection(ngx_event_conf_t *ecf, ngx_connection_t *c);
+#endif
+
+
+void ngx_process_events_and_timers(ngx_cycle_t *cycle);
+ngx_int_t ngx_handle_read_event(ngx_event_t *rev, ngx_uint_t flags);
+ngx_int_t ngx_handle_write_event(ngx_event_t *wev, size_t lowat);
+
+
+#if (NGX_WIN32)
+void ngx_event_acceptex(ngx_event_t *ev);
+ngx_int_t ngx_event_post_acceptex(ngx_listening_t *ls, ngx_uint_t n);
+u_char *ngx_acceptex_log_error(ngx_log_t *log, u_char *buf, size_t len);
+#endif
+
+
+ngx_int_t ngx_send_lowat(ngx_connection_t *c, size_t lowat);
+
+
+/* used in ngx_log_debugX() */
+#define ngx_event_ident(p)  ((ngx_connection_t *) (p))->fd
+
+
+#include <ngx_event_timer.h>
+#include <ngx_event_posted.h>
+#include <ngx_event_udp.h>
+
+#if (NGX_WIN32)
+#include <ngx_iocp_module.h>
+#endif
+
+
+#endif /* _NGX_EVENT_H_INCLUDED_ */

nginx/src/event/ngx_event_accept.c

@@ -0,0 +1,589 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+static ngx_int_t ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all);
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+static void ngx_reorder_accept_events(ngx_listening_t *ls);
+#endif
+static void ngx_close_accepted_connection(ngx_connection_t *c);
+
+
+void
+ngx_event_accept(ngx_event_t *ev)
+{
+    socklen_t          socklen;
+    ngx_err_t          err;
+    ngx_log_t         *log;
+    ngx_uint_t         level;
+    ngx_socket_t       s;
+    ngx_event_t       *rev, *wev;
+    ngx_sockaddr_t     sa;
+    ngx_listening_t   *ls;
+    ngx_connection_t  *c, *lc;
+    ngx_event_conf_t  *ecf;
+#if (NGX_HAVE_ACCEPT4)
+    static ngx_uint_t  use_accept4 = 1;
+#endif
+
+    if (ev->timedout) {
+        if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) {
+            return;
+        }
+
+        ev->timedout = 0;
+    }
+
+    ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
+
+    if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) {
+        ev->available = ecf->multi_accept;
+    }
+
+    lc = ev->data;
+    ls = lc->listening;
+    ev->ready = 0;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "accept on %V, ready: %d", &ls->addr_text, ev->available);
+
+    do {
+        socklen = sizeof(ngx_sockaddr_t);
+
+#if (NGX_HAVE_ACCEPT4)
+        if (use_accept4) {
+            s = accept4(lc->fd, &sa.sockaddr, &socklen, SOCK_NONBLOCK);
+        } else {
+            s = accept(lc->fd, &sa.sockaddr, &socklen);
+        }
+#else
+        s = accept(lc->fd, &sa.sockaddr, &socklen);
+#endif
+
+        if (s == (ngx_socket_t) -1) {
+            err = ngx_socket_errno;
+
+            if (err == NGX_EAGAIN) {
+                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err,
+                               "accept() not ready");
+                return;
+            }
+
+            level = NGX_LOG_ALERT;
+
+            if (err == NGX_ECONNABORTED) {
+                level = NGX_LOG_ERR;
+
+            } else if (err == NGX_EMFILE || err == NGX_ENFILE) {
+                level = NGX_LOG_CRIT;
+            }
+
+#if (NGX_HAVE_ACCEPT4)
+            ngx_log_error(level, ev->log, err,
+                          use_accept4 ? "accept4() failed" : "accept() failed");
+
+            if (use_accept4 && err == NGX_ENOSYS) {
+                use_accept4 = 0;
+                ngx_inherited_nonblocking = 0;
+                continue;
+            }
+#else
+            ngx_log_error(level, ev->log, err, "accept() failed");
+#endif
+
+            if (err == NGX_ECONNABORTED) {
+                if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
+                    ev->available--;
+                }
+
+                if (ev->available) {
+                    continue;
+                }
+            }
+
+            if (err == NGX_EMFILE || err == NGX_ENFILE) {
+                if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle, 1)
+                    != NGX_OK)
+                {
+                    return;
+                }
+
+                if (ngx_use_accept_mutex) {
+                    if (ngx_accept_mutex_held) {
+                        ngx_shmtx_unlock(&ngx_accept_mutex);
+                        ngx_accept_mutex_held = 0;
+                    }
+
+                    ngx_accept_disabled = 1;
+
+                } else {
+                    ngx_add_timer(ev, ecf->accept_mutex_delay);
+                }
+            }
+
+            return;
+        }
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1);
+#endif
+
+        ngx_accept_disabled = ngx_cycle->connection_n / 8
+                              - ngx_cycle->free_connection_n;
+
+        c = ngx_get_connection(s, ev->log);
+
+        if (c == NULL) {
+            if (ngx_close_socket(s) == -1) {
+                ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+                              ngx_close_socket_n " failed");
+            }
+
+            return;
+        }
+
+        c->type = SOCK_STREAM;
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_active, 1);
+#endif
+
+        c->pool = ngx_create_pool(ls->pool_size, ev->log);
+        if (c->pool == NULL) {
+            ngx_close_accepted_connection(c);
+            return;
+        }
+
+        if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
+            socklen = sizeof(ngx_sockaddr_t);
+        }
+
+        c->sockaddr = ngx_palloc(c->pool, socklen);
+        if (c->sockaddr == NULL) {
+            ngx_close_accepted_connection(c);
+            return;
+        }
+
+        ngx_memcpy(c->sockaddr, &sa, socklen);
+
+        log = ngx_palloc(c->pool, sizeof(ngx_log_t));
+        if (log == NULL) {
+            ngx_close_accepted_connection(c);
+            return;
+        }
+
+        /* set a blocking mode for iocp and non-blocking mode for others */
+
+        if (ngx_inherited_nonblocking) {
+            if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
+                if (ngx_blocking(s) == -1) {
+                    ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+                                  ngx_blocking_n " failed");
+                    ngx_close_accepted_connection(c);
+                    return;
+                }
+            }
+
+        } else {
+            if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) {
+                if (ngx_nonblocking(s) == -1) {
+                    ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+                                  ngx_nonblocking_n " failed");
+                    ngx_close_accepted_connection(c);
+                    return;
+                }
+            }
+        }
+
+#if (NGX_HAVE_KEEPALIVE_TUNABLE && NGX_DARWIN)
+
+        /* Darwin doesn't inherit TCP_KEEPALIVE from a listening socket */
+
+        if (ls->keepidle) {
+            if (setsockopt(s, IPPROTO_TCP, TCP_KEEPALIVE,
+                           (const void *) &ls->keepidle, sizeof(int))
+                == -1)
+            {
+                ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno,
+                              "setsockopt(TCP_KEEPALIVE, %d) failed, ignored",
+                              ls->keepidle);
+            }
+        }
+
+#endif
+
+        *log = ls->log;
+
+        c->recv = ngx_recv;
+        c->send = ngx_send;
+        c->recv_chain = ngx_recv_chain;
+        c->send_chain = ngx_send_chain;
+
+        c->log = log;
+        c->pool->log = log;
+
+        c->socklen = socklen;
+        c->listening = ls;
+        c->local_sockaddr = ls->sockaddr;
+        c->local_socklen = ls->socklen;
+
+#if (NGX_HAVE_UNIX_DOMAIN)
+        if (c->sockaddr->sa_family == AF_UNIX) {
+            c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
+            c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
+#if (NGX_SOLARIS)
+            /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */
+            c->sendfile = 0;
+#endif
+        }
+#endif
+
+        rev = c->read;
+        wev = c->write;
+
+        wev->ready = 1;
+
+        if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
+            rev->ready = 1;
+        }
+
+        if (ev->deferred_accept) {
+            rev->ready = 1;
+#if (NGX_HAVE_KQUEUE || NGX_HAVE_EPOLLRDHUP)
+            rev->available = 1;
+#endif
+        }
+
+        rev->log = log;
+        wev->log = log;
+
+        /*
+         * TODO: MT: - ngx_atomic_fetch_add()
+         *             or protection by critical section or light mutex
+         *
+         * TODO: MP: - allocated in a shared memory
+         *           - ngx_atomic_fetch_add()
+         *             or protection by critical section or light mutex
+         */
+
+        c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
+
+        c->start_time = ngx_current_msec;
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_handled, 1);
+#endif
+
+        if (ls->addr_ntop) {
+            c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
+            if (c->addr_text.data == NULL) {
+                ngx_close_accepted_connection(c);
+                return;
+            }
+
+            c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
+                                             c->addr_text.data,
+                                             ls->addr_text_max_len, 0);
+            if (c->addr_text.len == 0) {
+                ngx_close_accepted_connection(c);
+                return;
+            }
+        }
+
+#if (NGX_DEBUG)
+        {
+        ngx_str_t  addr;
+        u_char     text[NGX_SOCKADDR_STRLEN];
+
+        ngx_debug_accepted_connection(ecf, c);
+
+        if (log->log_level & NGX_LOG_DEBUG_EVENT) {
+            addr.data = text;
+            addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text,
+                                     NGX_SOCKADDR_STRLEN, 1);
+
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0,
+                           "*%uA accept: %V fd:%d", c->number, &addr, s);
+        }
+
+        }
+#endif
+
+        if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) {
+            if (ngx_add_conn(c) == NGX_ERROR) {
+                ngx_close_accepted_connection(c);
+                return;
+            }
+        }
+
+        log->data = NULL;
+        log->handler = NULL;
+
+        ls->handler(c);
+
+        if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
+            ev->available--;
+        }
+
+    } while (ev->available);
+
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+    ngx_reorder_accept_events(ls);
+#endif
+}
+
+
+ngx_int_t
+ngx_trylock_accept_mutex(ngx_cycle_t *cycle)
+{
+    if (ngx_shmtx_trylock(&ngx_accept_mutex)) {
+
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                       "accept mutex locked");
+
+        if (ngx_accept_mutex_held && ngx_accept_events == 0) {
+            return NGX_OK;
+        }
+
+        if (ngx_enable_accept_events(cycle) == NGX_ERROR) {
+            ngx_shmtx_unlock(&ngx_accept_mutex);
+            return NGX_ERROR;
+        }
+
+        ngx_accept_events = 0;
+        ngx_accept_mutex_held = 1;
+
+        return NGX_OK;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "accept mutex lock failed: %ui", ngx_accept_mutex_held);
+
+    if (ngx_accept_mutex_held) {
+        if (ngx_disable_accept_events(cycle, 0) == NGX_ERROR) {
+            return NGX_ERROR;
+        }
+
+        ngx_accept_mutex_held = 0;
+    }
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_enable_accept_events(ngx_cycle_t *cycle)
+{
+    ngx_uint_t         i;
+    ngx_listening_t   *ls;
+    ngx_connection_t  *c;
+
+    ls = cycle->listening.elts;
+    for (i = 0; i < cycle->listening.nelts; i++) {
+
+        c = ls[i].connection;
+
+        if (c == NULL || c->read->active) {
+            continue;
+        }
+
+        if (ngx_add_event(c->read, NGX_READ_EVENT, 0) == NGX_ERROR) {
+            return NGX_ERROR;
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all)
+{
+    ngx_uint_t         i;
+    ngx_listening_t   *ls;
+    ngx_connection_t  *c;
+
+    ls = cycle->listening.elts;
+    for (i = 0; i < cycle->listening.nelts; i++) {
+
+        c = ls[i].connection;
+
+        if (c == NULL || !c->read->active) {
+            continue;
+        }
+
+#if (NGX_HAVE_REUSEPORT)
+
+        /*
+         * do not disable accept on worker's own sockets
+         * when disabling accept events due to accept mutex
+         */
+
+        if (ls[i].reuseport && !all) {
+            continue;
+        }
+
+#endif
+
+        if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT)
+            == NGX_ERROR)
+        {
+            return NGX_ERROR;
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+#if (NGX_HAVE_EPOLLEXCLUSIVE)
+
+static void
+ngx_reorder_accept_events(ngx_listening_t *ls)
+{
+    ngx_connection_t  *c;
+
+    /*
+     * Linux with EPOLLEXCLUSIVE usually notifies only the process which
+     * was first to add the listening socket to the epoll instance.  As
+     * a result most of the connections are handled by the first worker
+     * process.  To fix this, we re-add the socket periodically, so other
+     * workers will get a chance to accept connections.
+     */
+
+    if (!ngx_use_exclusive_accept) {
+        return;
+    }
+
+#if (NGX_HAVE_REUSEPORT)
+
+    if (ls->reuseport) {
+        return;
+    }
+
+#endif
+
+    c = ls->connection;
+
+    if (c->requests++ % 16 != 0
+        && ngx_accept_disabled <= 0)
+    {
+        return;
+    }
+
+    if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT)
+        == NGX_ERROR)
+    {
+        return;
+    }
+
+    if (ngx_add_event(c->read, NGX_READ_EVENT, NGX_EXCLUSIVE_EVENT)
+        == NGX_ERROR)
+    {
+        return;
+    }
+}
+
+#endif
+
+
+static void
+ngx_close_accepted_connection(ngx_connection_t *c)
+{
+    ngx_socket_t  fd;
+
+    ngx_free_connection(c);
+
+    fd = c->fd;
+    c->fd = (ngx_socket_t) -1;
+
+    if (ngx_close_socket(fd) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_socket_errno,
+                      ngx_close_socket_n " failed");
+    }
+
+    if (c->pool) {
+        ngx_destroy_pool(c->pool);
+    }
+
+#if (NGX_STAT_STUB)
+    (void) ngx_atomic_fetch_add(ngx_stat_active, -1);
+#endif
+}
+
+
+u_char *
+ngx_accept_log_error(ngx_log_t *log, u_char *buf, size_t len)
+{
+    return ngx_snprintf(buf, len, " while accepting new connection on %V",
+                        log->data);
+}
+
+
+#if (NGX_DEBUG)
+
+void
+ngx_debug_accepted_connection(ngx_event_conf_t *ecf, ngx_connection_t *c)
+{
+    struct sockaddr_in   *sin;
+    ngx_cidr_t           *cidr;
+    ngx_uint_t            i;
+#if (NGX_HAVE_INET6)
+    struct sockaddr_in6  *sin6;
+    ngx_uint_t            n;
+#endif
+
+    cidr = ecf->debug_connection.elts;
+    for (i = 0; i < ecf->debug_connection.nelts; i++) {
+        if (cidr[i].family != (ngx_uint_t) c->sockaddr->sa_family) {
+            goto next;
+        }
+
+        switch (cidr[i].family) {
+
+#if (NGX_HAVE_INET6)
+        case AF_INET6:
+            sin6 = (struct sockaddr_in6 *) c->sockaddr;
+            for (n = 0; n < 16; n++) {
+                if ((sin6->sin6_addr.s6_addr[n]
+                    & cidr[i].u.in6.mask.s6_addr[n])
+                    != cidr[i].u.in6.addr.s6_addr[n])
+                {
+                    goto next;
+                }
+            }
+            break;
+#endif
+
+#if (NGX_HAVE_UNIX_DOMAIN)
+        case AF_UNIX:
+            break;
+#endif
+
+        default: /* AF_INET */
+            sin = (struct sockaddr_in *) c->sockaddr;
+            if ((sin->sin_addr.s_addr & cidr[i].u.in.mask)
+                != cidr[i].u.in.addr)
+            {
+                goto next;
+            }
+            break;
+        }
+
+        c->log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL;
+        break;
+
+    next:
+        continue;
+    }
+}
+
+#endif

nginx/src/event/ngx_event_acceptex.c

@@ -0,0 +1,229 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+static void ngx_close_posted_connection(ngx_connection_t *c);
+
+
+void
+ngx_event_acceptex(ngx_event_t *rev)
+{
+    ngx_listening_t   *ls;
+    ngx_connection_t  *c;
+
+    c = rev->data;
+    ls = c->listening;
+
+    c->log->handler = ngx_accept_log_error;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "AcceptEx: %d", c->fd);
+
+    if (rev->ovlp.error) {
+        ngx_log_error(NGX_LOG_CRIT, c->log, rev->ovlp.error,
+                      "AcceptEx() %V failed", &ls->addr_text);
+        return;
+    }
+
+    /* SO_UPDATE_ACCEPT_CONTEXT is required for shutdown() to work */
+
+    if (setsockopt(c->fd, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT,
+                   (char *) &ls->fd, sizeof(ngx_socket_t))
+        == -1)
+    {
+        ngx_log_error(NGX_LOG_CRIT, c->log, ngx_socket_errno,
+                      "setsockopt(SO_UPDATE_ACCEPT_CONTEXT) failed for %V",
+                      &c->addr_text);
+        /* TODO: close socket */
+        return;
+    }
+
+    ngx_getacceptexsockaddrs(c->buffer->pos,
+                             ls->post_accept_buffer_size,
+                             ls->socklen + 16,
+                             ls->socklen + 16,
+                             &c->local_sockaddr, &c->local_socklen,
+                             &c->sockaddr, &c->socklen);
+
+    if (ls->post_accept_buffer_size) {
+        c->buffer->last += rev->available;
+        c->buffer->end = c->buffer->start + ls->post_accept_buffer_size;
+
+    } else {
+        c->buffer = NULL;
+    }
+
+    if (ls->addr_ntop) {
+        c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
+        if (c->addr_text.data == NULL) {
+            /* TODO: close socket */
+            return;
+        }
+
+        c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
+                                         c->addr_text.data,
+                                         ls->addr_text_max_len, 0);
+        if (c->addr_text.len == 0) {
+            /* TODO: close socket */
+            return;
+        }
+    }
+
+    ngx_event_post_acceptex(ls, 1);
+
+    c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
+
+    c->start_time = ngx_current_msec;
+
+    ls->handler(c);
+
+    return;
+
+}
+
+
+ngx_int_t
+ngx_event_post_acceptex(ngx_listening_t *ls, ngx_uint_t n)
+{
+    u_long             rcvd;
+    ngx_err_t          err;
+    ngx_log_t         *log;
+    ngx_uint_t         i;
+    ngx_event_t       *rev, *wev;
+    ngx_socket_t       s;
+    ngx_connection_t  *c;
+
+    for (i = 0; i < n; i++) {
+
+        /* TODO: look up reused sockets */
+
+        s = ngx_socket(ls->sockaddr->sa_family, ls->type, 0);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, &ls->log, 0,
+                       ngx_socket_n " s:%d", s);
+
+        if (s == (ngx_socket_t) -1) {
+            ngx_log_error(NGX_LOG_ALERT, &ls->log, ngx_socket_errno,
+                          ngx_socket_n " failed");
+
+            return NGX_ERROR;
+        }
+
+        c = ngx_get_connection(s, &ls->log);
+
+        if (c == NULL) {
+            return NGX_ERROR;
+        }
+
+        c->pool = ngx_create_pool(ls->pool_size, &ls->log);
+        if (c->pool == NULL) {
+            ngx_close_posted_connection(c);
+            return NGX_ERROR;
+        }
+
+        log = ngx_palloc(c->pool, sizeof(ngx_log_t));
+        if (log == NULL) {
+            ngx_close_posted_connection(c);
+            return NGX_ERROR;
+        }
+
+        c->buffer = ngx_create_temp_buf(c->pool, ls->post_accept_buffer_size
+                                                 + 2 * (ls->socklen + 16));
+        if (c->buffer == NULL) {
+            ngx_close_posted_connection(c);
+            return NGX_ERROR;
+        }
+
+        c->local_sockaddr = ngx_palloc(c->pool, ls->socklen);
+        if (c->local_sockaddr == NULL) {
+            ngx_close_posted_connection(c);
+            return NGX_ERROR;
+        }
+
+        c->sockaddr = ngx_palloc(c->pool, ls->socklen);
+        if (c->sockaddr == NULL) {
+            ngx_close_posted_connection(c);
+            return NGX_ERROR;
+        }
+
+        *log = ls->log;
+        c->log = log;
+
+        c->recv = ngx_recv;
+        c->send = ngx_send;
+        c->recv_chain = ngx_recv_chain;
+        c->send_chain = ngx_send_chain;
+
+        c->listening = ls;
+
+        rev = c->read;
+        wev = c->write;
+
+        rev->ovlp.event = rev;
+        wev->ovlp.event = wev;
+        rev->handler = ngx_event_acceptex;
+
+        rev->ready = 1;
+        wev->ready = 1;
+
+        rev->log = c->log;
+        wev->log = c->log;
+
+        if (ngx_add_event(rev, 0, NGX_IOCP_IO) == NGX_ERROR) {
+            ngx_close_posted_connection(c);
+            return NGX_ERROR;
+        }
+
+        if (ngx_acceptex(ls->fd, s, c->buffer->pos, ls->post_accept_buffer_size,
+                         ls->socklen + 16, ls->socklen + 16,
+                         &rcvd, (LPOVERLAPPED) &rev->ovlp)
+            == 0)
+        {
+            err = ngx_socket_errno;
+            if (err != WSA_IO_PENDING) {
+                ngx_log_error(NGX_LOG_ALERT, &ls->log, err,
+                              "AcceptEx() %V failed", &ls->addr_text);
+
+                ngx_close_posted_connection(c);
+                return NGX_ERROR;
+            }
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_close_posted_connection(ngx_connection_t *c)
+{
+    ngx_socket_t  fd;
+
+    ngx_free_connection(c);
+
+    fd = c->fd;
+    c->fd = (ngx_socket_t) -1;
+
+    if (ngx_close_socket(fd) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_socket_errno,
+                      ngx_close_socket_n " failed");
+    }
+
+    if (c->pool) {
+        ngx_destroy_pool(c->pool);
+    }
+}
+
+
+u_char *
+ngx_acceptex_log_error(ngx_log_t *log, u_char *buf, size_t len)
+{
+    return ngx_snprintf(buf, len, " while posting AcceptEx() on %V", log->data);
+}

nginx/src/event/ngx_event_connect.c

@@ -0,0 +1,435 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_connect.h>
+
+
+#if (NGX_HAVE_TRANSPARENT_PROXY)
+static ngx_int_t ngx_event_connect_set_transparent(ngx_peer_connection_t *pc,
+    ngx_socket_t s);
+#endif
+
+
+ngx_int_t
+ngx_event_connect_peer(ngx_peer_connection_t *pc)
+{
+    int                rc, type, value;
+#if (NGX_HAVE_IP_BIND_ADDRESS_NO_PORT || NGX_LINUX)
+    in_port_t          port;
+#endif
+    ngx_int_t          event;
+    ngx_err_t          err;
+    ngx_uint_t         level;
+    ngx_socket_t       s;
+    ngx_event_t       *rev, *wev;
+    ngx_connection_t  *c;
+
+    rc = pc->get(pc, pc->data);
+    if (rc != NGX_OK) {
+        return rc;
+    }
+
+    type = (pc->type ? pc->type : SOCK_STREAM);
+
+    s = ngx_socket(pc->sockaddr->sa_family, type, 0);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, pc->log, 0, "%s socket %d",
+                   (type == SOCK_STREAM) ? "stream" : "dgram", s);
+
+    if (s == (ngx_socket_t) -1) {
+        ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                      ngx_socket_n " failed");
+        return NGX_ERROR;
+    }
+
+
+    c = ngx_get_connection(s, pc->log);
+
+    if (c == NULL) {
+        if (ngx_close_socket(s) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          ngx_close_socket_n " failed");
+        }
+
+        return NGX_ERROR;
+    }
+
+    c->type = type;
+
+    if (pc->rcvbuf) {
+        if (setsockopt(s, SOL_SOCKET, SO_RCVBUF,
+                       (const void *) &pc->rcvbuf, sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(SO_RCVBUF) failed");
+            goto failed;
+        }
+    }
+
+    if (pc->so_keepalive) {
+        value = 1;
+
+        if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE,
+                       (const void *) &value, sizeof(int))
+            == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(SO_KEEPALIVE) failed, ignored");
+        }
+    }
+
+    if (ngx_nonblocking(s) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                      ngx_nonblocking_n " failed");
+
+        goto failed;
+    }
+
+    if (pc->local) {
+
+#if (NGX_HAVE_TRANSPARENT_PROXY)
+        if (pc->transparent) {
+            if (ngx_event_connect_set_transparent(pc, s) != NGX_OK) {
+                goto failed;
+            }
+        }
+#endif
+
+#if (NGX_HAVE_IP_BIND_ADDRESS_NO_PORT || NGX_LINUX)
+        port = ngx_inet_get_port(pc->local->sockaddr);
+#endif
+
+#if (NGX_HAVE_IP_BIND_ADDRESS_NO_PORT)
+
+        if (pc->sockaddr->sa_family != AF_UNIX && port == 0) {
+            static int  bind_address_no_port = 1;
+
+            if (bind_address_no_port) {
+                if (setsockopt(s, IPPROTO_IP, IP_BIND_ADDRESS_NO_PORT,
+                               (const void *) &bind_address_no_port,
+                               sizeof(int)) == -1)
+                {
+                    err = ngx_socket_errno;
+
+                    if (err != NGX_EOPNOTSUPP && err != NGX_ENOPROTOOPT) {
+                        ngx_log_error(NGX_LOG_ALERT, pc->log, err,
+                                      "setsockopt(IP_BIND_ADDRESS_NO_PORT) "
+                                      "failed, ignored");
+
+                    } else {
+                        bind_address_no_port = 0;
+                    }
+                }
+            }
+        }
+
+#endif
+
+#if (NGX_LINUX)
+
+        if (pc->type == SOCK_DGRAM && port != 0) {
+            int  reuse_addr = 1;
+
+            if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
+                           (const void *) &reuse_addr, sizeof(int))
+                 == -1)
+            {
+                ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                              "setsockopt(SO_REUSEADDR) failed");
+                goto failed;
+            }
+        }
+
+#endif
+
+        if (bind(s, pc->local->sockaddr, pc->local->socklen) == -1) {
+            ngx_log_error(NGX_LOG_CRIT, pc->log, ngx_socket_errno,
+                          "bind(%V) failed", &pc->local->name);
+
+            goto failed;
+        }
+    }
+
+    if (type == SOCK_STREAM) {
+        c->recv = ngx_recv;
+        c->send = ngx_send;
+        c->recv_chain = ngx_recv_chain;
+        c->send_chain = ngx_send_chain;
+
+        c->sendfile = 1;
+
+        if (pc->sockaddr->sa_family == AF_UNIX) {
+            c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
+            c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
+
+#if (NGX_SOLARIS)
+            /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */
+            c->sendfile = 0;
+#endif
+        }
+
+    } else { /* type == SOCK_DGRAM */
+        c->recv = ngx_udp_recv;
+        c->send = ngx_send;
+        c->send_chain = ngx_udp_send_chain;
+
+        c->need_flush_buf = 1;
+    }
+
+    c->log_error = pc->log_error;
+
+    rev = c->read;
+    wev = c->write;
+
+    rev->log = pc->log;
+    wev->log = pc->log;
+
+    pc->connection = c;
+
+    c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
+
+    c->start_time = ngx_current_msec;
+
+    if (ngx_add_conn) {
+        if (ngx_add_conn(c) == NGX_ERROR) {
+            goto failed;
+        }
+    }
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, pc->log, 0,
+                   "connect to %V, fd:%d #%uA", pc->name, s, c->number);
+
+    rc = connect(s, pc->sockaddr, pc->socklen);
+
+    if (rc == -1) {
+        err = ngx_socket_errno;
+
+
+        if (err != NGX_EINPROGRESS
+#if (NGX_WIN32)
+            /* Winsock returns WSAEWOULDBLOCK (NGX_EAGAIN) */
+            && err != NGX_EAGAIN
+#endif
+            )
+        {
+            if (err == NGX_ECONNREFUSED
+#if (NGX_LINUX)
+                /*
+                 * Linux returns EAGAIN instead of ECONNREFUSED
+                 * for unix sockets if listen queue is full
+                 */
+                || err == NGX_EAGAIN
+#endif
+                || err == NGX_ECONNRESET
+                || err == NGX_ENETDOWN
+                || err == NGX_ENETUNREACH
+                || err == NGX_EHOSTDOWN
+                || err == NGX_EHOSTUNREACH)
+            {
+                level = NGX_LOG_ERR;
+
+            } else {
+                level = NGX_LOG_CRIT;
+            }
+
+            ngx_log_error(level, c->log, err, "connect() to %V failed",
+                          pc->name);
+
+            ngx_close_connection(c);
+            pc->connection = NULL;
+
+            return NGX_DECLINED;
+        }
+    }
+
+    if (ngx_add_conn) {
+        if (rc == -1) {
+
+            /* NGX_EINPROGRESS */
+
+            return NGX_AGAIN;
+        }
+
+        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected");
+
+        wev->ready = 1;
+
+        return NGX_OK;
+    }
+
+    if (ngx_event_flags & NGX_USE_IOCP_EVENT) {
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, ngx_socket_errno,
+                       "connect(): %d", rc);
+
+        if (ngx_blocking(s) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          ngx_blocking_n " failed");
+            goto failed;
+        }
+
+        /*
+         * FreeBSD's aio allows to post an operation on non-connected socket.
+         * NT does not support it.
+         *
+         * TODO: check in Win32, etc. As workaround we can use NGX_ONESHOT_EVENT
+         */
+
+        rev->ready = 1;
+        wev->ready = 1;
+
+        return NGX_OK;
+    }
+
+    if (ngx_event_flags & NGX_USE_CLEAR_EVENT) {
+
+        /* kqueue */
+
+        event = NGX_CLEAR_EVENT;
+
+    } else {
+
+        /* select, poll, /dev/poll */
+
+        event = NGX_LEVEL_EVENT;
+    }
+
+    if (ngx_add_event(rev, NGX_READ_EVENT, event) != NGX_OK) {
+        goto failed;
+    }
+
+    if (rc == -1) {
+
+        /* NGX_EINPROGRESS */
+
+        if (ngx_add_event(wev, NGX_WRITE_EVENT, event) != NGX_OK) {
+            goto failed;
+        }
+
+        return NGX_AGAIN;
+    }
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected");
+
+    wev->ready = 1;
+
+    return NGX_OK;
+
+failed:
+
+    ngx_close_connection(c);
+    pc->connection = NULL;
+
+    return NGX_ERROR;
+}
+
+
+#if (NGX_HAVE_TRANSPARENT_PROXY)
+
+static ngx_int_t
+ngx_event_connect_set_transparent(ngx_peer_connection_t *pc, ngx_socket_t s)
+{
+    int  value;
+
+    value = 1;
+
+#if defined(SO_BINDANY)
+
+    if (setsockopt(s, SOL_SOCKET, SO_BINDANY,
+                   (const void *) &value, sizeof(int)) == -1)
+    {
+        ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                      "setsockopt(SO_BINDANY) failed");
+        return NGX_ERROR;
+    }
+
+#else
+
+    switch (pc->local->sockaddr->sa_family) {
+
+    case AF_INET:
+
+#if defined(IP_TRANSPARENT)
+
+        if (setsockopt(s, IPPROTO_IP, IP_TRANSPARENT,
+                       (const void *) &value, sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(IP_TRANSPARENT) failed");
+            return NGX_ERROR;
+        }
+
+#elif defined(IP_BINDANY)
+
+        if (setsockopt(s, IPPROTO_IP, IP_BINDANY,
+                       (const void *) &value, sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(IP_BINDANY) failed");
+            return NGX_ERROR;
+        }
+
+#endif
+
+        break;
+
+#if (NGX_HAVE_INET6)
+
+    case AF_INET6:
+
+#if defined(IPV6_TRANSPARENT)
+
+        if (setsockopt(s, IPPROTO_IPV6, IPV6_TRANSPARENT,
+                       (const void *) &value, sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(IPV6_TRANSPARENT) failed");
+            return NGX_ERROR;
+        }
+
+#elif defined(IPV6_BINDANY)
+
+        if (setsockopt(s, IPPROTO_IPV6, IPV6_BINDANY,
+                       (const void *) &value, sizeof(int)) == -1)
+        {
+            ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
+                          "setsockopt(IPV6_BINDANY) failed");
+            return NGX_ERROR;
+        }
+
+#else
+
+        ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
+                      "could not enable transparent proxying for IPv6 "
+                      "on this platform");
+
+        return NGX_ERROR;
+
+#endif
+
+        break;
+
+#endif /* NGX_HAVE_INET6 */
+
+    }
+
+#endif /* SO_BINDANY */
+
+    return NGX_OK;
+}
+
+#endif
+
+
+ngx_int_t
+ngx_event_get_peer(ngx_peer_connection_t *pc, void *data)
+{
+    return NGX_OK;
+}

nginx/src/event/ngx_event_connect.h

@@ -0,0 +1,85 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_CONNECT_H_INCLUDED_
+#define _NGX_EVENT_CONNECT_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#define NGX_PEER_KEEPALIVE           1
+#define NGX_PEER_NEXT                2
+#define NGX_PEER_FAILED              4
+
+
+typedef struct ngx_peer_connection_s  ngx_peer_connection_t;
+
+typedef ngx_int_t (*ngx_event_get_peer_pt)(ngx_peer_connection_t *pc,
+    void *data);
+typedef void (*ngx_event_free_peer_pt)(ngx_peer_connection_t *pc, void *data,
+    ngx_uint_t state);
+typedef void (*ngx_event_notify_peer_pt)(ngx_peer_connection_t *pc,
+    void *data, ngx_uint_t type);
+typedef ngx_int_t (*ngx_event_set_peer_session_pt)(ngx_peer_connection_t *pc,
+    void *data);
+typedef void (*ngx_event_save_peer_session_pt)(ngx_peer_connection_t *pc,
+    void *data);
+
+
+struct ngx_peer_connection_s {
+    ngx_connection_t                *connection;
+
+    struct sockaddr                 *sockaddr;
+    socklen_t                        socklen;
+    ngx_str_t                       *name;
+
+    ngx_uint_t                       tries;
+    ngx_msec_t                       start_time;
+
+    ngx_event_get_peer_pt            get;
+    ngx_event_free_peer_pt           free;
+    ngx_event_notify_peer_pt         notify;
+    void                            *data;
+
+#if (NGX_SSL || NGX_COMPAT)
+    ngx_event_set_peer_session_pt    set_session;
+    ngx_event_save_peer_session_pt   save_session;
+#endif
+
+    ngx_addr_t                      *local;
+
+    int                              type;
+    int                              rcvbuf;
+
+    ngx_log_t                       *log;
+
+#if (NGX_HTTP_UPSTREAM_SID || NGX_COMPAT)
+    ngx_str_t                       *hint;
+    ngx_str_t                       *sid;
+#endif
+
+    unsigned                         cached:1;
+    unsigned                         transparent:1;
+    unsigned                         so_keepalive:1;
+    unsigned                         down:1;
+
+                                     /* ngx_connection_log_error_e */
+    unsigned                         log_error:2;
+
+    NGX_COMPAT_BEGIN(1)
+    NGX_COMPAT_END
+};
+
+
+ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc);
+ngx_int_t ngx_event_get_peer(ngx_peer_connection_t *pc, void *data);
+
+
+#endif /* _NGX_EVENT_CONNECT_H_INCLUDED_ */

nginx/src/event/ngx_event_connectex.c

@@ -0,0 +1,206 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#define NGX_MAX_PENDING_CONN  10
+
+
+static CRITICAL_SECTION  connect_lock;
+static int               nconnects;
+static ngx_connection_t  pending_connects[NGX_MAX_PENDING_CONN];
+
+static HANDLE            pending_connect_event;
+
+__declspec(thread) int                nevents = 0;
+__declspec(thread) WSAEVENT           events[WSA_MAXIMUM_WAIT_EVENTS + 1];
+__declspec(thread) ngx_connection_t  *conn[WSA_MAXIMUM_WAIT_EVENTS + 1];
+
+
+
+int ngx_iocp_wait_connect(ngx_connection_t *c)
+{
+    for ( ;; ) {
+        EnterCriticalSection(&connect_lock);
+
+        if (nconnects < NGX_MAX_PENDING_CONN) {
+            pending_connects[--nconnects] = c;
+            LeaveCriticalSection(&connect_lock);
+
+            if (SetEvent(pending_connect_event) == 0) {
+                ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                              "SetEvent() failed");
+                return NGX_ERROR;
+
+            break;
+        }
+
+        LeaveCriticalSection(&connect_lock);
+        ngx_log_error(NGX_LOG_NOTICE, c->log, 0,
+                      "max number of pending connect()s is %d",
+                      NGX_MAX_PENDING_CONN);
+        msleep(100);
+    }
+
+    if (!started) {
+        if (ngx_iocp_new_thread(1) == NGX_ERROR) {
+            return NGX_ERROR;
+        }
+        started = 1;
+    }
+
+    return NGX_OK;
+}
+
+
+int ngx_iocp_new_thread(int main)
+{
+    u_int  id;
+
+    if (main) {
+        pending_connect_event = CreateEvent(NULL, 0, 1, NULL);
+        if (pending_connect_event == INVALID_HANDLE_VALUE) {
+            ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                          "CreateThread() failed");
+            return NGX_ERROR;
+        }
+    }
+
+    if (CreateThread(NULL, 0, ngx_iocp_wait_events, main, 0, &id)
+                                                       == INVALID_HANDLE_VALUE)
+    {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      "CreateThread() failed");
+        return NGX_ERROR;
+    }
+
+    SetEvent(event) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      "SetEvent() failed");
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+int ngx_iocp_new_connect()
+{
+    EnterCriticalSection(&connect_lock);
+    c = pending_connects[--nconnects];
+    LeaveCriticalSection(&connect_lock);
+
+    conn[nevents] = c;
+
+    events[nevents] = WSACreateEvent();
+    if (events[nevents] == INVALID_HANDLE_VALUE) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_socket_errno,
+                      "WSACreateEvent() failed");
+        return NGX_ERROR;
+    }
+
+    if (WSAEventSelect(c->fd, events[nevents], FD_CONNECT) == -1)
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_socket_errno,
+                      "WSAEventSelect() failed");
+        return NGX_ERROR;
+    }
+
+    nevents++;
+
+    return NGX_OK;
+}
+
+
+void ngx_iocp_wait_events(int main)
+{
+    WSANETWORKEVENTS  ne;
+
+    nevents = 1;
+    events[0] = pending_connect_event;
+    conn[0] = NULL;
+
+    for ( ;; ) {
+        offset = (nevents == WSA_MAXIMUM_WAIT_EVENTS + 1) ? 1 : 0;
+        timeout = (nevents == 1 && !first) ? 60000 : INFINITE;
+
+        n = WSAWaitForMultipleEvents(nevents - offset, events[offset],
+                                     0, timeout, 0);
+        if (n == WAIT_FAILED) {
+            ngx_log_error(NGX_LOG_ALERT, log, ngx_socket_errno,
+                          "WSAWaitForMultipleEvents() failed");
+            continue;
+        }
+
+        if (n == WAIT_TIMEOUT) {
+            if (nevents == 2 && !main) {
+                ExitThread(0);
+            }
+
+            ngx_log_error(NGX_LOG_ALERT, log, 0,
+                          "WSAWaitForMultipleEvents() "
+                          "returned unexpected WAIT_TIMEOUT");
+            continue;
+        }
+
+        n -= WSA_WAIT_EVENT_0;
+
+        if (events[n] == NULL) {
+
+            /* the pending_connect_event */
+
+            if (nevents == WSA_MAXIMUM_WAIT_EVENTS) {
+                ngx_iocp_new_thread(0);
+            } else {
+                ngx_iocp_new_connect();
+            }
+
+            continue;
+        }
+
+        if (WSAEnumNetworkEvents(c[n].fd, events[n], &ne) == -1) {
+            ngx_log_error(NGX_LOG_ALERT, log, ngx_socket_errno,
+                          "WSAEnumNetworkEvents() failed");
+            continue;
+        }
+
+        if (ne.lNetworkEvents & FD_CONNECT) {
+            conn[n].write->ovlp.error = ne.iErrorCode[FD_CONNECT_BIT];
+
+            if (PostQueuedCompletionStatus(iocp, 0, NGX_IOCP_CONNECT,
+                                           &conn[n].write->ovlp) == 0)
+            {
+                ngx_log_error(NGX_LOG_ALERT, log, ngx_socket_errno,
+                              "PostQueuedCompletionStatus() failed");
+                continue;
+            }
+
+            if (n < nevents) {
+                conn[n] = conn[nevents];
+                events[n] = events[nevents];
+            }
+
+            nevents--;
+            continue;
+        }
+
+        if (ne.lNetworkEvents & FD_ACCEPT) {
+
+            /* CHECK ERROR ??? */
+
+            ngx_event_post_acceptex(conn[n].listening, 1);
+            continue;
+        }
+
+        ngx_log_error(NGX_LOG_ALERT, c[n].log, 0,
+                      "WSAWaitForMultipleEvents() "
+                      "returned unexpected network event %ul",
+                      ne.lNetworkEvents);
+    }
+}

nginx/src/event/ngx_event_openssl.h

@@ -0,0 +1,416 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_OPENSSL_H_INCLUDED_
+#define _NGX_EVENT_OPENSSL_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+#define OPENSSL_SUPPRESS_DEPRECATED
+
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <openssl/bn.h>
+#include <openssl/conf.h>
+#include <openssl/crypto.h>
+#ifndef OPENSSL_NO_DH
+#include <openssl/dh.h>
+#endif
+#ifndef OPENSSL_NO_ENGINE
+#include <openssl/engine.h>
+#endif
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#ifndef OPENSSL_NO_OCSP
+#include <openssl/ocsp.h>
+#endif
+#include <openssl/rand.h>
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+
+#define NGX_SSL_NAME     "OpenSSL"
+
+
+#if (defined LIBRESSL_VERSION_NUMBER && OPENSSL_VERSION_NUMBER == 0x20000000L)
+#undef OPENSSL_VERSION_NUMBER
+#if (LIBRESSL_VERSION_NUMBER >= 0x3050000fL)
+#define OPENSSL_VERSION_NUMBER  0x1010000fL
+#else
+#define OPENSSL_VERSION_NUMBER  0x1000107fL
+#endif
+#endif
+
+
+#if (OPENSSL_VERSION_NUMBER >= 0x10100001L)
+
+#define ngx_ssl_version()       OpenSSL_version(OPENSSL_VERSION)
+
+#else
+
+#define ngx_ssl_version()       SSLeay_version(SSLEAY_VERSION)
+
+#endif
+
+
+#define ngx_ssl_session_t       SSL_SESSION
+#define ngx_ssl_conn_t          SSL
+
+
+#if (OPENSSL_VERSION_NUMBER < 0x10002000L)
+#define SSL_is_server(s)        (s)->server
+#endif
+
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010000fL)
+#define ASN1_STRING_get0_data(x)     (x)->data
+#endif
+
+
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined SSL_get_peer_certificate)
+#define SSL_get_peer_certificate(s)  SSL_get1_peer_certificate(s)
+#endif
+
+
+#if (OPENSSL_VERSION_NUMBER < 0x30000000L && !defined ERR_peek_error_data)
+#define ERR_peek_error_data(d, f)    ERR_peek_error_line_data(NULL, NULL, d, f)
+#endif
+
+
+#ifdef OPENSSL_NO_DEPRECATED_3_4
+#define SSL_SESSION_get_time(s)      SSL_SESSION_get_time_ex(s)
+#define SSL_SESSION_set_time(s, t)   SSL_SESSION_set_time_ex(s, t)
+#endif
+
+
+#ifdef OPENSSL_NO_DEPRECATED_3_0
+#define EVP_CIPHER_CTX_cipher(c)     EVP_CIPHER_CTX_get0_cipher(c)
+#endif
+
+
+#if (OPENSSL_VERSION_NUMBER < 0x30000000L)
+#define SSL_group_to_name(s, nid)    NULL
+#endif
+
+
+typedef struct ngx_ssl_ocsp_s   ngx_ssl_ocsp_t;
+
+
+struct ngx_ssl_s {
+    SSL_CTX                    *ctx;
+    ngx_log_t                  *log;
+    size_t                      buffer_size;
+
+    ngx_array_t                 certs;
+
+    ngx_rbtree_t                staple_rbtree;
+    ngx_rbtree_node_t           staple_sentinel;
+};
+
+
+struct ngx_ssl_connection_s {
+    ngx_ssl_conn_t             *connection;
+    SSL_CTX                    *session_ctx;
+
+    ngx_int_t                   last;
+    ngx_buf_t                  *buf;
+    size_t                      buffer_size;
+
+    ngx_connection_handler_pt   handler;
+
+    ngx_ssl_session_t          *session;
+    ngx_connection_handler_pt   save_session;
+
+    ngx_event_handler_pt        saved_read_handler;
+    ngx_event_handler_pt        saved_write_handler;
+
+    ngx_ssl_ocsp_t             *ocsp;
+
+    u_char                      early_buf;
+
+    unsigned                    handshaked:1;
+    unsigned                    handshake_rejected:1;
+    unsigned                    renegotiation:1;
+    unsigned                    buffer:1;
+    unsigned                    sendfile:1;
+    unsigned                    no_wait_shutdown:1;
+    unsigned                    no_send_shutdown:1;
+    unsigned                    shutdown_without_free:1;
+    unsigned                    handshake_buffer_set:1;
+    unsigned                    session_timeout_set:1;
+    unsigned                    try_early_data:1;
+    unsigned                    in_early:1;
+    unsigned                    in_ocsp:1;
+    unsigned                    early_preread:1;
+    unsigned                    write_blocked:1;
+    unsigned                    sni_accepted:1;
+};
+
+
+#define NGX_SSL_NO_SCACHE            -2
+#define NGX_SSL_NONE_SCACHE          -3
+#define NGX_SSL_NO_BUILTIN_SCACHE    -4
+#define NGX_SSL_DFLT_BUILTIN_SCACHE  -5
+
+
+#define NGX_SSL_MAX_SESSION_SIZE  8192
+
+typedef struct ngx_ssl_sess_id_s  ngx_ssl_sess_id_t;
+
+struct ngx_ssl_sess_id_s {
+    ngx_rbtree_node_t           node;
+    size_t                      len;
+    ngx_queue_t                 queue;
+    time_t                      expire;
+    u_char                      id[32];
+#if (NGX_PTR_SIZE == 8)
+    u_char                     *session;
+#else
+    u_char                      session[1];
+#endif
+};
+
+
+typedef struct {
+    u_char                      name[16];
+    u_char                      hmac_key[32];
+    u_char                      aes_key[32];
+    time_t                      expire;
+    unsigned                    size:8;
+    unsigned                    shared:1;
+} ngx_ssl_ticket_key_t;
+
+
+typedef struct {
+    ngx_rbtree_t                session_rbtree;
+    ngx_rbtree_node_t           sentinel;
+    ngx_queue_t                 expire_queue;
+    ngx_ssl_ticket_key_t        ticket_keys[3];
+    time_t                      fail_time;
+} ngx_ssl_session_cache_t;
+
+
+typedef int (*ngx_ssl_servername_pt)(ngx_ssl_conn_t *, int *, void *);
+
+typedef struct {
+    ngx_ssl_servername_pt       servername;
+} ngx_ssl_client_hello_arg;
+
+
+#define NGX_SSL_SSLv2    0x0002
+#define NGX_SSL_SSLv3    0x0004
+#define NGX_SSL_TLSv1    0x0008
+#define NGX_SSL_TLSv1_1  0x0010
+#define NGX_SSL_TLSv1_2  0x0020
+#define NGX_SSL_TLSv1_3  0x0040
+
+
+#if (defined SSL_OP_NO_TLSv1_2 || defined SSL_OP_NO_TLSv1_3)
+#define NGX_SSL_DEFAULT_PROTOCOLS  (NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3)
+#else
+#define NGX_SSL_DEFAULT_PROTOCOLS  (NGX_SSL_TLSv1|NGX_SSL_TLSv1_1)
+#endif
+
+
+#define NGX_SSL_BUFFER   1
+#define NGX_SSL_CLIENT   2
+
+#define NGX_SSL_BUFSIZE  16384
+
+
+#define NGX_SSL_CACHE_CERT  0
+#define NGX_SSL_CACHE_PKEY  1
+#define NGX_SSL_CACHE_CRL   2
+#define NGX_SSL_CACHE_CA    3
+
+#define NGX_SSL_CACHE_INVALIDATE  0x80000000
+
+
+ngx_int_t ngx_ssl_init(ngx_log_t *log);
+ngx_int_t ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data);
+
+ngx_int_t ngx_ssl_certificates(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_array_t *certs, ngx_array_t *keys, ngx_array_t *passwords);
+ngx_int_t ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_str_t *cert, ngx_str_t *key, ngx_array_t *passwords);
+ngx_int_t ngx_ssl_connection_certificate(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *cert, ngx_str_t *key, ngx_ssl_cache_t *cache,
+    ngx_array_t *passwords);
+ngx_int_t ngx_ssl_certificate_compression(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_uint_t enable);
+
+ngx_int_t ngx_ssl_ciphers(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *ciphers,
+    ngx_uint_t prefer_server_ciphers);
+ngx_int_t ngx_ssl_client_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_str_t *cert, ngx_int_t depth);
+ngx_int_t ngx_ssl_trusted_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_str_t *cert, ngx_int_t depth);
+ngx_int_t ngx_ssl_crl(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *crl);
+ngx_int_t ngx_ssl_stapling(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_str_t *file, ngx_str_t *responder, ngx_uint_t verify);
+ngx_int_t ngx_ssl_stapling_resolver(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_resolver_t *resolver, ngx_msec_t resolver_timeout);
+ngx_int_t ngx_ssl_ocsp(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *responder,
+    ngx_uint_t depth, ngx_shm_zone_t *shm_zone);
+ngx_int_t ngx_ssl_ocsp_resolver(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_resolver_t *resolver, ngx_msec_t resolver_timeout);
+
+ngx_int_t ngx_ssl_ocsp_validate(ngx_connection_t *c);
+ngx_int_t ngx_ssl_ocsp_get_status(ngx_connection_t *c, const char **s);
+void ngx_ssl_ocsp_cleanup(ngx_connection_t *c);
+ngx_int_t ngx_ssl_ocsp_cache_init(ngx_shm_zone_t *shm_zone, void *data);
+
+ngx_ssl_cache_t *ngx_ssl_cache_init(ngx_pool_t *pool, ngx_uint_t max,
+    time_t valid, time_t inactive);
+void *ngx_ssl_cache_fetch(ngx_conf_t *cf, ngx_uint_t index, char **err,
+    ngx_str_t *path, void *data);
+void *ngx_ssl_cache_connection_fetch(ngx_ssl_cache_t *cache, ngx_pool_t *pool,
+    ngx_uint_t index, char **err, ngx_str_t *path, void *data);
+
+ngx_array_t *ngx_ssl_read_password_file(ngx_conf_t *cf, ngx_str_t *file);
+ngx_array_t *ngx_ssl_preserve_passwords(ngx_conf_t *cf,
+    ngx_array_t *passwords);
+ngx_int_t ngx_ssl_dhparam(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *file);
+ngx_int_t ngx_ssl_ech_files(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_array_t *filename);
+ngx_int_t ngx_ssl_ecdh_curve(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *name);
+ngx_int_t ngx_ssl_early_data(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_uint_t enable);
+ngx_int_t ngx_ssl_conf_commands(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_array_t *commands);
+
+ngx_int_t ngx_ssl_client_session_cache(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_uint_t enable);
+ngx_int_t ngx_ssl_session_cache(ngx_ssl_t *ssl, ngx_str_t *sess_ctx,
+    ngx_array_t *certificates, ssize_t builtin_session_cache,
+    ngx_shm_zone_t *shm_zone, time_t timeout);
+ngx_int_t ngx_ssl_session_ticket_keys(ngx_conf_t *cf, ngx_ssl_t *ssl,
+    ngx_array_t *paths);
+ngx_int_t ngx_ssl_session_cache_init(ngx_shm_zone_t *shm_zone, void *data);
+
+ngx_int_t ngx_ssl_set_client_hello_callback(ngx_ssl_t *ssl,
+    ngx_ssl_client_hello_arg *cb);
+#ifdef SSL_CLIENT_HELLO_SUCCESS
+int ngx_ssl_client_hello_callback(ngx_ssl_conn_t *ssl_conn, int *ad, void *arg);
+#elif defined OPENSSL_IS_BORINGSSL
+enum ssl_select_cert_result_t ngx_ssl_select_certificate(
+    const SSL_CLIENT_HELLO *client_hello);
+#endif
+
+ngx_int_t ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c,
+    ngx_uint_t flags);
+
+void ngx_ssl_remove_cached_session(SSL_CTX *ssl, ngx_ssl_session_t *sess);
+ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session);
+ngx_ssl_session_t *ngx_ssl_get_session(ngx_connection_t *c);
+ngx_ssl_session_t *ngx_ssl_get0_session(ngx_connection_t *c);
+#define ngx_ssl_free_session        SSL_SESSION_free
+#define ngx_ssl_get_connection(ssl_conn)                                      \
+    SSL_get_ex_data(ssl_conn, ngx_ssl_connection_index)
+#define ngx_ssl_get_server_conf(ssl_ctx)                                      \
+    SSL_CTX_get_ex_data(ssl_ctx, ngx_ssl_server_conf_index)
+
+#define ngx_ssl_verify_error_optional(n)                                      \
+    (n == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT                              \
+     || n == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN                             \
+     || n == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY                     \
+     || n == X509_V_ERR_CERT_UNTRUSTED                                        \
+     || n == X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE)
+
+ngx_int_t ngx_ssl_check_host(ngx_connection_t *c, ngx_str_t *name);
+
+
+ngx_int_t ngx_ssl_get_protocol(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_cipher_name(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_ciphers(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_curve(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_curves(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_sigalg(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_session_id(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_session_reused(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_early_data(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_server_name(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_ech_status(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_ech_outer_server_name(ngx_connection_t *c,
+    ngx_pool_t *pool, ngx_str_t *s);
+ngx_int_t ngx_ssl_get_alpn_protocol(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_raw_certificate(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_certificate(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_escaped_certificate(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_subject_dn(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_issuer_dn(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_subject_dn_legacy(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_issuer_dn_legacy(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_serial_number(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_fingerprint(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_client_verify(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_client_v_start(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_client_v_end(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_client_v_remain(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+ngx_int_t ngx_ssl_get_client_sigalg(ngx_connection_t *c, ngx_pool_t *pool,
+    ngx_str_t *s);
+
+
+ngx_int_t ngx_ssl_handshake(ngx_connection_t *c);
+#if (NGX_DEBUG)
+void ngx_ssl_handshake_log(ngx_connection_t *c);
+#endif
+ssize_t ngx_ssl_recv(ngx_connection_t *c, u_char *buf, size_t size);
+ssize_t ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size);
+ssize_t ngx_ssl_recv_chain(ngx_connection_t *c, ngx_chain_t *cl, off_t limit);
+ngx_chain_t *ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in,
+    off_t limit);
+void ngx_ssl_free_buffer(ngx_connection_t *c);
+ngx_int_t ngx_ssl_shutdown(ngx_connection_t *c);
+void ngx_ssl_connection_error(ngx_connection_t *c, int sslerr, ngx_err_t err,
+    char *text);
+void ngx_cdecl ngx_ssl_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err,
+    char *fmt, ...);
+void ngx_ssl_cleanup_ctx(void *data);
+
+
+extern int  ngx_ssl_connection_index;
+extern int  ngx_ssl_server_conf_index;
+extern int  ngx_ssl_session_cache_index;
+extern int  ngx_ssl_ticket_keys_index;
+extern int  ngx_ssl_ocsp_index;
+extern int  ngx_ssl_index;
+extern int  ngx_ssl_certificate_name_index;
+extern int  ngx_ssl_certificate_comp_index;
+extern int  ngx_ssl_client_hello_arg_index;
+
+
+extern u_char  ngx_ssl_session_buffer[NGX_SSL_MAX_SESSION_SIZE];
+
+
+#endif /* _NGX_EVENT_OPENSSL_H_INCLUDED_ */

nginx/src/event/ngx_event_pipe.h

@@ -0,0 +1,107 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_PIPE_H_INCLUDED_
+#define _NGX_EVENT_PIPE_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+typedef struct ngx_event_pipe_s  ngx_event_pipe_t;
+
+typedef ngx_int_t (*ngx_event_pipe_input_filter_pt)(ngx_event_pipe_t *p,
+                                                    ngx_buf_t *buf);
+typedef ngx_int_t (*ngx_event_pipe_output_filter_pt)(void *data,
+                                                     ngx_chain_t *chain);
+
+
+struct ngx_event_pipe_s {
+    ngx_connection_t  *upstream;
+    ngx_connection_t  *downstream;
+
+    ngx_chain_t       *free_raw_bufs;
+    ngx_chain_t       *in;
+    ngx_chain_t      **last_in;
+
+    ngx_chain_t       *writing;
+
+    ngx_chain_t       *out;
+    ngx_chain_t       *free;
+    ngx_chain_t       *busy;
+
+    /*
+     * the input filter i.e. that moves HTTP/1.1 chunks
+     * from the raw bufs to an incoming chain
+     */
+
+    ngx_event_pipe_input_filter_pt    input_filter;
+    void                             *input_ctx;
+
+    ngx_event_pipe_output_filter_pt   output_filter;
+    void                             *output_ctx;
+
+#if (NGX_THREADS || NGX_COMPAT)
+    ngx_int_t                       (*thread_handler)(ngx_thread_task_t *task,
+                                                      ngx_file_t *file);
+    void                             *thread_ctx;
+    ngx_thread_task_t                *thread_task;
+#endif
+
+    unsigned           read:1;
+    unsigned           cacheable:1;
+    unsigned           single_buf:1;
+    unsigned           free_bufs:1;
+    unsigned           upstream_done:1;
+    unsigned           upstream_error:1;
+    unsigned           upstream_eof:1;
+    unsigned           upstream_blocked:1;
+    unsigned           downstream_done:1;
+    unsigned           downstream_error:1;
+    unsigned           cyclic_temp_file:1;
+    unsigned           aio:1;
+
+    ngx_int_t          allocated;
+    ngx_bufs_t         bufs;
+    ngx_buf_tag_t      tag;
+
+    ssize_t            busy_size;
+
+    off_t              read_length;
+    off_t              length;
+
+    off_t              max_temp_file_size;
+    ssize_t            temp_file_write_size;
+
+    ngx_msec_t         read_timeout;
+    ngx_msec_t         send_timeout;
+    ssize_t            send_lowat;
+
+    ngx_pool_t        *pool;
+    ngx_log_t         *log;
+
+    ngx_chain_t       *preread_bufs;
+    size_t             preread_size;
+    ngx_buf_t         *buf_to_file;
+
+    size_t             limit_rate;
+    time_t             start_sec;
+
+    ngx_temp_file_t   *temp_file;
+
+    /* STUB */ int     num;
+};
+
+
+ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, ngx_int_t do_write);
+ngx_int_t ngx_event_pipe_copy_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf);
+ngx_int_t ngx_event_pipe_add_free_buf(ngx_event_pipe_t *p, ngx_buf_t *b);
+
+
+#endif /* _NGX_EVENT_PIPE_H_INCLUDED_ */

nginx/src/event/ngx_event_posted.c

@@ -0,0 +1,60 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+ngx_queue_t  ngx_posted_accept_events;
+ngx_queue_t  ngx_posted_next_events;
+ngx_queue_t  ngx_posted_events;
+
+
+void
+ngx_event_process_posted(ngx_cycle_t *cycle, ngx_queue_t *posted)
+{
+    ngx_queue_t  *q;
+    ngx_event_t  *ev;
+
+    while (!ngx_queue_empty(posted)) {
+
+        q = ngx_queue_head(posted);
+        ev = ngx_queue_data(q, ngx_event_t, queue);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                      "posted event %p", ev);
+
+        ngx_delete_posted_event(ev);
+
+        ev->handler(ev);
+    }
+}
+
+
+void
+ngx_event_move_posted_next(ngx_cycle_t *cycle)
+{
+    ngx_queue_t  *q;
+    ngx_event_t  *ev;
+
+    for (q = ngx_queue_head(&ngx_posted_next_events);
+         q != ngx_queue_sentinel(&ngx_posted_next_events);
+         q = ngx_queue_next(q))
+    {
+        ev = ngx_queue_data(q, ngx_event_t, queue);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                      "posted next event %p", ev);
+
+        ev->ready = 1;
+        ev->available = -1;
+    }
+
+    ngx_queue_add(&ngx_posted_events, &ngx_posted_next_events);
+    ngx_queue_init(&ngx_posted_next_events);
+}

nginx/src/event/ngx_event_posted.h

@@ -0,0 +1,50 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_POSTED_H_INCLUDED_
+#define _NGX_EVENT_POSTED_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#define ngx_post_event(ev, q)                                                 \
+                                                                              \
+    if (!(ev)->posted) {                                                      \
+        (ev)->posted = 1;                                                     \
+        ngx_queue_insert_tail(q, &(ev)->queue);                               \
+                                                                              \
+        ngx_log_debug1(NGX_LOG_DEBUG_CORE, (ev)->log, 0, "post event %p", ev);\
+                                                                              \
+    } else  {                                                                 \
+        ngx_log_debug1(NGX_LOG_DEBUG_CORE, (ev)->log, 0,                      \
+                       "update posted event %p", ev);                         \
+    }
+
+
+#define ngx_delete_posted_event(ev)                                           \
+                                                                              \
+    (ev)->posted = 0;                                                         \
+    ngx_queue_remove(&(ev)->queue);                                           \
+                                                                              \
+    ngx_log_debug1(NGX_LOG_DEBUG_CORE, (ev)->log, 0,                          \
+                   "delete posted event %p", ev);
+
+
+
+void ngx_event_process_posted(ngx_cycle_t *cycle, ngx_queue_t *posted);
+void ngx_event_move_posted_next(ngx_cycle_t *cycle);
+
+
+extern ngx_queue_t  ngx_posted_accept_events;
+extern ngx_queue_t  ngx_posted_next_events;
+extern ngx_queue_t  ngx_posted_events;
+
+
+#endif /* _NGX_EVENT_POSTED_H_INCLUDED_ */

nginx/src/event/ngx_event_timer.c

@@ -0,0 +1,126 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+ngx_rbtree_t              ngx_event_timer_rbtree;
+static ngx_rbtree_node_t  ngx_event_timer_sentinel;
+
+/*
+ * the event timer rbtree may contain the duplicate keys, however,
+ * it should not be a problem, because we use the rbtree to find
+ * a minimum timer value only
+ */
+
+ngx_int_t
+ngx_event_timer_init(ngx_log_t *log)
+{
+    ngx_rbtree_init(&ngx_event_timer_rbtree, &ngx_event_timer_sentinel,
+                    ngx_rbtree_insert_timer_value);
+
+    return NGX_OK;
+}
+
+
+ngx_msec_t
+ngx_event_find_timer(void)
+{
+    ngx_msec_int_t      timer;
+    ngx_rbtree_node_t  *node, *root, *sentinel;
+
+    if (ngx_event_timer_rbtree.root == &ngx_event_timer_sentinel) {
+        return NGX_TIMER_INFINITE;
+    }
+
+    root = ngx_event_timer_rbtree.root;
+    sentinel = ngx_event_timer_rbtree.sentinel;
+
+    node = ngx_rbtree_min(root, sentinel);
+
+    timer = (ngx_msec_int_t) (node->key - ngx_current_msec);
+
+    return (ngx_msec_t) (timer > 0 ? timer : 0);
+}
+
+
+void
+ngx_event_expire_timers(void)
+{
+    ngx_event_t        *ev;
+    ngx_rbtree_node_t  *node, *root, *sentinel;
+
+    sentinel = ngx_event_timer_rbtree.sentinel;
+
+    for ( ;; ) {
+        root = ngx_event_timer_rbtree.root;
+
+        if (root == sentinel) {
+            return;
+        }
+
+        node = ngx_rbtree_min(root, sentinel);
+
+        /* node->key > ngx_current_msec */
+
+        if ((ngx_msec_int_t) (node->key - ngx_current_msec) > 0) {
+            return;
+        }
+
+        ev = ngx_rbtree_data(node, ngx_event_t, timer);
+
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                       "event timer del: %d: %M",
+                       ngx_event_ident(ev->data), ev->timer.key);
+
+        ngx_rbtree_delete(&ngx_event_timer_rbtree, &ev->timer);
+
+#if (NGX_DEBUG)
+        ev->timer.left = NULL;
+        ev->timer.right = NULL;
+        ev->timer.parent = NULL;
+#endif
+
+        ev->timer_set = 0;
+
+        ev->timedout = 1;
+
+        ev->handler(ev);
+    }
+}
+
+
+ngx_int_t
+ngx_event_no_timers_left(void)
+{
+    ngx_event_t        *ev;
+    ngx_rbtree_node_t  *node, *root, *sentinel;
+
+    sentinel = ngx_event_timer_rbtree.sentinel;
+    root = ngx_event_timer_rbtree.root;
+
+    if (root == sentinel) {
+        return NGX_OK;
+    }
+
+    for (node = ngx_rbtree_min(root, sentinel);
+         node;
+         node = ngx_rbtree_next(&ngx_event_timer_rbtree, node))
+    {
+        ev = ngx_rbtree_data(node, ngx_event_t, timer);
+
+        if (!ev->cancelable) {
+            return NGX_AGAIN;
+        }
+    }
+
+    /* only cancelable timers left */
+
+    return NGX_OK;
+}

nginx/src/event/ngx_event_timer.h

@@ -0,0 +1,90 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_TIMER_H_INCLUDED_
+#define _NGX_EVENT_TIMER_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#define NGX_TIMER_INFINITE  (ngx_msec_t) -1
+
+#define NGX_TIMER_LAZY_DELAY  300
+
+
+ngx_int_t ngx_event_timer_init(ngx_log_t *log);
+ngx_msec_t ngx_event_find_timer(void);
+void ngx_event_expire_timers(void);
+ngx_int_t ngx_event_no_timers_left(void);
+
+
+extern ngx_rbtree_t  ngx_event_timer_rbtree;
+
+
+static ngx_inline void
+ngx_event_del_timer(ngx_event_t *ev)
+{
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "event timer del: %d: %M",
+                    ngx_event_ident(ev->data), ev->timer.key);
+
+    ngx_rbtree_delete(&ngx_event_timer_rbtree, &ev->timer);
+
+#if (NGX_DEBUG)
+    ev->timer.left = NULL;
+    ev->timer.right = NULL;
+    ev->timer.parent = NULL;
+#endif
+
+    ev->timer_set = 0;
+}
+
+
+static ngx_inline void
+ngx_event_add_timer(ngx_event_t *ev, ngx_msec_t timer)
+{
+    ngx_msec_t      key;
+    ngx_msec_int_t  diff;
+
+    key = ngx_current_msec + timer;
+
+    if (ev->timer_set) {
+
+        /*
+         * Use a previous timer value if difference between it and a new
+         * value is less than NGX_TIMER_LAZY_DELAY milliseconds: this allows
+         * to minimize the rbtree operations for fast connections.
+         */
+
+        diff = (ngx_msec_int_t) (key - ev->timer.key);
+
+        if (ngx_abs(diff) < NGX_TIMER_LAZY_DELAY) {
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                           "event timer: %d, old: %M, new: %M",
+                            ngx_event_ident(ev->data), ev->timer.key, key);
+            return;
+        }
+
+        ngx_del_timer(ev);
+    }
+
+    ev->timer.key = key;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "event timer add: %d: %M:%M",
+                    ngx_event_ident(ev->data), timer, ev->timer.key);
+
+    ngx_rbtree_insert(&ngx_event_timer_rbtree, &ev->timer);
+
+    ev->timer_set = 1;
+}
+
+
+#endif /* _NGX_EVENT_TIMER_H_INCLUDED_ */

nginx/src/event/ngx_event_udp.c

@@ -0,0 +1,590 @@
+
+/*
+ * Copyright (C) Roman Arutyunyan
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+#if !(NGX_WIN32)
+
+static void ngx_close_accepted_udp_connection(ngx_connection_t *c);
+static ssize_t ngx_udp_shared_recv(ngx_connection_t *c, u_char *buf,
+    size_t size);
+static ngx_int_t ngx_insert_udp_connection(ngx_connection_t *c);
+static ngx_connection_t *ngx_lookup_udp_connection(ngx_listening_t *ls,
+    struct sockaddr *sockaddr, socklen_t socklen,
+    struct sockaddr *local_sockaddr, socklen_t local_socklen);
+
+
+void
+ngx_event_recvmsg(ngx_event_t *ev)
+{
+    ssize_t            n;
+    ngx_buf_t          buf;
+    ngx_log_t         *log;
+    ngx_err_t          err;
+    socklen_t          socklen, local_socklen;
+    ngx_event_t       *rev, *wev;
+    struct iovec       iov[1];
+    struct msghdr      msg;
+    ngx_sockaddr_t     sa, lsa;
+    struct sockaddr   *sockaddr, *local_sockaddr;
+    ngx_listening_t   *ls;
+    ngx_event_conf_t  *ecf;
+    ngx_connection_t  *c, *lc;
+    static u_char      buffer[65535];
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+    u_char             msg_control[CMSG_SPACE(sizeof(ngx_addrinfo_t))];
+#endif
+
+    if (ev->timedout) {
+        if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) {
+            return;
+        }
+
+        ev->timedout = 0;
+    }
+
+    ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
+
+    if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) {
+        ev->available = ecf->multi_accept;
+    }
+
+    lc = ev->data;
+    ls = lc->listening;
+    ev->ready = 0;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "recvmsg on %V, ready: %d", &ls->addr_text, ev->available);
+
+    do {
+        ngx_memzero(&msg, sizeof(struct msghdr));
+
+        iov[0].iov_base = (void *) buffer;
+        iov[0].iov_len = sizeof(buffer);
+
+        msg.msg_name = &sa;
+        msg.msg_namelen = sizeof(ngx_sockaddr_t);
+        msg.msg_iov = iov;
+        msg.msg_iovlen = 1;
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+        if (ls->wildcard) {
+            msg.msg_control = &msg_control;
+            msg.msg_controllen = sizeof(msg_control);
+
+            ngx_memzero(&msg_control, sizeof(msg_control));
+        }
+#endif
+
+        n = recvmsg(lc->fd, &msg, 0);
+
+        if (n == -1) {
+            err = ngx_socket_errno;
+
+            if (err == NGX_EAGAIN) {
+                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err,
+                               "recvmsg() not ready");
+                return;
+            }
+
+            ngx_log_error(NGX_LOG_ALERT, ev->log, err, "recvmsg() failed");
+
+            return;
+        }
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+        if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                          "recvmsg() truncated data");
+            continue;
+        }
+#endif
+
+        sockaddr = msg.msg_name;
+        socklen = msg.msg_namelen;
+
+        if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
+            socklen = sizeof(ngx_sockaddr_t);
+        }
+
+        if (socklen == 0) {
+
+            /*
+             * on Linux recvmsg() returns zero msg_namelen
+             * when receiving packets from unbound AF_UNIX sockets
+             */
+
+            socklen = sizeof(struct sockaddr);
+            ngx_memzero(&sa, sizeof(struct sockaddr));
+            sa.sockaddr.sa_family = ls->sockaddr->sa_family;
+        }
+
+        local_sockaddr = ls->sockaddr;
+        local_socklen = ls->socklen;
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+
+        if (ls->wildcard) {
+            struct cmsghdr  *cmsg;
+
+            ngx_memcpy(&lsa, local_sockaddr, local_socklen);
+            local_sockaddr = &lsa.sockaddr;
+
+            for (cmsg = CMSG_FIRSTHDR(&msg);
+                 cmsg != NULL;
+                 cmsg = CMSG_NXTHDR(&msg, cmsg))
+            {
+                if (ngx_get_srcaddr_cmsg(cmsg, local_sockaddr) == NGX_OK) {
+                    break;
+                }
+            }
+        }
+
+#endif
+
+        c = ngx_lookup_udp_connection(ls, sockaddr, socklen, local_sockaddr,
+                                      local_socklen);
+
+        if (c) {
+
+#if (NGX_DEBUG)
+            if (c->log->log_level & NGX_LOG_DEBUG_EVENT) {
+                ngx_log_handler_pt  handler;
+
+                handler = c->log->handler;
+                c->log->handler = NULL;
+
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                               "recvmsg: fd:%d n:%z", c->fd, n);
+
+                c->log->handler = handler;
+            }
+#endif
+
+            ngx_memzero(&buf, sizeof(ngx_buf_t));
+
+            buf.pos = buffer;
+            buf.last = buffer + n;
+
+            rev = c->read;
+
+            c->udp->buffer = &buf;
+
+            rev->ready = 1;
+            rev->active = 0;
+
+            rev->handler(rev);
+
+            if (c->udp) {
+                c->udp->buffer = NULL;
+            }
+
+            rev->ready = 0;
+            rev->active = 1;
+
+            goto next;
+        }
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1);
+#endif
+
+        ngx_accept_disabled = ngx_cycle->connection_n / 8
+                              - ngx_cycle->free_connection_n;
+
+        c = ngx_get_connection(lc->fd, ev->log);
+        if (c == NULL) {
+            return;
+        }
+
+        c->shared = 1;
+        c->type = SOCK_DGRAM;
+        c->socklen = socklen;
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_active, 1);
+#endif
+
+        c->pool = ngx_create_pool(ls->pool_size, ev->log);
+        if (c->pool == NULL) {
+            ngx_close_accepted_udp_connection(c);
+            return;
+        }
+
+        c->sockaddr = ngx_palloc(c->pool, socklen);
+        if (c->sockaddr == NULL) {
+            ngx_close_accepted_udp_connection(c);
+            return;
+        }
+
+        ngx_memcpy(c->sockaddr, sockaddr, socklen);
+
+        log = ngx_palloc(c->pool, sizeof(ngx_log_t));
+        if (log == NULL) {
+            ngx_close_accepted_udp_connection(c);
+            return;
+        }
+
+        *log = ls->log;
+
+        c->recv = ngx_udp_shared_recv;
+        c->send = ngx_udp_send;
+        c->send_chain = ngx_udp_send_chain;
+
+        c->need_flush_buf = 1;
+
+        c->log = log;
+        c->pool->log = log;
+        c->listening = ls;
+
+        if (local_sockaddr == &lsa.sockaddr) {
+            local_sockaddr = ngx_palloc(c->pool, local_socklen);
+            if (local_sockaddr == NULL) {
+                ngx_close_accepted_udp_connection(c);
+                return;
+            }
+
+            ngx_memcpy(local_sockaddr, &lsa, local_socklen);
+        }
+
+        c->local_sockaddr = local_sockaddr;
+        c->local_socklen = local_socklen;
+
+        c->buffer = ngx_create_temp_buf(c->pool, n);
+        if (c->buffer == NULL) {
+            ngx_close_accepted_udp_connection(c);
+            return;
+        }
+
+        c->buffer->last = ngx_cpymem(c->buffer->last, buffer, n);
+
+        rev = c->read;
+        wev = c->write;
+
+        rev->active = 1;
+        wev->ready = 1;
+
+        rev->log = log;
+        wev->log = log;
+
+        /*
+         * TODO: MT: - ngx_atomic_fetch_add()
+         *             or protection by critical section or light mutex
+         *
+         * TODO: MP: - allocated in a shared memory
+         *           - ngx_atomic_fetch_add()
+         *             or protection by critical section or light mutex
+         */
+
+        c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
+
+        c->start_time = ngx_current_msec;
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_handled, 1);
+#endif
+
+        if (ls->addr_ntop) {
+            c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
+            if (c->addr_text.data == NULL) {
+                ngx_close_accepted_udp_connection(c);
+                return;
+            }
+
+            c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
+                                             c->addr_text.data,
+                                             ls->addr_text_max_len, 0);
+            if (c->addr_text.len == 0) {
+                ngx_close_accepted_udp_connection(c);
+                return;
+            }
+        }
+
+#if (NGX_DEBUG)
+        {
+        ngx_str_t  addr;
+        u_char     text[NGX_SOCKADDR_STRLEN];
+
+        ngx_debug_accepted_connection(ecf, c);
+
+        if (log->log_level & NGX_LOG_DEBUG_EVENT) {
+            addr.data = text;
+            addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text,
+                                     NGX_SOCKADDR_STRLEN, 1);
+
+            ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0,
+                           "*%uA recvmsg: %V fd:%d n:%z",
+                           c->number, &addr, c->fd, n);
+        }
+
+        }
+#endif
+
+        if (ngx_insert_udp_connection(c) != NGX_OK) {
+            ngx_close_accepted_udp_connection(c);
+            return;
+        }
+
+        log->data = NULL;
+        log->handler = NULL;
+
+        ls->handler(c);
+
+    next:
+
+        if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
+            ev->available -= n;
+        }
+
+    } while (ev->available);
+}
+
+
+static void
+ngx_close_accepted_udp_connection(ngx_connection_t *c)
+{
+    ngx_free_connection(c);
+
+    c->fd = (ngx_socket_t) -1;
+
+    if (c->pool) {
+        ngx_destroy_pool(c->pool);
+    }
+
+#if (NGX_STAT_STUB)
+    (void) ngx_atomic_fetch_add(ngx_stat_active, -1);
+#endif
+}
+
+
+static ssize_t
+ngx_udp_shared_recv(ngx_connection_t *c, u_char *buf, size_t size)
+{
+    ssize_t     n;
+    ngx_buf_t  *b;
+
+    if (c->udp == NULL || c->udp->buffer == NULL) {
+        return NGX_AGAIN;
+    }
+
+    b = c->udp->buffer;
+
+    n = ngx_min(b->last - b->pos, (ssize_t) size);
+
+    ngx_memcpy(buf, b->pos, n);
+
+    c->udp->buffer = NULL;
+
+    c->read->ready = 0;
+    c->read->active = 1;
+
+    return n;
+}
+
+
+void
+ngx_udp_rbtree_insert_value(ngx_rbtree_node_t *temp,
+    ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
+{
+    ngx_int_t               rc;
+    ngx_connection_t       *c, *ct;
+    ngx_rbtree_node_t     **p;
+    ngx_udp_connection_t   *udp, *udpt;
+
+    for ( ;; ) {
+
+        if (node->key < temp->key) {
+
+            p = &temp->left;
+
+        } else if (node->key > temp->key) {
+
+            p = &temp->right;
+
+        } else { /* node->key == temp->key */
+
+            udp = (ngx_udp_connection_t *) node;
+            c = udp->connection;
+
+            udpt = (ngx_udp_connection_t *) temp;
+            ct = udpt->connection;
+
+            rc = ngx_memn2cmp(udp->key.data, udpt->key.data,
+                              udp->key.len, udpt->key.len);
+
+            if (rc == 0 && c->listening->wildcard) {
+                rc = ngx_cmp_sockaddr(c->local_sockaddr, c->local_socklen,
+                                      ct->local_sockaddr, ct->local_socklen, 1);
+            }
+
+            p = (rc < 0) ? &temp->left : &temp->right;
+        }
+
+        if (*p == sentinel) {
+            break;
+        }
+
+        temp = *p;
+    }
+
+    *p = node;
+    node->parent = temp;
+    node->left = sentinel;
+    node->right = sentinel;
+    ngx_rbt_red(node);
+}
+
+
+static ngx_int_t
+ngx_insert_udp_connection(ngx_connection_t *c)
+{
+    uint32_t               hash;
+    ngx_pool_cleanup_t    *cln;
+    ngx_udp_connection_t  *udp;
+
+    if (c->udp) {
+        return NGX_OK;
+    }
+
+    udp = ngx_pcalloc(c->pool, sizeof(ngx_udp_connection_t));
+    if (udp == NULL) {
+        return NGX_ERROR;
+    }
+
+    udp->connection = c;
+
+    ngx_crc32_init(hash);
+    ngx_crc32_update(&hash, (u_char *) c->sockaddr, c->socklen);
+
+    if (c->listening->wildcard) {
+        ngx_crc32_update(&hash, (u_char *) c->local_sockaddr, c->local_socklen);
+    }
+
+    ngx_crc32_final(hash);
+
+    udp->node.key = hash;
+    udp->key.data = (u_char *) c->sockaddr;
+    udp->key.len = c->socklen;
+
+    cln = ngx_pool_cleanup_add(c->pool, 0);
+    if (cln == NULL) {
+        return NGX_ERROR;
+    }
+
+    cln->data = c;
+    cln->handler = ngx_delete_udp_connection;
+
+    ngx_rbtree_insert(&c->listening->rbtree, &udp->node);
+
+    c->udp = udp;
+
+    return NGX_OK;
+}
+
+
+void
+ngx_delete_udp_connection(void *data)
+{
+    ngx_connection_t  *c = data;
+
+    if (c->udp == NULL) {
+        return;
+    }
+
+    ngx_rbtree_delete(&c->listening->rbtree, &c->udp->node);
+
+    c->udp = NULL;
+}
+
+
+static ngx_connection_t *
+ngx_lookup_udp_connection(ngx_listening_t *ls, struct sockaddr *sockaddr,
+    socklen_t socklen, struct sockaddr *local_sockaddr, socklen_t local_socklen)
+{
+    uint32_t               hash;
+    ngx_int_t              rc;
+    ngx_connection_t      *c;
+    ngx_rbtree_node_t     *node, *sentinel;
+    ngx_udp_connection_t  *udp;
+
+#if (NGX_HAVE_UNIX_DOMAIN)
+
+    if (sockaddr->sa_family == AF_UNIX) {
+        struct sockaddr_un *saun = (struct sockaddr_un *) sockaddr;
+
+        if (socklen <= (socklen_t) offsetof(struct sockaddr_un, sun_path)
+            || saun->sun_path[0] == '\0')
+        {
+            ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0,
+                           "unbound unix socket");
+            return NULL;
+        }
+    }
+
+#endif
+
+    node = ls->rbtree.root;
+    sentinel = ls->rbtree.sentinel;
+
+    ngx_crc32_init(hash);
+    ngx_crc32_update(&hash, (u_char *) sockaddr, socklen);
+
+    if (ls->wildcard) {
+        ngx_crc32_update(&hash, (u_char *) local_sockaddr, local_socklen);
+    }
+
+    ngx_crc32_final(hash);
+
+    while (node != sentinel) {
+
+        if (hash < node->key) {
+            node = node->left;
+            continue;
+        }
+
+        if (hash > node->key) {
+            node = node->right;
+            continue;
+        }
+
+        /* hash == node->key */
+
+        udp = (ngx_udp_connection_t *) node;
+
+        c = udp->connection;
+
+        rc = ngx_cmp_sockaddr(sockaddr, socklen,
+                              c->sockaddr, c->socklen, 1);
+
+        if (rc == 0 && ls->wildcard) {
+            rc = ngx_cmp_sockaddr(local_sockaddr, local_socklen,
+                                  c->local_sockaddr, c->local_socklen, 1);
+        }
+
+        if (rc == 0) {
+            return c;
+        }
+
+        node = (rc < 0) ? node->left : node->right;
+    }
+
+    return NULL;
+}
+
+#else
+
+void
+ngx_delete_udp_connection(void *data)
+{
+    return;
+}
+
+#endif

nginx/src/event/ngx_event_udp.h

@@ -0,0 +1,66 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_UDP_H_INCLUDED_
+#define _NGX_EVENT_UDP_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+#if !(NGX_WIN32)
+
+#if ((NGX_HAVE_MSGHDR_MSG_CONTROL)                                            \
+     && (NGX_HAVE_IP_SENDSRCADDR || NGX_HAVE_IP_RECVDSTADDR                   \
+         || NGX_HAVE_IP_PKTINFO                                               \
+         || (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO)))
+#define NGX_HAVE_ADDRINFO_CMSG  1
+
+#endif
+
+
+struct ngx_udp_connection_s {
+    ngx_rbtree_node_t   node;
+    ngx_connection_t   *connection;
+    ngx_buf_t          *buffer;
+    ngx_str_t           key;
+};
+
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+
+typedef union {
+#if (NGX_HAVE_IP_SENDSRCADDR || NGX_HAVE_IP_RECVDSTADDR)
+    struct in_addr        addr;
+#endif
+
+#if (NGX_HAVE_IP_PKTINFO)
+    struct in_pktinfo     pkt;
+#endif
+
+#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO)
+    struct in6_pktinfo    pkt6;
+#endif
+} ngx_addrinfo_t;
+
+size_t ngx_set_srcaddr_cmsg(struct cmsghdr *cmsg,
+    struct sockaddr *local_sockaddr);
+ngx_int_t ngx_get_srcaddr_cmsg(struct cmsghdr *cmsg,
+    struct sockaddr *local_sockaddr);
+
+#endif
+
+void ngx_event_recvmsg(ngx_event_t *ev);
+ssize_t ngx_sendmsg(ngx_connection_t *c, struct msghdr *msg, int flags);
+void ngx_udp_rbtree_insert_value(ngx_rbtree_node_t *temp,
+    ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
+#endif
+
+void ngx_delete_udp_connection(void *data);
+
+
+#endif /* _NGX_EVENT_UDP_H_INCLUDED_ */

nginx/src/event/quic/bpf/bpfgen.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+export LANG=C
+
+set -e
+
+if [ $# -lt 1 ]; then
+    echo "Usage: PROGNAME=foo LICENSE=bar $0 <bpf object file>"
+    exit 1
+fi
+
+
+self=$0
+filename=$1
+funcname=$PROGNAME
+
+generate_head()
+{
+    cat << END
+/* AUTO-GENERATED, DO NOT EDIT. */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "ngx_bpf.h"
+
+
+END
+}
+
+generate_tail()
+{
+    cat << END
+
+ngx_bpf_program_t $PROGNAME = {
+    .relocs = bpf_reloc_prog_$funcname,
+    .nrelocs = sizeof(bpf_reloc_prog_$funcname)
+               / sizeof(bpf_reloc_prog_$funcname[0]),
+    .ins = bpf_insn_prog_$funcname,
+    .nins = sizeof(bpf_insn_prog_$funcname)
+            / sizeof(bpf_insn_prog_$funcname[0]),
+    .license = "$LICENSE",
+    .type = BPF_PROG_TYPE_SK_REUSEPORT,
+};
+
+END
+}
+
+process_relocations()
+{
+    echo "static ngx_bpf_reloc_t bpf_reloc_prog_$funcname[] = {"
+
+    objdump -r $filename | awk '{
+
+    if (enabled && $NF > 0) {
+        off = strtonum(sprintf("0x%s", $1));
+        name = $3;
+
+        printf("    { \"%s\", %d },\n", name, off/8);
+    }
+
+    if ($1 == "OFFSET") {
+        enabled=1;
+    }
+}'
+    echo "};"
+    echo
+}
+
+process_section()
+{
+    echo "static struct bpf_insn bpf_insn_prog_$funcname[] = {"
+    echo "    /* opcode dst          src         offset imm */"
+
+    section_info=$(objdump -h $filename --section=$funcname | grep "1 $funcname")
+
+    # dd doesn't know hex
+    length=$(printf "%d" 0x$(echo $section_info | cut -d ' ' -f3))
+    offset=$(printf "%d" 0x$(echo $section_info | cut -d ' ' -f6))
+
+    for ins in $(dd if="$filename" bs=1 count=$length skip=$offset status=none | xxd -p -c 8)
+    do
+        opcode=0x${ins:0:2}
+        srcdst=0x${ins:2:2}
+
+        # bytes are dumped in LE order
+        offset=0x${ins:6:2}${ins:4:2}                        # short
+        immedi=0x${ins:14:2}${ins:12:2}${ins:10:2}${ins:8:2} # int
+
+        dst="$(($srcdst & 0xF))"
+        src="$(($srcdst & 0xF0))"
+        src="$(($src >> 4))"
+
+        opcode=$(printf "0x%x" $opcode)
+        dst=$(printf "BPF_REG_%d" $dst)
+        src=$(printf "BPF_REG_%d" $src)
+        offset=$(printf "%d" $offset)
+        immedi=$(printf "0x%x" $immedi)
+
+        printf "    { %4s, %11s, %11s, (int16_t) %6s, %10s },\n" $opcode $dst $src $offset $immedi
+    done
+
+cat << END
+};
+
+END
+}
+
+generate_head
+process_relocations
+process_section
+generate_tail
+

nginx/src/event/quic/bpf/makefile

@@ -0,0 +1,30 @@
+CFLAGS=-O2 -Wall
+
+LICENSE=BSD
+
+PROGNAME=ngx_quic_reuseport_helper
+RESULT=ngx_event_quic_bpf_code
+DEST=../$(RESULT).c
+
+all: $(RESULT)
+
+$(RESULT): $(PROGNAME).o
+	LICENSE=$(LICENSE) PROGNAME=$(PROGNAME) bash ./bpfgen.sh $< > $@
+
+DEFS=-DPROGNAME=\"$(PROGNAME)\"                                               \
+     -DLICENSE_$(LICENSE)                                                     \
+     -DLICENSE=\"$(LICENSE)\"                                                 \
+
+$(PROGNAME).o: $(PROGNAME).c
+	clang $(CFLAGS) $(DEFS) -target bpf -c $< -o $@
+
+install: $(RESULT)
+	cp $(RESULT) $(DEST)
+
+clean:
+	@rm -f $(RESULT) *.o
+
+debug: $(PROGNAME).o
+	llvm-objdump -S --no-show-raw-insn $<
+
+.DELETE_ON_ERROR:

nginx/src/event/quic/bpf/ngx_quic_reuseport_helper.c

@@ -0,0 +1,140 @@
+#include <errno.h>
+#include <linux/string.h>
+#include <linux/udp.h>
+#include <linux/bpf.h>
+/*
+ * the bpf_helpers.h is not included into linux-headers, only available
+ * with kernel sources in "tools/lib/bpf/bpf_helpers.h" or in libbpf.
+ */
+#include <bpf/bpf_helpers.h>
+
+
+#if !defined(SEC)
+#define SEC(NAME)  __attribute__((section(NAME), used))
+#endif
+
+
+#if defined(LICENSE_GPL)
+
+/*
+ * To see debug:
+ *
+ *  echo 1 > /sys/kernel/debug/tracing/events/bpf_trace/enable
+ *  cat /sys/kernel/debug/tracing/trace_pipe
+ *  echo 0 > /sys/kernel/debug/tracing/events/bpf_trace/enable
+ */
+
+#define debugmsg(fmt, ...)                                                    \
+do {                                                                          \
+    char __buf[] = fmt;                                                       \
+    bpf_trace_printk(__buf, sizeof(__buf), ##__VA_ARGS__);                    \
+} while (0)
+
+#else
+
+#define debugmsg(fmt, ...)
+
+#endif
+
+char _license[] SEC("license") = LICENSE;
+
+/*****************************************************************************/
+
+#define NGX_QUIC_PKT_LONG        0x80  /* header form */
+#define NGX_QUIC_SERVER_CID_LEN  20
+
+
+#define advance_data(nbytes)                                                  \
+    offset += nbytes;                                                         \
+    if (start + offset > end) {                                               \
+        debugmsg("cannot read %ld bytes at offset %ld", nbytes, offset);      \
+        goto failed;                                                          \
+    }                                                                         \
+    data = start + offset - 1;
+
+
+#define ngx_quic_parse_uint64(p)                                              \
+    (((__u64)(p)[0] << 56) |                                                  \
+     ((__u64)(p)[1] << 48) |                                                  \
+     ((__u64)(p)[2] << 40) |                                                  \
+     ((__u64)(p)[3] << 32) |                                                  \
+     ((__u64)(p)[4] << 24) |                                                  \
+     ((__u64)(p)[5] << 16) |                                                  \
+     ((__u64)(p)[6] << 8)  |                                                  \
+     ((__u64)(p)[7]))
+
+/*
+ * actual map object is created by the "bpf" system call,
+ * all pointers to this variable are replaced by the bpf loader
+ */
+extern int ngx_quic_sockmap;
+
+
+SEC(PROGNAME)
+int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx)
+{
+    int             rc;
+    __u64           key;
+    size_t          len, offset;
+    unsigned char  *start, *end, *data, *dcid;
+
+    start = ctx->data;
+    end = (unsigned char *) ctx->data_end;
+    offset = 0;
+
+    advance_data(sizeof(struct udphdr)); /* data at UDP header */
+    advance_data(1); /* data at QUIC flags */
+
+    if (data[0] & NGX_QUIC_PKT_LONG) {
+
+        advance_data(4); /* data at QUIC version */
+        advance_data(1); /* data at DCID len */
+
+        len = data[0];   /* read DCID length */
+
+        if (len < 8) {
+            /* it's useless to search for key in such short DCID */
+            return SK_PASS;
+        }
+
+    } else {
+        len = NGX_QUIC_SERVER_CID_LEN;
+    }
+
+    dcid = &data[1];
+    advance_data(len); /* we expect the packet to have full DCID */
+
+    /* make verifier happy */
+    if (dcid + sizeof(__u64) > end) {
+        goto failed;
+    }
+
+    key = ngx_quic_parse_uint64(dcid);
+
+    rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0);
+
+    switch (rc) {
+    case 0:
+        debugmsg("nginx quic socket selected by key 0x%llx", key);
+        return SK_PASS;
+
+    /* kernel returns positive error numbers, errno.h defines positive */
+    case -ENOENT:
+        debugmsg("nginx quic default route for key 0x%llx", key);
+        /* let the default reuseport logic decide which socket to choose */
+        return SK_PASS;
+
+    default:
+        debugmsg("nginx quic bpf_sk_select_reuseport err: %d key 0x%llx",
+                 rc, key);
+        goto failed;
+    }
+
+failed:
+    /*
+     * SK_DROP will generate ICMP, but we may want to process "invalid" packet
+     * in userspace quic to investigate further and finally react properly
+     * (maybe ignore, maybe send something in response or close connection)
+     */
+    return SK_PASS;
+}

nginx/src/event/quic/ngx_event_quic.h

@@ -0,0 +1,145 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_H_INCLUDED_
+#define _NGX_EVENT_QUIC_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+#if (OPENSSL_VERSION_NUMBER >= 0x30500010L)
+#define NGX_QUIC_OPENSSL_API                 1
+
+#elif (defined SSL_R_MISSING_QUIC_TRANSPORT_PARAMETERS_EXTENSION)
+#define NGX_QUIC_QUICTLS_API                 1
+
+#elif (defined OPENSSL_IS_BORINGSSL || defined OPENSSL_IS_AWSLC               \
+       || defined LIBRESSL_VERSION_NUMBER)
+#define NGX_QUIC_BORINGSSL_API               1
+
+#else
+#define NGX_QUIC_BORINGSSL_API               1
+#define NGX_QUIC_OPENSSL_COMPAT              1
+#endif
+
+
+#define NGX_QUIC_MAX_UDP_PAYLOAD_SIZE        65527
+
+#define NGX_QUIC_DEFAULT_ACK_DELAY_EXPONENT  3
+#define NGX_QUIC_DEFAULT_MAX_ACK_DELAY       25
+#define NGX_QUIC_DEFAULT_HOST_KEY_LEN        32
+#define NGX_QUIC_SR_KEY_LEN                  32
+#define NGX_QUIC_AV_KEY_LEN                  32
+
+#define NGX_QUIC_SR_TOKEN_LEN                16
+
+#define NGX_QUIC_MIN_INITIAL_SIZE            1200
+
+#define NGX_QUIC_STREAM_SERVER_INITIATED     0x01
+#define NGX_QUIC_STREAM_UNIDIRECTIONAL       0x02
+
+
+typedef ngx_int_t (*ngx_quic_init_pt)(ngx_connection_t *c);
+typedef void (*ngx_quic_shutdown_pt)(ngx_connection_t *c);
+
+
+typedef enum {
+    NGX_QUIC_STREAM_SEND_READY = 0,
+    NGX_QUIC_STREAM_SEND_SEND,
+    NGX_QUIC_STREAM_SEND_DATA_SENT,
+    NGX_QUIC_STREAM_SEND_DATA_RECVD,
+    NGX_QUIC_STREAM_SEND_RESET_SENT,
+    NGX_QUIC_STREAM_SEND_RESET_RECVD
+} ngx_quic_stream_send_state_e;
+
+
+typedef enum {
+    NGX_QUIC_STREAM_RECV_RECV = 0,
+    NGX_QUIC_STREAM_RECV_SIZE_KNOWN,
+    NGX_QUIC_STREAM_RECV_DATA_RECVD,
+    NGX_QUIC_STREAM_RECV_DATA_READ,
+    NGX_QUIC_STREAM_RECV_RESET_RECVD,
+    NGX_QUIC_STREAM_RECV_RESET_READ
+} ngx_quic_stream_recv_state_e;
+
+
+typedef struct {
+    uint64_t                       size;
+    uint64_t                       offset;
+    uint64_t                       last_offset;
+    ngx_chain_t                   *chain;
+    ngx_chain_t                   *last_chain;
+} ngx_quic_buffer_t;
+
+
+typedef struct {
+    ngx_ssl_t                     *ssl;
+
+    ngx_flag_t                     retry;
+    ngx_flag_t                     gso_enabled;
+    ngx_flag_t                     disable_active_migration;
+    ngx_msec_t                     handshake_timeout;
+    ngx_msec_t                     idle_timeout;
+    ngx_str_t                      host_key;
+    size_t                         stream_buffer_size;
+    ngx_uint_t                     max_concurrent_streams_bidi;
+    ngx_uint_t                     max_concurrent_streams_uni;
+    ngx_uint_t                     active_connection_id_limit;
+    ngx_int_t                      stream_close_code;
+    ngx_int_t                      stream_reject_code_uni;
+    ngx_int_t                      stream_reject_code_bidi;
+
+    ngx_quic_init_pt               init;
+    ngx_quic_shutdown_pt           shutdown;
+
+    u_char                         av_token_key[NGX_QUIC_AV_KEY_LEN];
+    u_char                         sr_token_key[NGX_QUIC_SR_KEY_LEN];
+} ngx_quic_conf_t;
+
+
+struct ngx_quic_stream_s {
+    ngx_rbtree_node_t              node;
+    ngx_queue_t                    queue;
+    ngx_connection_t              *parent;
+    ngx_connection_t              *connection;
+    uint64_t                       id;
+    uint64_t                       sent;
+    uint64_t                       acked;
+    uint64_t                       send_max_data;
+    uint64_t                       send_offset;
+    uint64_t                       send_final_size;
+    uint64_t                       recv_max_data;
+    uint64_t                       recv_offset;
+    uint64_t                       recv_window;
+    uint64_t                       recv_last;
+    uint64_t                       recv_final_size;
+    ngx_quic_buffer_t              send;
+    ngx_quic_buffer_t              recv;
+    ngx_quic_stream_send_state_e   send_state;
+    ngx_quic_stream_recv_state_e   recv_state;
+    unsigned                       cancelable:1;
+    unsigned                       fin_acked:1;
+};
+
+
+void ngx_quic_recvmsg(ngx_event_t *ev);
+void ngx_quic_run(ngx_connection_t *c, ngx_quic_conf_t *conf);
+ngx_connection_t *ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi);
+void ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
+    const char *reason);
+void ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err,
+    const char *reason);
+ngx_int_t ngx_quic_reset_stream(ngx_connection_t *c, ngx_uint_t err);
+ngx_int_t ngx_quic_shutdown_stream(ngx_connection_t *c, int how);
+void ngx_quic_cancelable_stream(ngx_connection_t *c);
+ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len,
+    ngx_str_t *dcid);
+ngx_int_t ngx_quic_derive_key(ngx_log_t *log, const char *label,
+    ngx_str_t *secret, ngx_str_t *salt, u_char *out, size_t len);
+
+#endif /* _NGX_EVENT_QUIC_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_ack.h

@@ -0,0 +1,31 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_ACK_H_INCLUDED_
+#define _NGX_EVENT_QUIC_ACK_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+ngx_int_t ngx_quic_handle_ack_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_frame_t *f);
+
+void ngx_quic_congestion_ack(ngx_connection_t *c,
+    ngx_quic_frame_t *frame);
+void ngx_quic_congestion_idle(ngx_connection_t *c, ngx_uint_t idle);
+void ngx_quic_resend_frames(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx);
+void ngx_quic_set_lost_timer(ngx_connection_t *c);
+void ngx_quic_pto_handler(ngx_event_t *ev);
+ngx_msec_t ngx_quic_pto(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx);
+
+ngx_int_t ngx_quic_ack_packet(ngx_connection_t *c,
+    ngx_quic_header_t *pkt);
+ngx_int_t ngx_quic_generate_ack(ngx_connection_t *c,
+    ngx_quic_send_ctx_t *ctx);
+
+#endif /* _NGX_EVENT_QUIC_ACK_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_bpf.c

@@ -0,0 +1,657 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+#define NGX_QUIC_BPF_VARNAME  "NGINX_BPF_MAPS"
+#define NGX_QUIC_BPF_VARSEP    ';'
+#define NGX_QUIC_BPF_ADDRSEP   '#'
+
+
+#define ngx_quic_bpf_get_conf(cycle)                                          \
+    (ngx_quic_bpf_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_quic_bpf_module)
+
+#define ngx_quic_bpf_get_old_conf(cycle)                                      \
+    cycle->old_cycle->conf_ctx ? ngx_quic_bpf_get_conf(cycle->old_cycle)      \
+                               : NULL
+
+#define ngx_core_get_conf(cycle)                                              \
+    (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module)
+
+
+typedef struct {
+    ngx_queue_t           queue;
+    int                   map_fd;
+
+    struct sockaddr      *sockaddr;
+    socklen_t             socklen;
+    ngx_uint_t            unused;     /* unsigned  unused:1; */
+} ngx_quic_sock_group_t;
+
+
+typedef struct {
+    ngx_flag_t            enabled;
+    ngx_uint_t            map_size;
+    ngx_queue_t           groups;     /* of ngx_quic_sock_group_t */
+} ngx_quic_bpf_conf_t;
+
+
+static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle);
+static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle);
+
+static void ngx_quic_bpf_cleanup(void *data);
+static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd,
+    const char *name);
+
+static ngx_quic_sock_group_t *ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf,
+    ngx_listening_t *ls);
+static ngx_quic_sock_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle,
+    struct sockaddr *sa, socklen_t socklen);
+static ngx_quic_sock_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle,
+    ngx_listening_t *ls);
+static ngx_quic_sock_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle,
+    ngx_listening_t *ls);
+static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle,
+    ngx_listening_t *ls);
+static uint64_t ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log);
+
+static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle);
+static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle);
+
+extern ngx_bpf_program_t  ngx_quic_reuseport_helper;
+
+
+static ngx_command_t  ngx_quic_bpf_commands[] = {
+
+    { ngx_string("quic_bpf"),
+      NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      0,
+      offsetof(ngx_quic_bpf_conf_t, enabled),
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_core_module_t  ngx_quic_bpf_module_ctx = {
+    ngx_string("quic_bpf"),
+    ngx_quic_bpf_create_conf,
+    NULL
+};
+
+
+ngx_module_t  ngx_quic_bpf_module = {
+    NGX_MODULE_V1,
+    &ngx_quic_bpf_module_ctx,              /* module context */
+    ngx_quic_bpf_commands,                 /* module directives */
+    NGX_CORE_MODULE,                       /* module type */
+    NULL,                                  /* init master */
+    ngx_quic_bpf_module_init,              /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static void *
+ngx_quic_bpf_create_conf(ngx_cycle_t *cycle)
+{
+    ngx_quic_bpf_conf_t  *bcf;
+
+    bcf = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_conf_t));
+    if (bcf == NULL) {
+        return NULL;
+    }
+
+    bcf->enabled = NGX_CONF_UNSET;
+    bcf->map_size = NGX_CONF_UNSET_UINT;
+
+    ngx_queue_init(&bcf->groups);
+
+    return bcf;
+}
+
+
+static ngx_int_t
+ngx_quic_bpf_module_init(ngx_cycle_t *cycle)
+{
+    ngx_uint_t            i;
+    ngx_listening_t      *ls;
+    ngx_core_conf_t      *ccf;
+    ngx_pool_cleanup_t   *cln;
+    ngx_quic_bpf_conf_t  *bcf;
+
+    if (ngx_test_config) {
+        /*
+         * during config test, SO_REUSEPORT socket option is
+         * not set, thus making further processing meaningless
+         */
+        return NGX_OK;
+    }
+
+    ccf = ngx_core_get_conf(cycle);
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+    ngx_conf_init_value(bcf->enabled, 0);
+
+    bcf->map_size = ccf->worker_processes * 4;
+
+    cln = ngx_pool_cleanup_add(cycle->pool, 0);
+    if (cln == NULL) {
+        goto failed;
+    }
+
+    cln->data = bcf;
+    cln->handler = ngx_quic_bpf_cleanup;
+
+    if (ngx_inherited && ngx_is_init_cycle(cycle->old_cycle)) {
+        if (ngx_quic_bpf_import_maps(cycle) != NGX_OK) {
+            goto failed;
+        }
+    }
+
+    ls = cycle->listening.elts;
+
+    for (i = 0; i < cycle->listening.nelts; i++) {
+        if (ls[i].quic && ls[i].reuseport) {
+            if (ngx_quic_bpf_group_add_socket(cycle, &ls[i]) != NGX_OK) {
+                goto failed;
+            }
+        }
+    }
+
+    if (ngx_quic_bpf_export_maps(cycle) != NGX_OK) {
+        goto failed;
+    }
+
+    return NGX_OK;
+
+failed:
+
+    if (ngx_is_init_cycle(cycle->old_cycle)) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                      "ngx_quic_bpf_module failed to initialize, check limits");
+
+        /* refuse to start */
+        return NGX_ERROR;
+    }
+
+    /*
+     * returning error now will lead to master process exiting immediately
+     * leaving worker processes orphaned, what is really unexpected.
+     * Instead, just issue a not about failed initialization and try
+     * to cleanup a bit. Still program can be already loaded to kernel
+     * for some reuseport groups, and there is no way to revert, so
+     * behaviour may be inconsistent.
+     */
+
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                  "ngx_quic_bpf_module failed to initialize properly, ignored."
+                  "please check limits and note that nginx state now "
+                  "can be inconsistent and restart may be required");
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_quic_bpf_cleanup(void *data)
+{
+    ngx_quic_bpf_conf_t  *bcf = (ngx_quic_bpf_conf_t *) data;
+
+    ngx_queue_t            *q;
+    ngx_quic_sock_group_t  *grp;
+
+    for (q = ngx_queue_head(&bcf->groups);
+         q != ngx_queue_sentinel(&bcf->groups);
+         q = ngx_queue_next(q))
+    {
+        grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+
+        ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map");
+    }
+}
+
+
+static ngx_inline void
+ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name)
+{
+    if (close(fd) != -1) {
+        return;
+    }
+
+    ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,
+                  "quic bpf close %s fd:%d failed", name, fd);
+}
+
+
+static ngx_quic_sock_group_t *
+ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls)
+{
+    ngx_queue_t            *q;
+    ngx_quic_sock_group_t  *grp;
+
+    for (q = ngx_queue_head(&bcf->groups);
+         q != ngx_queue_sentinel(&bcf->groups);
+         q = ngx_queue_next(q))
+    {
+        grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+
+        if (ngx_cmp_sockaddr(ls->sockaddr, ls->socklen,
+                             grp->sockaddr, grp->socklen, 1)
+            == NGX_OK)
+        {
+            return grp;
+        }
+    }
+
+    return NULL;
+}
+
+
+static ngx_quic_sock_group_t *
+ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa,
+    socklen_t socklen)
+{
+    ngx_quic_bpf_conf_t    *bcf;
+    ngx_quic_sock_group_t  *grp;
+
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+    grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_sock_group_t));
+    if (grp == NULL) {
+        return NULL;
+    }
+
+    grp->socklen = socklen;
+    grp->sockaddr = ngx_palloc(cycle->pool, socklen);
+    if (grp->sockaddr == NULL) {
+        return NULL;
+    }
+    ngx_memcpy(grp->sockaddr, sa, socklen);
+
+    ngx_queue_insert_tail(&bcf->groups, &grp->queue);
+
+    return grp;
+}
+
+
+static ngx_quic_sock_group_t *
+ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
+{
+    int                     progfd, failed, flags, rc;
+    ngx_quic_bpf_conf_t    *bcf;
+    ngx_quic_sock_group_t  *grp;
+
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+    if (!bcf->enabled) {
+        return NULL;
+    }
+
+    grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen);
+    if (grp == NULL) {
+        return NULL;
+    }
+
+    grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH,
+                                     sizeof(uint64_t), sizeof(uint64_t),
+                                     bcf->map_size, 0);
+    if (grp->map_fd == -1) {
+        goto failed;
+    }
+
+    flags = fcntl(grp->map_fd, F_GETFD);
+    if (flags == -1) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, errno,
+                      "quic bpf getfd failed");
+        goto failed;
+    }
+
+    /* need to inherit map during binary upgrade after exec */
+    flags &= ~FD_CLOEXEC;
+
+    rc = fcntl(grp->map_fd, F_SETFD, flags);
+    if (rc == -1) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, errno,
+                      "quic bpf setfd failed");
+        goto failed;
+    }
+
+    ngx_bpf_program_link(&ngx_quic_reuseport_helper,
+                         "ngx_quic_sockmap", grp->map_fd);
+
+    progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper);
+    if (progfd < 0) {
+        goto failed;
+    }
+
+    failed = 0;
+
+    if (setsockopt(ls->fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF,
+                   &progfd, sizeof(int))
+        == -1)
+    {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                      "quic bpf setsockopt(SO_ATTACH_REUSEPORT_EBPF) failed");
+        failed = 1;
+    }
+
+    ngx_quic_bpf_close(cycle->log, progfd, "program");
+
+    if (failed) {
+        goto failed;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "quic bpf sockmap created fd:%d", grp->map_fd);
+    return grp;
+
+failed:
+
+    if (grp->map_fd != -1) {
+        ngx_quic_bpf_close(cycle->log, grp->map_fd, "map");
+    }
+
+    ngx_queue_remove(&grp->queue);
+
+    return NULL;
+}
+
+
+static ngx_quic_sock_group_t *
+ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
+{
+    ngx_quic_bpf_conf_t    *bcf, *old_bcf;
+    ngx_quic_sock_group_t  *grp, *ogrp;
+
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+    grp = ngx_quic_bpf_find_group(bcf, ls);
+    if (grp) {
+        return grp;
+    }
+
+    old_bcf = ngx_quic_bpf_get_old_conf(cycle);
+
+    if (old_bcf == NULL) {
+        return ngx_quic_bpf_create_group(cycle, ls);
+    }
+
+    ogrp = ngx_quic_bpf_find_group(old_bcf, ls);
+    if (ogrp == NULL) {
+        return ngx_quic_bpf_create_group(cycle, ls);
+    }
+
+    grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen);
+    if (grp == NULL) {
+        return NULL;
+    }
+
+    grp->map_fd = dup(ogrp->map_fd);
+    if (grp->map_fd == -1) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                      "quic bpf failed to duplicate bpf map descriptor");
+
+        ngx_queue_remove(&grp->queue);
+
+        return NULL;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                   "quic bpf sockmap fd duplicated old:%d new:%d",
+                   ogrp->map_fd, grp->map_fd);
+
+    return grp;
+}
+
+
+static ngx_int_t
+ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle,  ngx_listening_t *ls)
+{
+    uint64_t                cookie;
+    ngx_quic_bpf_conf_t    *bcf;
+    ngx_quic_sock_group_t  *grp;
+
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+    grp = ngx_quic_bpf_get_group(cycle, ls);
+
+    if (grp == NULL) {
+        if (!bcf->enabled) {
+            return NGX_OK;
+        }
+
+        return NGX_ERROR;
+    }
+
+    grp->unused = 0;
+
+    cookie = ngx_quic_bpf_socket_key(ls->fd, cycle->log);
+    if (cookie == (uint64_t) NGX_ERROR) {
+        return NGX_ERROR;
+    }
+
+    /* map[cookie] = socket; for use in kernel helper */
+    if (ngx_bpf_map_update(grp->map_fd, &cookie, &ls->fd, BPF_ANY) == -1) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                      "quic bpf failed to update socket map key=%xL", cookie);
+        return NGX_ERROR;
+    }
+
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                 "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui",
+                 grp->map_fd, ls->fd, cookie, ls->worker);
+
+    /* do not inherit this socket */
+    ls->ignore = 1;
+
+    return NGX_OK;
+}
+
+
+static uint64_t
+ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log)
+{
+    uint64_t   cookie;
+    socklen_t  optlen;
+
+    optlen = sizeof(cookie);
+
+    if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) {
+        ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+                      "quic bpf getsockopt(SO_COOKIE) failed");
+
+        return (ngx_uint_t) NGX_ERROR;
+    }
+
+    return cookie;
+}
+
+
+static ngx_int_t
+ngx_quic_bpf_export_maps(ngx_cycle_t *cycle)
+{
+    u_char                 *p, *buf;
+    size_t                  len;
+    ngx_str_t              *var;
+    ngx_queue_t            *q;
+    ngx_core_conf_t        *ccf;
+    ngx_quic_bpf_conf_t    *bcf;
+    ngx_quic_sock_group_t  *grp;
+
+    ccf = ngx_core_get_conf(cycle);
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+    len = sizeof(NGX_QUIC_BPF_VARNAME) + 1;
+
+    q = ngx_queue_head(&bcf->groups);
+
+    while (q != ngx_queue_sentinel(&bcf->groups)) {
+
+        grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+
+        q = ngx_queue_next(q);
+
+        if (grp->unused) {
+            /*
+             * map was inherited, but it is not used in this configuration;
+             * do not pass such map further and drop the group to prevent
+             * interference with changes during reload
+             */
+
+            ngx_quic_bpf_close(cycle->log, grp->map_fd, "map");
+            ngx_queue_remove(&grp->queue);
+
+            continue;
+        }
+
+        len += NGX_INT32_LEN + 1 + NGX_SOCKADDR_STRLEN + 1;
+    }
+
+    len++;
+
+    buf = ngx_palloc(cycle->pool, len);
+    if (buf == NULL) {
+        return NGX_ERROR;
+    }
+
+    p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=",
+                   sizeof(NGX_QUIC_BPF_VARNAME));
+
+    for (q = ngx_queue_head(&bcf->groups);
+         q != ngx_queue_sentinel(&bcf->groups);
+         q = ngx_queue_next(q))
+    {
+        grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+
+        p = ngx_sprintf(p, "%ud", grp->map_fd);
+
+        *p++ = NGX_QUIC_BPF_ADDRSEP;
+
+        p += ngx_sock_ntop(grp->sockaddr, grp->socklen, p,
+                           NGX_SOCKADDR_STRLEN, 1);
+
+        *p++ = NGX_QUIC_BPF_VARSEP;
+    }
+
+    *p = '\0';
+
+    var = ngx_array_push(&ccf->env);
+    if (var == NULL) {
+        return NGX_ERROR;
+    }
+
+    var->data = buf;
+    var->len = sizeof(NGX_QUIC_BPF_VARNAME) - 1;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_quic_bpf_import_maps(ngx_cycle_t *cycle)
+{
+    int                     s;
+    u_char                 *inherited, *p, *v;
+    ngx_uint_t              in_fd;
+    ngx_addr_t              tmp;
+    ngx_quic_bpf_conf_t    *bcf;
+    ngx_quic_sock_group_t  *grp;
+
+    inherited = (u_char *) getenv(NGX_QUIC_BPF_VARNAME);
+
+    if (inherited == NULL) {
+        return NGX_OK;
+    }
+
+    bcf = ngx_quic_bpf_get_conf(cycle);
+
+#if (NGX_SUPPRESS_WARN)
+    s = -1;
+#endif
+
+    in_fd = 1;
+
+    for (p = inherited, v = p; *p; p++) {
+
+        switch (*p) {
+
+        case NGX_QUIC_BPF_ADDRSEP:
+
+            if (!in_fd) {
+                ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                              "quic bpf failed to parse inherited env");
+                return NGX_ERROR;
+            }
+            in_fd = 0;
+
+            s = ngx_atoi(v, p - v);
+            if (s == NGX_ERROR) {
+                ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                              "quic bpf failed to parse inherited map fd");
+                return NGX_ERROR;
+            }
+
+            v = p + 1;
+            break;
+
+        case NGX_QUIC_BPF_VARSEP:
+
+            if (in_fd) {
+                ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                              "quic bpf failed to parse inherited env");
+                return NGX_ERROR;
+            }
+            in_fd = 1;
+
+            grp = ngx_pcalloc(cycle->pool,
+                              sizeof(ngx_quic_sock_group_t));
+            if (grp == NULL) {
+                return NGX_ERROR;
+            }
+
+            grp->map_fd = s;
+
+            if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v)
+                != NGX_OK)
+            {
+                ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                              "quic bpf failed to parse inherited"
+                              " address '%*s'", p - v , v);
+
+                ngx_quic_bpf_close(cycle->log, s, "inherited map");
+
+                return NGX_ERROR;
+            }
+
+            grp->sockaddr = tmp.sockaddr;
+            grp->socklen = tmp.socklen;
+
+            grp->unused = 1;
+
+            ngx_queue_insert_tail(&bcf->groups, &grp->queue);
+
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+                           "quic bpf sockmap inherited with "
+                           "fd:%d address:%*s",
+                           grp->map_fd, p - v, v);
+            v = p + 1;
+            break;
+
+        default:
+            break;
+        }
+    }
+
+    return NGX_OK;
+}

nginx/src/event/quic/ngx_event_quic_bpf_code.c

@@ -0,0 +1,93 @@
+/* AUTO-GENERATED, DO NOT EDIT. */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "ngx_bpf.h"
+
+
+static ngx_bpf_reloc_t bpf_reloc_prog_ngx_quic_reuseport_helper[] = {
+    { "ngx_quic_sockmap", 59 },
+};
+
+static struct bpf_insn bpf_insn_prog_ngx_quic_reuseport_helper[] = {
+    /* opcode dst          src         offset imm */
+    { 0x79,   BPF_REG_2,   BPF_REG_1, (int16_t)      0,        0x0 },
+    { 0x79,   BPF_REG_3,   BPF_REG_1, (int16_t)      8,        0x0 },
+    { 0xbf,   BPF_REG_6,   BPF_REG_2, (int16_t)      0,        0x0 },
+    {  0x7,   BPF_REG_6,   BPF_REG_0, (int16_t)      0,        0x8 },
+    { 0x2d,   BPF_REG_6,   BPF_REG_3, (int16_t)     58,        0x0 },
+    { 0xbf,   BPF_REG_4,   BPF_REG_2, (int16_t)      0,        0x0 },
+    {  0x7,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0x9 },
+    { 0x2d,   BPF_REG_4,   BPF_REG_3, (int16_t)     55,        0x0 },
+    { 0xb7,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0x8 },
+    { 0xb7,   BPF_REG_5,   BPF_REG_0, (int16_t)      0,       0x14 },
+    { 0xb7,   BPF_REG_0,   BPF_REG_0, (int16_t)      0,        0x9 },
+    { 0x71,   BPF_REG_6,   BPF_REG_6, (int16_t)      0,        0x0 },
+    { 0x67,   BPF_REG_6,   BPF_REG_0, (int16_t)      0,       0x38 },
+    { 0xc7,   BPF_REG_6,   BPF_REG_0, (int16_t)      0,       0x38 },
+    { 0x65,   BPF_REG_6,   BPF_REG_0, (int16_t)     11, 0xffffffff },
+    { 0xbf,   BPF_REG_5,   BPF_REG_2, (int16_t)      0,        0x0 },
+    {  0x7,   BPF_REG_5,   BPF_REG_0, (int16_t)      0,        0xd },
+    { 0x2d,   BPF_REG_5,   BPF_REG_3, (int16_t)     45,        0x0 },
+    { 0xbf,   BPF_REG_4,   BPF_REG_2, (int16_t)      0,        0x0 },
+    {  0x7,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0xe },
+    { 0x2d,   BPF_REG_4,   BPF_REG_3, (int16_t)     42,        0x0 },
+    { 0xb7,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0xd },
+    { 0xb7,   BPF_REG_0,   BPF_REG_0, (int16_t)      0,        0xe },
+    { 0x71,   BPF_REG_5,   BPF_REG_5, (int16_t)      0,        0x0 },
+    { 0xb7,   BPF_REG_6,   BPF_REG_0, (int16_t)      0,        0x8 },
+    { 0x2d,   BPF_REG_6,   BPF_REG_5, (int16_t)     37,        0x0 },
+    { 0xbf,   BPF_REG_6,   BPF_REG_2, (int16_t)      0,        0x0 },
+    {  0xf,   BPF_REG_6,   BPF_REG_0, (int16_t)      0,        0x0 },
+    {  0xf,   BPF_REG_6,   BPF_REG_5, (int16_t)      0,        0x0 },
+    { 0x2d,   BPF_REG_6,   BPF_REG_3, (int16_t)     33,        0x0 },
+    {  0xf,   BPF_REG_2,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0xbf,   BPF_REG_4,   BPF_REG_2, (int16_t)      0,        0x0 },
+    {  0x7,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0x9 },
+    { 0x2d,   BPF_REG_4,   BPF_REG_3, (int16_t)     29,        0x0 },
+    { 0x71,   BPF_REG_4,   BPF_REG_2, (int16_t)      1,        0x0 },
+    { 0x67,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,       0x38 },
+    { 0x71,   BPF_REG_3,   BPF_REG_2, (int16_t)      2,        0x0 },
+    { 0x67,   BPF_REG_3,   BPF_REG_0, (int16_t)      0,       0x30 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0x71,   BPF_REG_4,   BPF_REG_2, (int16_t)      3,        0x0 },
+    { 0x67,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,       0x28 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0x71,   BPF_REG_4,   BPF_REG_2, (int16_t)      4,        0x0 },
+    { 0x67,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,       0x20 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0x71,   BPF_REG_4,   BPF_REG_2, (int16_t)      5,        0x0 },
+    { 0x67,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,       0x18 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0x71,   BPF_REG_4,   BPF_REG_2, (int16_t)      6,        0x0 },
+    { 0x67,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,       0x10 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0x71,   BPF_REG_4,   BPF_REG_2, (int16_t)      7,        0x0 },
+    { 0x67,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0x8 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_4, (int16_t)      0,        0x0 },
+    { 0x71,   BPF_REG_2,   BPF_REG_2, (int16_t)      8,        0x0 },
+    { 0x4f,   BPF_REG_3,   BPF_REG_2, (int16_t)      0,        0x0 },
+    { 0x7b,  BPF_REG_10,   BPF_REG_3, (int16_t)  65528,        0x0 },
+    { 0xbf,   BPF_REG_3,  BPF_REG_10, (int16_t)      0,        0x0 },
+    {  0x7,   BPF_REG_3,   BPF_REG_0, (int16_t)      0, 0xfffffff8 },
+    { 0x18,   BPF_REG_2,   BPF_REG_0, (int16_t)      0,        0x0 },
+    {  0x0,   BPF_REG_0,   BPF_REG_0, (int16_t)      0,        0x0 },
+    { 0xb7,   BPF_REG_4,   BPF_REG_0, (int16_t)      0,        0x0 },
+    { 0x85,   BPF_REG_0,   BPF_REG_0, (int16_t)      0,       0x52 },
+    { 0xb7,   BPF_REG_0,   BPF_REG_0, (int16_t)      0,        0x1 },
+    { 0x95,   BPF_REG_0,   BPF_REG_0, (int16_t)      0,        0x0 },
+};
+
+
+ngx_bpf_program_t ngx_quic_reuseport_helper = {
+    .relocs = bpf_reloc_prog_ngx_quic_reuseport_helper,
+    .nrelocs = sizeof(bpf_reloc_prog_ngx_quic_reuseport_helper)
+               / sizeof(bpf_reloc_prog_ngx_quic_reuseport_helper[0]),
+    .ins = bpf_insn_prog_ngx_quic_reuseport_helper,
+    .nins = sizeof(bpf_insn_prog_ngx_quic_reuseport_helper)
+            / sizeof(bpf_insn_prog_ngx_quic_reuseport_helper[0]),
+    .license = "BSD",
+    .type = BPF_PROG_TYPE_SK_REUSEPORT,
+};
+

nginx/src/event/quic/ngx_event_quic_connection.h

@@ -0,0 +1,326 @@
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_CONNECTION_H_INCLUDED_
+#define _NGX_EVENT_QUIC_CONNECTION_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+/* #define NGX_QUIC_DEBUG_PACKETS */  /* dump packet contents */
+/* #define NGX_QUIC_DEBUG_FRAMES */   /* dump frames contents */
+/* #define NGX_QUIC_DEBUG_ALLOC */    /* log frames and bufs alloc */
+/* #define NGX_QUIC_DEBUG_CRYPTO */
+
+#define NGX_QUIC_ENCRYPTION_INITIAL          0
+#define NGX_QUIC_ENCRYPTION_EARLY_DATA       1
+#define NGX_QUIC_ENCRYPTION_HANDSHAKE        2
+#define NGX_QUIC_ENCRYPTION_APPLICATION      3
+#define NGX_QUIC_ENCRYPTION_LAST             4
+
+#define NGX_QUIC_SEND_CTX_LAST               (NGX_QUIC_ENCRYPTION_LAST - 1)
+
+
+typedef struct ngx_quic_connection_s  ngx_quic_connection_t;
+typedef struct ngx_quic_server_id_s   ngx_quic_server_id_t;
+typedef struct ngx_quic_client_id_s   ngx_quic_client_id_t;
+typedef struct ngx_quic_send_ctx_s    ngx_quic_send_ctx_t;
+typedef struct ngx_quic_socket_s      ngx_quic_socket_t;
+typedef struct ngx_quic_path_s        ngx_quic_path_t;
+typedef struct ngx_quic_keys_s        ngx_quic_keys_t;
+
+#if (NGX_QUIC_OPENSSL_COMPAT)
+#include <ngx_event_quic_openssl_compat.h>
+#endif
+#include <ngx_event_quic_transport.h>
+#include <ngx_event_quic_protection.h>
+#include <ngx_event_quic_frames.h>
+#include <ngx_event_quic_migration.h>
+#include <ngx_event_quic_connid.h>
+#include <ngx_event_quic_streams.h>
+#include <ngx_event_quic_ssl.h>
+#include <ngx_event_quic_tokens.h>
+#include <ngx_event_quic_ack.h>
+#include <ngx_event_quic_output.h>
+#include <ngx_event_quic_socket.h>
+
+
+/* RFC 9002, 6.2.2.  Handshakes and New Paths: kInitialRtt */
+#define NGX_QUIC_INITIAL_RTT                 333 /* ms */
+
+#define NGX_QUIC_UNSET_PN                    (uint64_t) -1
+
+/*  0-RTT and 1-RTT data exist in the same packet number space,
+ *  so we have 3 packet number spaces:
+ *
+ *  0 - Initial
+ *  1 - Handshake
+ *  2 - 0-RTT and 1-RTT
+ */
+#define ngx_quic_get_send_ctx(qc, level)                                      \
+    ((level) == NGX_QUIC_ENCRYPTION_INITIAL) ? &((qc)->send_ctx[0])           \
+        : (((level) == NGX_QUIC_ENCRYPTION_HANDSHAKE) ? &((qc)->send_ctx[1])  \
+                                                      : &((qc)->send_ctx[2]))
+
+#define ngx_quic_get_connection(c)                                            \
+    (((c)->udp) ? (((ngx_quic_socket_t *)((c)->udp))->quic) : NULL)
+
+#define ngx_quic_get_socket(c)               ((ngx_quic_socket_t *)((c)->udp))
+
+#define ngx_quic_init_rtt(qc)                                                 \
+    (qc)->avg_rtt = NGX_QUIC_INITIAL_RTT;                                     \
+    (qc)->rttvar = NGX_QUIC_INITIAL_RTT / 2;                                  \
+    (qc)->min_rtt = NGX_TIMER_INFINITE;                                       \
+    (qc)->first_rtt = NGX_TIMER_INFINITE;                                     \
+    (qc)->latest_rtt = 0;
+
+
+typedef enum {
+    NGX_QUIC_PATH_IDLE = 0,
+    NGX_QUIC_PATH_VALIDATING,
+    NGX_QUIC_PATH_WAITING,
+    NGX_QUIC_PATH_MTUD
+} ngx_quic_path_state_e;
+
+
+struct ngx_quic_client_id_s {
+    ngx_queue_t                       queue;
+    uint64_t                          seqnum;
+    size_t                            len;
+    u_char                            id[NGX_QUIC_CID_LEN_MAX];
+    u_char                            sr_token[NGX_QUIC_SR_TOKEN_LEN];
+    ngx_uint_t                        used;  /* unsigned  used:1; */
+};
+
+
+struct ngx_quic_server_id_s {
+    uint64_t                          seqnum;
+    size_t                            len;
+    u_char                            id[NGX_QUIC_CID_LEN_MAX];
+};
+
+
+struct ngx_quic_path_s {
+    ngx_queue_t                       queue;
+    struct sockaddr                  *sockaddr;
+    ngx_sockaddr_t                    sa;
+    socklen_t                         socklen;
+    ngx_quic_client_id_t             *cid;
+    ngx_quic_path_state_e             state;
+    ngx_msec_t                        expires;
+    ngx_uint_t                        tries;
+    ngx_uint_t                        tag;
+    size_t                            mtu;
+    size_t                            mtud;
+    size_t                            max_mtu;
+    off_t                             sent;
+    off_t                             received;
+    u_char                            challenge[2][8];
+    uint64_t                          seqnum;
+    uint64_t                          mtu_pnum[NGX_QUIC_PATH_RETRIES];
+    ngx_str_t                         addr_text;
+    u_char                            text[NGX_SOCKADDR_STRLEN];
+    unsigned                          validated:1;
+    unsigned                          mtu_unvalidated:1;
+};
+
+
+struct ngx_quic_socket_s {
+    ngx_udp_connection_t              udp;
+    ngx_quic_connection_t            *quic;
+    ngx_queue_t                       queue;
+    ngx_quic_server_id_t              sid;
+    ngx_sockaddr_t                    sockaddr;
+    socklen_t                         socklen;
+    ngx_uint_t                        used; /* unsigned  used:1; */
+};
+
+
+typedef struct {
+    ngx_rbtree_t                      tree;
+    ngx_rbtree_node_t                 sentinel;
+
+    ngx_queue_t                       uninitialized;
+    ngx_queue_t                       free;
+
+    uint64_t                          sent;
+    uint64_t                          recv_offset;
+    uint64_t                          recv_window;
+    uint64_t                          recv_last;
+    uint64_t                          recv_max_data;
+    uint64_t                          send_offset;
+    uint64_t                          send_max_data;
+
+    uint64_t                          server_max_streams_uni;
+    uint64_t                          server_max_streams_bidi;
+    uint64_t                          server_streams_uni;
+    uint64_t                          server_streams_bidi;
+
+    uint64_t                          client_max_streams_uni;
+    uint64_t                          client_max_streams_bidi;
+    uint64_t                          client_streams_uni;
+    uint64_t                          client_streams_bidi;
+
+    ngx_uint_t                        initialized;
+                                                 /* unsigned  initialized:1; */
+} ngx_quic_streams_t;
+
+
+typedef struct {
+    size_t                            in_flight;
+    size_t                            window;
+    size_t                            ssthresh;
+    size_t                            w_max;
+    size_t                            w_est;
+    size_t                            w_prior;
+    size_t                            mtu;
+    ngx_msec_t                        recovery_start;
+    ngx_msec_t                        idle_start;
+    ngx_msec_t                        k;
+    ngx_uint_t                        idle; /* unsigned  idle:1; */
+} ngx_quic_congestion_t;
+
+
+/*
+ * RFC 9000, 12.3.  Packet Numbers
+ *
+ *  Conceptually, a packet number space is the context in which a packet
+ *  can be processed and acknowledged.  Initial packets can only be sent
+ *  with Initial packet protection keys and acknowledged in packets that
+ *  are also Initial packets.
+ */
+struct ngx_quic_send_ctx_s {
+    ngx_uint_t                        level;
+
+    ngx_quic_buffer_t                 crypto;
+    uint64_t                          crypto_sent;
+
+    uint64_t                          pnum;        /* to be sent */
+    uint64_t                          largest_ack; /* received from peer */
+    uint64_t                          largest_pn;  /* received from peer */
+
+    ngx_queue_t                       frames;      /* generated frames */
+    ngx_queue_t                       sending;     /* frames assigned to pkt */
+    ngx_queue_t                       sent;        /* frames waiting ACK */
+
+    uint64_t                          pending_ack; /* non sent ack-eliciting */
+    uint64_t                          largest_range;
+    uint64_t                          first_range;
+    ngx_msec_t                        largest_received;
+    ngx_msec_t                        ack_delay_start;
+    ngx_uint_t                        nranges;
+    ngx_quic_ack_range_t              ranges[NGX_QUIC_MAX_RANGES];
+    ngx_uint_t                        send_ack;
+};
+
+
+struct ngx_quic_connection_s {
+    uint32_t                          version;
+
+    ngx_quic_path_t                  *path;
+
+    ngx_queue_t                       sockets;
+    ngx_queue_t                       paths;
+    ngx_queue_t                       client_ids;
+    ngx_queue_t                       free_sockets;
+    ngx_queue_t                       free_paths;
+    ngx_queue_t                       free_client_ids;
+
+    ngx_uint_t                        nsockets;
+    ngx_uint_t                        nclient_ids;
+    uint64_t                          max_retired_seqnum;
+    uint64_t                          client_seqnum;
+    uint64_t                          server_seqnum;
+    uint64_t                          path_seqnum;
+
+    ngx_quic_tp_t                     tp;
+    ngx_quic_tp_t                     ctp;
+
+    ngx_quic_send_ctx_t               send_ctx[NGX_QUIC_SEND_CTX_LAST];
+
+    ngx_quic_keys_t                  *keys;
+
+    ngx_quic_conf_t                  *conf;
+
+    ngx_event_t                       push;
+    ngx_event_t                       pto;
+    ngx_event_t                       close;
+    ngx_event_t                       path_validation;
+    ngx_event_t                       key_update;
+
+    ngx_msec_t                        last_cc;
+
+    ngx_msec_t                        first_rtt;
+    ngx_msec_t                        latest_rtt;
+    ngx_msec_t                        avg_rtt;
+    ngx_msec_t                        min_rtt;
+    ngx_msec_t                        rttvar;
+
+    ngx_uint_t                        pto_count;
+
+    ngx_queue_t                       free_frames;
+    ngx_buf_t                        *free_bufs;
+    ngx_buf_t                        *free_shadow_bufs;
+
+    ngx_uint_t                        nframes;
+    ngx_uint_t                        max_frames;
+#ifdef NGX_QUIC_DEBUG_ALLOC
+    ngx_uint_t                        nbufs;
+    ngx_uint_t                        nshadowbufs;
+#endif
+
+#if (NGX_QUIC_OPENSSL_COMPAT)
+    ngx_quic_compat_t                *compat;
+#endif
+
+    ngx_quic_streams_t                streams;
+    ngx_quic_congestion_t             congestion;
+
+    uint64_t                          rst_pnum;    /* first on validated path */
+
+    off_t                             received;
+
+    ngx_uint_t                        error;
+    ngx_uint_t                        error_level;
+    ngx_uint_t                        error_ftype;
+    const char                       *error_reason;
+
+    ngx_uint_t                        shutdown_code;
+    const char                       *shutdown_reason;
+
+    unsigned                          error_app:1;
+    unsigned                          send_timer_set:1;
+    unsigned                          closing:1;
+    unsigned                          shutdown:1;
+    unsigned                          draining:1;
+    unsigned                          key_phase:1;
+    unsigned                          validated:1;
+    unsigned                          client_tp_done:1;
+
+#if (NGX_QUIC_OPENSSL_API)
+    unsigned                          read_level:2;
+    unsigned                          write_level:2;
+#endif
+};
+
+
+ngx_int_t ngx_quic_apply_transport_params(ngx_connection_t *c,
+    ngx_quic_tp_t *ctp);
+void ngx_quic_discard_ctx(ngx_connection_t *c, ngx_uint_t level);
+void ngx_quic_close_connection(ngx_connection_t *c, ngx_int_t rc);
+void ngx_quic_shutdown_quic(ngx_connection_t *c);
+void ngx_quic_address_hash(struct sockaddr *sockaddr, socklen_t socklen,
+    ngx_uint_t no_port, u_char *salt, size_t saltlen, u_char buf[20]);
+
+#if (NGX_DEBUG)
+void ngx_quic_connstate_dbg(ngx_connection_t *c);
+#else
+#define ngx_quic_connstate_dbg(c)
+#endif
+
+#endif /* _NGX_EVENT_QUIC_CONNECTION_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_connid.c

@@ -0,0 +1,502 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+#define NGX_QUIC_MAX_SERVER_IDS   8
+
+
+#if (NGX_QUIC_BPF)
+static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id);
+#endif
+static ngx_int_t ngx_quic_retire_client_id(ngx_connection_t *c,
+    ngx_quic_client_id_t *cid);
+static ngx_quic_client_id_t *ngx_quic_alloc_client_id(ngx_connection_t *c,
+    ngx_quic_connection_t *qc);
+static ngx_int_t ngx_quic_send_server_id(ngx_connection_t *c,
+    ngx_quic_server_id_t *sid);
+
+
+ngx_int_t
+ngx_quic_create_server_id(ngx_connection_t *c, u_char *id)
+{
+    if (RAND_bytes(id, NGX_QUIC_SERVER_CID_LEN) != 1) {
+        return NGX_ERROR;
+    }
+
+#if (NGX_QUIC_BPF)
+    if (ngx_quic_bpf_attach_id(c, id) != NGX_OK) {
+        ngx_log_error(NGX_LOG_ERR, c->log, 0,
+                      "quic bpf failed to generate socket key");
+        /* ignore error, things still may work */
+    }
+#endif
+
+    return NGX_OK;
+}
+
+
+#if (NGX_QUIC_BPF)
+
+static ngx_int_t
+ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id)
+{
+    int        fd;
+    uint64_t   cookie;
+    socklen_t  optlen;
+
+    fd = c->listening->fd;
+
+    optlen = sizeof(cookie);
+
+    if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) {
+        ngx_log_error(NGX_LOG_ERR, c->log, ngx_socket_errno,
+                      "quic getsockopt(SO_COOKIE) failed");
+
+        return NGX_ERROR;
+    }
+
+    ngx_quic_dcid_encode_key(id, cookie);
+
+    return NGX_OK;
+}
+
+#endif
+
+
+ngx_int_t
+ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c,
+    ngx_quic_new_conn_id_frame_t *f)
+{
+    ngx_str_t               id;
+    ngx_queue_t            *q;
+    ngx_quic_frame_t       *frame;
+    ngx_quic_client_id_t   *cid, *item;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (f->seqnum < qc->max_retired_seqnum) {
+        /*
+         * RFC 9000, 19.15.  NEW_CONNECTION_ID Frame
+         *
+         *  An endpoint that receives a NEW_CONNECTION_ID frame with
+         *  a sequence number smaller than the Retire Prior To field
+         *  of a previously received NEW_CONNECTION_ID frame MUST send
+         *  a corresponding RETIRE_CONNECTION_ID frame that retires
+         *  the newly received connection ID, unless it has already
+         *  done so for that sequence number.
+         */
+
+        frame = ngx_quic_alloc_frame(c);
+        if (frame == NULL) {
+            return NGX_ERROR;
+        }
+
+        frame->level = NGX_QUIC_ENCRYPTION_APPLICATION;
+        frame->type = NGX_QUIC_FT_RETIRE_CONNECTION_ID;
+        frame->u.retire_cid.sequence_number = f->seqnum;
+
+        ngx_quic_queue_frame(qc, frame);
+
+        goto retire;
+    }
+
+    cid = NULL;
+
+    for (q = ngx_queue_head(&qc->client_ids);
+         q != ngx_queue_sentinel(&qc->client_ids);
+         q = ngx_queue_next(q))
+    {
+        item = ngx_queue_data(q, ngx_quic_client_id_t, queue);
+
+        if (item->seqnum == f->seqnum) {
+            cid = item;
+            break;
+        }
+    }
+
+    if (cid) {
+        /*
+         * Transmission errors, timeouts, and retransmissions might cause the
+         * same NEW_CONNECTION_ID frame to be received multiple times.
+         */
+
+        if (cid->len != f->len
+            || ngx_strncmp(cid->id, f->cid, f->len) != 0
+            || ngx_strncmp(cid->sr_token, f->srt, NGX_QUIC_SR_TOKEN_LEN) != 0)
+        {
+            /*
+             * ..if a sequence number is used for different connection IDs,
+             * the endpoint MAY treat that receipt as a connection error
+             * of type PROTOCOL_VIOLATION.
+             */
+            qc->error = NGX_QUIC_ERR_PROTOCOL_VIOLATION;
+            qc->error_reason = "seqnum refers to different connection id/token";
+            return NGX_ERROR;
+        }
+
+    } else {
+
+        id.data = f->cid;
+        id.len = f->len;
+
+        if (ngx_quic_create_client_id(c, &id, f->seqnum, f->srt) == NULL) {
+            return NGX_ERROR;
+        }
+    }
+
+retire:
+
+    if (qc->max_retired_seqnum && f->retire <= qc->max_retired_seqnum) {
+        /*
+         * Once a sender indicates a Retire Prior To value, smaller values sent
+         * in subsequent NEW_CONNECTION_ID frames have no effect.  A receiver
+         * MUST ignore any Retire Prior To fields that do not increase the
+         * largest received Retire Prior To value.
+         */
+        goto done;
+    }
+
+    qc->max_retired_seqnum = f->retire;
+
+    q = ngx_queue_head(&qc->client_ids);
+
+    while (q != ngx_queue_sentinel(&qc->client_ids)) {
+
+        cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
+        q = ngx_queue_next(q);
+
+        if (cid->seqnum >= f->retire) {
+            continue;
+        }
+
+        if (ngx_quic_retire_client_id(c, cid) != NGX_OK) {
+            return NGX_ERROR;
+        }
+    }
+
+done:
+
+    if (qc->nclient_ids > qc->tp.active_connection_id_limit) {
+        /*
+         * RFC 9000, 5.1.1.  Issuing Connection IDs
+         *
+         * After processing a NEW_CONNECTION_ID frame and
+         * adding and retiring active connection IDs, if the number of active
+         * connection IDs exceeds the value advertised in its
+         * active_connection_id_limit transport parameter, an endpoint MUST
+         * close the connection with an error of type CONNECTION_ID_LIMIT_ERROR.
+         */
+        qc->error = NGX_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR;
+        qc->error_reason = "too many connection ids received";
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_quic_retire_client_id(ngx_connection_t *c, ngx_quic_client_id_t *cid)
+{
+    ngx_queue_t            *q;
+    ngx_quic_path_t        *path;
+    ngx_quic_client_id_t   *new_cid;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (!cid->used) {
+        return ngx_quic_free_client_id(c, cid);
+    }
+
+    /* we are going to retire client id which is in use */
+
+    q = ngx_queue_head(&qc->paths);
+
+    while (q != ngx_queue_sentinel(&qc->paths)) {
+
+        path = ngx_queue_data(q, ngx_quic_path_t, queue);
+        q = ngx_queue_next(q);
+
+        if (path->cid != cid) {
+            continue;
+        }
+
+        if (path == qc->path) {
+            /* this is the active path: update it with new CID */
+            new_cid = ngx_quic_next_client_id(c);
+            if (new_cid == NULL) {
+                return NGX_ERROR;
+            }
+
+            qc->path->cid = new_cid;
+            new_cid->used = 1;
+
+            return ngx_quic_free_client_id(c, cid);
+        }
+
+        return ngx_quic_free_path(c, path);
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_quic_client_id_t *
+ngx_quic_alloc_client_id(ngx_connection_t *c, ngx_quic_connection_t *qc)
+{
+    ngx_queue_t           *q;
+    ngx_quic_client_id_t  *cid;
+
+    if (!ngx_queue_empty(&qc->free_client_ids)) {
+
+        q = ngx_queue_head(&qc->free_client_ids);
+        cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
+
+        ngx_queue_remove(&cid->queue);
+
+        ngx_memzero(cid, sizeof(ngx_quic_client_id_t));
+
+    } else {
+
+        cid = ngx_pcalloc(c->pool, sizeof(ngx_quic_client_id_t));
+        if (cid == NULL) {
+            return NULL;
+        }
+    }
+
+    return cid;
+}
+
+
+ngx_quic_client_id_t *
+ngx_quic_create_client_id(ngx_connection_t *c, ngx_str_t *id,
+    uint64_t seqnum, u_char *token)
+{
+    ngx_quic_client_id_t   *cid;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    cid = ngx_quic_alloc_client_id(c, qc);
+    if (cid == NULL) {
+        return NULL;
+    }
+
+    cid->seqnum = seqnum;
+
+    cid->len = id->len;
+    ngx_memcpy(cid->id, id->data, id->len);
+
+    if (token) {
+        ngx_memcpy(cid->sr_token, token, NGX_QUIC_SR_TOKEN_LEN);
+    }
+
+    ngx_queue_insert_tail(&qc->client_ids, &cid->queue);
+    qc->nclient_ids++;
+
+    if (seqnum > qc->client_seqnum) {
+        qc->client_seqnum = seqnum;
+    }
+
+    ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic cid seq:%uL received id:%uz:%xV:%*xs",
+                    cid->seqnum, id->len, id,
+                    (size_t) NGX_QUIC_SR_TOKEN_LEN, cid->sr_token);
+
+    return cid;
+}
+
+
+ngx_quic_client_id_t *
+ngx_quic_next_client_id(ngx_connection_t *c)
+{
+    ngx_queue_t            *q;
+    ngx_quic_client_id_t   *cid;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    for (q = ngx_queue_head(&qc->client_ids);
+         q != ngx_queue_sentinel(&qc->client_ids);
+         q = ngx_queue_next(q))
+    {
+        cid = ngx_queue_data(q, ngx_quic_client_id_t, queue);
+
+        if (!cid->used) {
+            return cid;
+        }
+    }
+
+    return NULL;
+}
+
+
+ngx_int_t
+ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c,
+    ngx_quic_retire_cid_frame_t *f)
+{
+    ngx_quic_socket_t      *qsock;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (f->sequence_number >= qc->server_seqnum) {
+        /*
+         * RFC 9000, 19.16.
+         *
+         *  Receipt of a RETIRE_CONNECTION_ID frame containing a sequence
+         *  number greater than any previously sent to the peer MUST be
+         *  treated as a connection error of type PROTOCOL_VIOLATION.
+         */
+        qc->error = NGX_QUIC_ERR_PROTOCOL_VIOLATION;
+        qc->error_reason = "sequence number of id to retire was never issued";
+
+        return NGX_ERROR;
+    }
+
+    qsock = ngx_quic_get_socket(c);
+
+    if (qsock->sid.seqnum == f->sequence_number) {
+
+        /*
+         * RFC 9000, 19.16.
+         *
+         * The sequence number specified in a RETIRE_CONNECTION_ID frame MUST
+         * NOT refer to the Destination Connection ID field of the packet in
+         * which the frame is contained.  The peer MAY treat this as a
+         * connection error of type PROTOCOL_VIOLATION.
+         */
+
+        qc->error = NGX_QUIC_ERR_PROTOCOL_VIOLATION;
+        qc->error_reason = "sequence number of id to retire refers DCID";
+
+        return NGX_ERROR;
+    }
+
+    qsock = ngx_quic_find_socket(c, f->sequence_number);
+    if (qsock == NULL) {
+        return NGX_OK;
+    }
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic socket seq:%uL is retired", qsock->sid.seqnum);
+
+    ngx_quic_close_socket(c, qsock);
+
+    /* restore socket count up to a limit after deletion */
+    if (ngx_quic_create_sockets(c) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_create_sockets(ngx_connection_t *c)
+{
+    ngx_uint_t              n;
+    ngx_quic_socket_t      *qsock;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    n = ngx_min(NGX_QUIC_MAX_SERVER_IDS, qc->ctp.active_connection_id_limit);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic create sockets has:%ui max:%ui", qc->nsockets, n);
+
+    while (qc->nsockets < n) {
+
+        qsock = ngx_quic_create_socket(c, qc);
+        if (qsock == NULL) {
+            return NGX_ERROR;
+        }
+
+        if (ngx_quic_listen(c, qc, qsock) != NGX_OK) {
+            return NGX_ERROR;
+        }
+
+        if (ngx_quic_send_server_id(c, &qsock->sid) != NGX_OK) {
+            return NGX_ERROR;
+        }
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_quic_send_server_id(ngx_connection_t *c, ngx_quic_server_id_t *sid)
+{
+    ngx_str_t               dcid;
+    ngx_quic_frame_t       *frame;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    dcid.len = sid->len;
+    dcid.data = sid->id;
+
+    frame = ngx_quic_alloc_frame(c);
+    if (frame == NULL) {
+        return NGX_ERROR;
+    }
+
+    frame->level = NGX_QUIC_ENCRYPTION_APPLICATION;
+    frame->type = NGX_QUIC_FT_NEW_CONNECTION_ID;
+    frame->u.ncid.seqnum = sid->seqnum;
+    frame->u.ncid.retire = 0;
+    frame->u.ncid.len = NGX_QUIC_SERVER_CID_LEN;
+    ngx_memcpy(frame->u.ncid.cid, sid->id, NGX_QUIC_SERVER_CID_LEN);
+
+    if (ngx_quic_new_sr_token(c, &dcid, qc->conf->sr_token_key,
+                              frame->u.ncid.srt)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
+    ngx_quic_queue_frame(qc, frame);
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_free_client_id(ngx_connection_t *c, ngx_quic_client_id_t *cid)
+{
+    ngx_quic_frame_t       *frame;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    frame = ngx_quic_alloc_frame(c);
+    if (frame == NULL) {
+        return NGX_ERROR;
+    }
+
+    frame->level = NGX_QUIC_ENCRYPTION_APPLICATION;
+    frame->type = NGX_QUIC_FT_RETIRE_CONNECTION_ID;
+    frame->u.retire_cid.sequence_number = cid->seqnum;
+
+    ngx_quic_queue_frame(qc, frame);
+
+    /* we are no longer going to use this client id */
+
+    ngx_queue_remove(&cid->queue);
+    ngx_queue_insert_head(&qc->free_client_ids, &cid->queue);
+
+    qc->nclient_ids--;
+
+    return NGX_OK;
+}

nginx/src/event/quic/ngx_event_quic_connid.h

@@ -0,0 +1,29 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_CONNID_H_INCLUDED_
+#define _NGX_EVENT_QUIC_CONNID_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+ngx_int_t ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c,
+    ngx_quic_retire_cid_frame_t *f);
+ngx_int_t ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c,
+    ngx_quic_new_conn_id_frame_t *f);
+
+ngx_int_t ngx_quic_create_sockets(ngx_connection_t *c);
+ngx_int_t ngx_quic_create_server_id(ngx_connection_t *c, u_char *id);
+
+ngx_quic_client_id_t *ngx_quic_create_client_id(ngx_connection_t *c,
+    ngx_str_t *id, uint64_t seqnum, u_char *token);
+ngx_quic_client_id_t *ngx_quic_next_client_id(ngx_connection_t *c);
+ngx_int_t ngx_quic_free_client_id(ngx_connection_t *c,
+    ngx_quic_client_id_t *cid);
+
+#endif /* _NGX_EVENT_QUIC_CONNID_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_frames.c

@@ -0,0 +1,895 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+
+#define NGX_QUIC_BUFFER_SIZE  4096
+
+#define ngx_quic_buf_refs(b)         (b)->shadow->num
+#define ngx_quic_buf_inc_refs(b)     ngx_quic_buf_refs(b)++
+#define ngx_quic_buf_dec_refs(b)     ngx_quic_buf_refs(b)--
+#define ngx_quic_buf_set_refs(b, v)  ngx_quic_buf_refs(b) = v
+
+
+static ngx_buf_t *ngx_quic_alloc_buf(ngx_connection_t *c);
+static void ngx_quic_free_buf(ngx_connection_t *c, ngx_buf_t *b);
+static ngx_buf_t *ngx_quic_clone_buf(ngx_connection_t *c, ngx_buf_t *b);
+static ngx_int_t ngx_quic_split_chain(ngx_connection_t *c, ngx_chain_t *cl,
+    off_t offset);
+
+
+static ngx_buf_t *
+ngx_quic_alloc_buf(ngx_connection_t *c)
+{
+    u_char                 *p;
+    ngx_buf_t              *b;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    b = qc->free_bufs;
+
+    if (b) {
+        qc->free_bufs = b->shadow;
+        p = b->start;
+
+    } else {
+        b = qc->free_shadow_bufs;
+
+        if (b) {
+            qc->free_shadow_bufs = b->shadow;
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "quic use shadow buffer n:%ui %ui",
+                           ++qc->nbufs, --qc->nshadowbufs);
+#endif
+
+        } else {
+            b = ngx_palloc(c->pool, sizeof(ngx_buf_t));
+            if (b == NULL) {
+                return NULL;
+            }
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+            ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "quic new buffer n:%ui", ++qc->nbufs);
+#endif
+        }
+
+        p = ngx_pnalloc(c->pool, NGX_QUIC_BUFFER_SIZE);
+        if (p == NULL) {
+            return NULL;
+        }
+    }
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic alloc buffer %p", b);
+#endif
+
+    ngx_memzero(b, sizeof(ngx_buf_t));
+
+    b->tag = (ngx_buf_tag_t) &ngx_quic_alloc_buf;
+    b->temporary = 1;
+    b->shadow = b;
+
+    b->start = p;
+    b->pos = p;
+    b->last = p;
+    b->end = p + NGX_QUIC_BUFFER_SIZE;
+
+    ngx_quic_buf_set_refs(b, 1);
+
+    return b;
+}
+
+
+static void
+ngx_quic_free_buf(ngx_connection_t *c, ngx_buf_t *b)
+{
+    ngx_buf_t              *shadow;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    ngx_quic_buf_dec_refs(b);
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic free buffer %p r:%ui",
+                   b, (ngx_uint_t) ngx_quic_buf_refs(b));
+#endif
+
+    shadow = b->shadow;
+
+    if (ngx_quic_buf_refs(b) == 0) {
+        shadow->shadow = qc->free_bufs;
+        qc->free_bufs = shadow;
+    }
+
+    if (b != shadow) {
+        b->shadow = qc->free_shadow_bufs;
+        qc->free_shadow_bufs = b;
+    }
+
+}
+
+
+static ngx_buf_t *
+ngx_quic_clone_buf(ngx_connection_t *c, ngx_buf_t *b)
+{
+    ngx_buf_t              *nb;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    nb = qc->free_shadow_bufs;
+
+    if (nb) {
+        qc->free_shadow_bufs = nb->shadow;
+
+    } else {
+        nb = ngx_palloc(c->pool, sizeof(ngx_buf_t));
+        if (nb == NULL) {
+            return NULL;
+        }
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic new shadow buffer n:%ui", ++qc->nshadowbufs);
+#endif
+    }
+
+    *nb = *b;
+
+    ngx_quic_buf_inc_refs(b);
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic clone buffer %p %p r:%ui",
+                   b, nb, (ngx_uint_t) ngx_quic_buf_refs(b));
+#endif
+
+    return nb;
+}
+
+
+static ngx_int_t
+ngx_quic_split_chain(ngx_connection_t *c, ngx_chain_t *cl, off_t offset)
+{
+    ngx_buf_t    *b, *tb;
+    ngx_chain_t  *tail;
+
+    b = cl->buf;
+
+    tail = ngx_alloc_chain_link(c->pool);
+    if (tail == NULL) {
+        return NGX_ERROR;
+    }
+
+    tb = ngx_quic_clone_buf(c, b);
+    if (tb == NULL) {
+        return NGX_ERROR;
+    }
+
+    tail->buf = tb;
+
+    tb->pos += offset;
+
+    b->last = tb->pos;
+    b->last_buf = 0;
+
+    tail->next = cl->next;
+    cl->next = tail;
+
+    return NGX_OK;
+}
+
+
+ngx_quic_frame_t *
+ngx_quic_alloc_frame(ngx_connection_t *c)
+{
+    ngx_queue_t            *q;
+    ngx_quic_frame_t       *frame;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (!ngx_queue_empty(&qc->free_frames)) {
+
+        q = ngx_queue_head(&qc->free_frames);
+        frame = ngx_queue_data(q, ngx_quic_frame_t, queue);
+
+        ngx_queue_remove(&frame->queue);
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic reuse frame n:%ui", qc->nframes);
+#endif
+
+    } else if (qc->nframes < qc->max_frames) {
+        frame = ngx_palloc(c->pool, sizeof(ngx_quic_frame_t));
+        if (frame == NULL) {
+            return NULL;
+        }
+
+        ++qc->nframes;
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic alloc frame n:%ui", qc->nframes);
+#endif
+
+    } else {
+        ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic flood detected");
+        return NULL;
+    }
+
+    ngx_memzero(frame, sizeof(ngx_quic_frame_t));
+
+    return frame;
+}
+
+
+void
+ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame)
+{
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (frame->data) {
+        ngx_quic_free_chain(c, frame->data);
+    }
+
+    ngx_queue_insert_head(&qc->free_frames, &frame->queue);
+
+#ifdef NGX_QUIC_DEBUG_ALLOC
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic free frame n:%ui", qc->nframes);
+#endif
+}
+
+
+void
+ngx_quic_free_chain(ngx_connection_t *c, ngx_chain_t *in)
+{
+    ngx_chain_t  *cl;
+
+    while (in) {
+        cl = in;
+        in = in->next;
+
+        ngx_quic_free_buf(c, cl->buf);
+        ngx_free_chain(c->pool, cl);
+    }
+}
+
+
+void
+ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames)
+{
+    ngx_queue_t       *q;
+    ngx_quic_frame_t  *f;
+
+    do {
+        q = ngx_queue_head(frames);
+
+        if (q == ngx_queue_sentinel(frames)) {
+            break;
+        }
+
+        ngx_queue_remove(q);
+
+        f = ngx_queue_data(q, ngx_quic_frame_t, queue);
+
+        ngx_quic_free_frame(c, f);
+    } while (1);
+}
+
+
+void
+ngx_quic_queue_frame(ngx_quic_connection_t *qc, ngx_quic_frame_t *frame)
+{
+    ngx_quic_send_ctx_t  *ctx;
+
+    ctx = ngx_quic_get_send_ctx(qc, frame->level);
+
+    ngx_queue_insert_tail(&ctx->frames, &frame->queue);
+
+    frame->len = ngx_quic_create_frame(NULL, frame);
+    /* always succeeds */
+
+    if (qc->closing) {
+        return;
+    }
+
+    ngx_post_event(&qc->push, &ngx_posted_events);
+}
+
+
+ngx_int_t
+ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f, size_t len)
+{
+    size_t                     shrink;
+    ngx_chain_t               *out;
+    ngx_quic_frame_t          *nf;
+    ngx_quic_buffer_t          qb;
+    ngx_quic_ordered_frame_t  *of, *onf;
+
+    switch (f->type) {
+    case NGX_QUIC_FT_CRYPTO:
+    case NGX_QUIC_FT_STREAM:
+        break;
+
+    default:
+        return NGX_DECLINED;
+    }
+
+    if ((size_t) f->len <= len) {
+        return NGX_OK;
+    }
+
+    shrink = f->len - len;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic split frame now:%uz need:%uz shrink:%uz",
+                   f->len, len, shrink);
+
+    of = &f->u.ord;
+
+    if (of->length <= shrink) {
+        return NGX_DECLINED;
+    }
+
+    of->length -= shrink;
+    f->len = ngx_quic_create_frame(NULL, f);
+
+    if ((size_t) f->len > len) {
+        ngx_log_error(NGX_LOG_ERR, c->log, 0, "could not split QUIC frame");
+        return NGX_ERROR;
+    }
+
+    ngx_memzero(&qb, sizeof(ngx_quic_buffer_t));
+    qb.chain = f->data;
+
+    out = ngx_quic_read_buffer(c, &qb, of->length);
+    if (out == NGX_CHAIN_ERROR) {
+        return NGX_ERROR;
+    }
+
+    f->data = out;
+
+    nf = ngx_quic_alloc_frame(c);
+    if (nf == NULL) {
+        return NGX_ERROR;
+    }
+
+    *nf = *f;
+    onf = &nf->u.ord;
+    onf->offset += of->length;
+    onf->length = shrink;
+    nf->len = ngx_quic_create_frame(NULL, nf);
+    nf->data = qb.chain;
+
+    if (f->type == NGX_QUIC_FT_STREAM) {
+        f->u.stream.fin = 0;
+    }
+
+    ngx_queue_insert_after(&f->queue, &nf->queue);
+
+    return NGX_OK;
+}
+
+
+ngx_chain_t *
+ngx_quic_copy_buffer(ngx_connection_t *c, u_char *data, size_t len)
+{
+    ngx_buf_t          buf;
+    ngx_chain_t        cl, *out;
+    ngx_quic_buffer_t  qb;
+
+    ngx_memzero(&buf, sizeof(ngx_buf_t));
+
+    buf.pos = data;
+    buf.last = buf.pos + len;
+    buf.temporary = 1;
+
+    cl.buf = &buf;
+    cl.next = NULL;
+
+    ngx_memzero(&qb, sizeof(ngx_quic_buffer_t));
+
+    if (ngx_quic_write_buffer(c, &qb, &cl, len, 0) == NGX_CHAIN_ERROR) {
+        return NGX_CHAIN_ERROR;
+    }
+
+    out = ngx_quic_read_buffer(c, &qb, len);
+    if (out == NGX_CHAIN_ERROR) {
+        return NGX_CHAIN_ERROR;
+    }
+
+    ngx_quic_free_buffer(c, &qb);
+
+    return out;
+}
+
+
+ngx_chain_t *
+ngx_quic_read_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb, uint64_t limit)
+{
+    uint64_t      n;
+    ngx_buf_t    *b;
+    ngx_chain_t  *out, **ll;
+
+    out = qb->chain;
+
+    for (ll = &out; *ll; ll = &(*ll)->next) {
+        b = (*ll)->buf;
+
+        if (b->sync) {
+            /* hole */
+            break;
+        }
+
+        if (limit == 0) {
+            break;
+        }
+
+        n = b->last - b->pos;
+
+        if (n > limit) {
+            if (ngx_quic_split_chain(c, *ll, limit) != NGX_OK) {
+                return NGX_CHAIN_ERROR;
+            }
+
+            n = limit;
+        }
+
+        limit -= n;
+        qb->offset += n;
+    }
+
+    if (qb->offset >= qb->last_offset) {
+        qb->last_chain = NULL;
+    }
+
+    qb->chain = *ll;
+    *ll = NULL;
+
+    return out;
+}
+
+
+void
+ngx_quic_skip_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb,
+    uint64_t offset)
+{
+    size_t        n;
+    ngx_buf_t    *b;
+    ngx_chain_t  *cl;
+
+    while (qb->chain) {
+        if (qb->offset >= offset) {
+            break;
+        }
+
+        cl = qb->chain;
+        b = cl->buf;
+        n = b->last - b->pos;
+
+        if (qb->offset + n > offset) {
+            n = offset - qb->offset;
+            b->pos += n;
+            qb->offset += n;
+            break;
+        }
+
+        qb->offset += n;
+        qb->chain = cl->next;
+
+        cl->next = NULL;
+        ngx_quic_free_chain(c, cl);
+    }
+
+    if (qb->chain == NULL) {
+        qb->offset = offset;
+    }
+
+    if (qb->offset >= qb->last_offset) {
+        qb->last_chain = NULL;
+    }
+}
+
+
+ngx_chain_t *
+ngx_quic_alloc_chain(ngx_connection_t *c)
+{
+    ngx_chain_t  *cl;
+
+    cl = ngx_alloc_chain_link(c->pool);
+    if (cl == NULL) {
+        return NULL;
+    }
+
+    cl->buf = ngx_quic_alloc_buf(c);
+    if (cl->buf == NULL) {
+        return NULL;
+    }
+
+    return cl;
+}
+
+
+ngx_chain_t *
+ngx_quic_write_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb,
+    ngx_chain_t *in, uint64_t limit, uint64_t offset)
+{
+    u_char       *p;
+    uint64_t      n, base;
+    ngx_buf_t    *b;
+    ngx_chain_t  *cl, **chain;
+
+    if (qb->last_chain && offset >= qb->last_offset) {
+        base = qb->last_offset;
+        chain = &qb->last_chain;
+
+    } else {
+        base = qb->offset;
+        chain = &qb->chain;
+    }
+
+    while (in && limit) {
+
+        if (offset < base) {
+            n = ngx_min((uint64_t) (in->buf->last - in->buf->pos),
+                        ngx_min(base - offset, limit));
+
+            in->buf->pos += n;
+            offset += n;
+            limit -= n;
+
+            if (in->buf->pos == in->buf->last) {
+                in = in->next;
+            }
+
+            continue;
+        }
+
+        cl = *chain;
+
+        if (cl == NULL) {
+            cl = ngx_quic_alloc_chain(c);
+            if (cl == NULL) {
+                return NGX_CHAIN_ERROR;
+            }
+
+            cl->buf->last = cl->buf->end;
+            cl->buf->sync = 1; /* hole */
+            cl->next = NULL;
+            *chain = cl;
+        }
+
+        b = cl->buf;
+        n = b->last - b->pos;
+
+        if (base + n <= offset) {
+            base += n;
+            chain = &cl->next;
+            continue;
+        }
+
+        if (b->sync && offset > base) {
+            if (ngx_quic_split_chain(c, cl, offset - base) != NGX_OK) {
+                return NGX_CHAIN_ERROR;
+            }
+
+            continue;
+        }
+
+        p = b->pos + (offset - base);
+
+        while (in) {
+
+            if (!ngx_buf_in_memory(in->buf) || in->buf->pos == in->buf->last) {
+                in = in->next;
+                continue;
+            }
+
+            if (p == b->last || limit == 0) {
+                break;
+            }
+
+            n = ngx_min(b->last - p, in->buf->last - in->buf->pos);
+            n = ngx_min(n, limit);
+
+            if (b->sync) {
+                ngx_memcpy(p, in->buf->pos, n);
+                qb->size += n;
+            }
+
+            p += n;
+            in->buf->pos += n;
+            offset += n;
+            limit -= n;
+        }
+
+        if (b->sync && p == b->last) {
+            b->sync = 0;
+            continue;
+        }
+
+        if (b->sync && p != b->pos) {
+            if (ngx_quic_split_chain(c, cl, p - b->pos) != NGX_OK) {
+                return NGX_CHAIN_ERROR;
+            }
+
+            b->sync = 0;
+        }
+    }
+
+    qb->last_offset = base;
+    qb->last_chain = *chain;
+
+    return in;
+}
+
+
+void
+ngx_quic_free_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb)
+{
+    ngx_quic_free_chain(c, qb->chain);
+
+    qb->chain = NULL;
+    qb->last_chain = NULL;
+}
+
+
+#if (NGX_DEBUG)
+
+void
+ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx)
+{
+    u_char      *p, *last, *pos, *end;
+    ssize_t      n;
+    uint64_t     gap, range, largest, smallest;
+    ngx_uint_t   i;
+    u_char       buf[NGX_MAX_ERROR_STR];
+
+    p = buf;
+    last = buf + sizeof(buf);
+
+    switch (f->type) {
+
+    case NGX_QUIC_FT_CRYPTO:
+        p = ngx_slprintf(p, last, "CRYPTO len:%uL off:%uL",
+                         f->u.crypto.length, f->u.crypto.offset);
+
+#ifdef NGX_QUIC_DEBUG_FRAMES
+        {
+            ngx_chain_t  *cl;
+
+            p = ngx_slprintf(p, last, " data:");
+
+            for (cl = f->data; cl; cl = cl->next) {
+                p = ngx_slprintf(p, last, "%*xs",
+                                 cl->buf->last - cl->buf->pos, cl->buf->pos);
+            }
+        }
+#endif
+
+        break;
+
+    case NGX_QUIC_FT_PADDING:
+        p = ngx_slprintf(p, last, "PADDING");
+        break;
+
+    case NGX_QUIC_FT_ACK:
+    case NGX_QUIC_FT_ACK_ECN:
+
+        p = ngx_slprintf(p, last, "ACK n:%ui delay:%uL ",
+                         f->u.ack.range_count, f->u.ack.delay);
+
+        if (f->data) {
+            pos = f->data->buf->pos;
+            end = f->data->buf->last;
+
+        } else {
+            pos = NULL;
+            end = NULL;
+        }
+
+        largest = f->u.ack.largest;
+        smallest = f->u.ack.largest - f->u.ack.first_range;
+
+        if (largest == smallest) {
+            p = ngx_slprintf(p, last, "%uL", largest);
+
+        } else {
+            p = ngx_slprintf(p, last, "%uL-%uL", largest, smallest);
+        }
+
+        for (i = 0; i < f->u.ack.range_count; i++) {
+            n = ngx_quic_parse_ack_range(log, pos, end, &gap, &range);
+            if (n == NGX_ERROR) {
+                break;
+            }
+
+            pos += n;
+
+            largest = smallest - gap - 2;
+            smallest = largest - range;
+
+            if (largest == smallest) {
+                p = ngx_slprintf(p, last, " %uL", largest);
+
+            } else {
+                p = ngx_slprintf(p, last, " %uL-%uL", largest, smallest);
+            }
+        }
+
+        if (f->type == NGX_QUIC_FT_ACK_ECN) {
+            p = ngx_slprintf(p, last, " ECN counters ect0:%uL ect1:%uL ce:%uL",
+                             f->u.ack.ect0, f->u.ack.ect1, f->u.ack.ce);
+        }
+        break;
+
+    case NGX_QUIC_FT_PING:
+        p = ngx_slprintf(p, last, "PING");
+        break;
+
+    case NGX_QUIC_FT_NEW_CONNECTION_ID:
+        p = ngx_slprintf(p, last,
+                         "NEW_CONNECTION_ID seq:%uL retire:%uL len:%ud",
+                         f->u.ncid.seqnum, f->u.ncid.retire, f->u.ncid.len);
+        break;
+
+    case NGX_QUIC_FT_RETIRE_CONNECTION_ID:
+        p = ngx_slprintf(p, last, "RETIRE_CONNECTION_ID seqnum:%uL",
+                         f->u.retire_cid.sequence_number);
+        break;
+
+    case NGX_QUIC_FT_CONNECTION_CLOSE:
+    case NGX_QUIC_FT_CONNECTION_CLOSE_APP:
+        p = ngx_slprintf(p, last, "CONNECTION_CLOSE%s err:%ui",
+                         f->type == NGX_QUIC_FT_CONNECTION_CLOSE ? "" : "_APP",
+                         f->u.close.error_code);
+
+        if (f->u.close.reason.len) {
+            p = ngx_slprintf(p, last, " %V", &f->u.close.reason);
+        }
+
+        if (f->type == NGX_QUIC_FT_CONNECTION_CLOSE) {
+            p = ngx_slprintf(p, last, " ft:%ui", f->u.close.frame_type);
+        }
+
+        break;
+
+    case NGX_QUIC_FT_STREAM:
+        p = ngx_slprintf(p, last, "STREAM id:0x%xL", f->u.stream.stream_id);
+
+        if (f->u.stream.off) {
+            p = ngx_slprintf(p, last, " off:%uL", f->u.stream.offset);
+        }
+
+        if (f->u.stream.len) {
+            p = ngx_slprintf(p, last, " len:%uL", f->u.stream.length);
+        }
+
+        if (f->u.stream.fin) {
+            p = ngx_slprintf(p, last, " fin:1");
+        }
+
+#ifdef NGX_QUIC_DEBUG_FRAMES
+        {
+            ngx_chain_t  *cl;
+
+            p = ngx_slprintf(p, last, " data:");
+
+            for (cl = f->data; cl; cl = cl->next) {
+                p = ngx_slprintf(p, last, "%*xs",
+                                 cl->buf->last - cl->buf->pos, cl->buf->pos);
+            }
+        }
+#endif
+
+        break;
+
+    case NGX_QUIC_FT_MAX_DATA:
+        p = ngx_slprintf(p, last, "MAX_DATA max_data:%uL on recv",
+                         f->u.max_data.max_data);
+        break;
+
+    case NGX_QUIC_FT_RESET_STREAM:
+        p = ngx_slprintf(p, last, "RESET_STREAM"
+                        " id:0x%xL error_code:0x%xL final_size:0x%xL",
+                        f->u.reset_stream.id, f->u.reset_stream.error_code,
+                        f->u.reset_stream.final_size);
+        break;
+
+    case NGX_QUIC_FT_STOP_SENDING:
+        p = ngx_slprintf(p, last, "STOP_SENDING id:0x%xL err:0x%xL",
+                         f->u.stop_sending.id, f->u.stop_sending.error_code);
+        break;
+
+    case NGX_QUIC_FT_STREAMS_BLOCKED:
+    case NGX_QUIC_FT_STREAMS_BLOCKED2:
+        p = ngx_slprintf(p, last, "STREAMS_BLOCKED limit:%uL bidi:%ui",
+                         f->u.streams_blocked.limit, f->u.streams_blocked.bidi);
+        break;
+
+    case NGX_QUIC_FT_MAX_STREAMS:
+    case NGX_QUIC_FT_MAX_STREAMS2:
+        p = ngx_slprintf(p, last, "MAX_STREAMS limit:%uL bidi:%ui",
+                         f->u.max_streams.limit, f->u.max_streams.bidi);
+        break;
+
+    case NGX_QUIC_FT_MAX_STREAM_DATA:
+        p = ngx_slprintf(p, last, "MAX_STREAM_DATA id:0x%xL limit:%uL",
+                         f->u.max_stream_data.id, f->u.max_stream_data.limit);
+        break;
+
+
+    case NGX_QUIC_FT_DATA_BLOCKED:
+        p = ngx_slprintf(p, last, "DATA_BLOCKED limit:%uL",
+                         f->u.data_blocked.limit);
+        break;
+
+    case NGX_QUIC_FT_STREAM_DATA_BLOCKED:
+        p = ngx_slprintf(p, last, "STREAM_DATA_BLOCKED id:0x%xL limit:%uL",
+                         f->u.stream_data_blocked.id,
+                         f->u.stream_data_blocked.limit);
+        break;
+
+    case NGX_QUIC_FT_PATH_CHALLENGE:
+        p = ngx_slprintf(p, last, "PATH_CHALLENGE data:0x%*xs",
+                         sizeof(f->u.path_challenge.data),
+                         f->u.path_challenge.data);
+        break;
+
+    case NGX_QUIC_FT_PATH_RESPONSE:
+        p = ngx_slprintf(p, last, "PATH_RESPONSE data:0x%*xs",
+                         sizeof(f->u.path_challenge.data),
+                         f->u.path_challenge.data);
+        break;
+
+    case NGX_QUIC_FT_NEW_TOKEN:
+        p = ngx_slprintf(p, last, "NEW_TOKEN");
+
+#ifdef NGX_QUIC_DEBUG_FRAMES
+        {
+            ngx_chain_t  *cl;
+
+            p = ngx_slprintf(p, last, " token:");
+
+            for (cl = f->data; cl; cl = cl->next) {
+                p = ngx_slprintf(p, last, "%*xs",
+                                 cl->buf->last - cl->buf->pos, cl->buf->pos);
+            }
+        }
+#endif
+
+        break;
+
+    case NGX_QUIC_FT_HANDSHAKE_DONE:
+        p = ngx_slprintf(p, last, "HANDSHAKE DONE");
+        break;
+
+    default:
+        p = ngx_slprintf(p, last, "unknown type 0x%xi", f->type);
+        break;
+    }
+
+    ngx_log_debug5(NGX_LOG_DEBUG_EVENT, log, 0, "quic frame %s %s:%uL %*s",
+                   tx ? "tx" : "rx", ngx_quic_level_name(f->level), f->pnum,
+                   p - buf, buf);
+}
+
+#endif

nginx/src/event/quic/ngx_event_quic_frames.h

@@ -0,0 +1,45 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_FRAMES_H_INCLUDED_
+#define _NGX_EVENT_QUIC_FRAMES_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+typedef ngx_int_t (*ngx_quic_frame_handler_pt)(ngx_connection_t *c,
+    ngx_quic_frame_t *frame, void *data);
+
+
+ngx_quic_frame_t *ngx_quic_alloc_frame(ngx_connection_t *c);
+void ngx_quic_free_frame(ngx_connection_t *c, ngx_quic_frame_t *frame);
+void ngx_quic_free_frames(ngx_connection_t *c, ngx_queue_t *frames);
+void ngx_quic_queue_frame(ngx_quic_connection_t *qc, ngx_quic_frame_t *frame);
+ngx_int_t ngx_quic_split_frame(ngx_connection_t *c, ngx_quic_frame_t *f,
+    size_t len);
+
+ngx_chain_t *ngx_quic_alloc_chain(ngx_connection_t *c);
+void ngx_quic_free_chain(ngx_connection_t *c, ngx_chain_t *in);
+
+ngx_chain_t *ngx_quic_copy_buffer(ngx_connection_t *c, u_char *data,
+    size_t len);
+ngx_chain_t *ngx_quic_read_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb,
+    uint64_t limit);
+ngx_chain_t *ngx_quic_write_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb,
+    ngx_chain_t *in, uint64_t limit, uint64_t offset);
+void ngx_quic_skip_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb,
+    uint64_t offset);
+void ngx_quic_free_buffer(ngx_connection_t *c, ngx_quic_buffer_t *qb);
+
+#if (NGX_DEBUG)
+void ngx_quic_log_frame(ngx_log_t *log, ngx_quic_frame_t *f, ngx_uint_t tx);
+#else
+#define ngx_quic_log_frame(log, f, tx)
+#endif
+
+#endif /* _NGX_EVENT_QUIC_FRAMES_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_migration.h

@@ -0,0 +1,45 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_
+#define _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+#define NGX_QUIC_PATH_RETRIES   3
+
+#define NGX_QUIC_PATH_PROBE     0
+#define NGX_QUIC_PATH_ACTIVE    1
+#define NGX_QUIC_PATH_BACKUP    2
+
+#define ngx_quic_path_dbg(c, msg, path)                                       \
+    ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0,                            \
+                   "quic path seq:%uL %s tx:%O rx:%O valid:%d st:%d mtu:%uz", \
+                   path->seqnum, msg, path->sent, path->received,             \
+                   path->validated, path->state, path->mtu);
+
+ngx_int_t ngx_quic_handle_path_challenge_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_path_challenge_frame_t *f);
+ngx_int_t ngx_quic_handle_path_response_frame(ngx_connection_t *c,
+    ngx_quic_path_challenge_frame_t *f);
+
+ngx_quic_path_t *ngx_quic_new_path(ngx_connection_t *c,
+    struct sockaddr *sockaddr, socklen_t socklen, ngx_quic_client_id_t *cid);
+ngx_int_t ngx_quic_free_path(ngx_connection_t *c, ngx_quic_path_t *path);
+
+ngx_int_t ngx_quic_set_path(ngx_connection_t *c, ngx_quic_header_t *pkt);
+ngx_int_t ngx_quic_handle_migration(ngx_connection_t *c,
+    ngx_quic_header_t *pkt);
+
+void ngx_quic_path_handler(ngx_event_t *ev);
+
+void ngx_quic_discover_path_mtu(ngx_connection_t *c, ngx_quic_path_t *path);
+ngx_int_t ngx_quic_handle_path_mtu(ngx_connection_t *c,
+    ngx_quic_path_t *path, uint64_t min, uint64_t max);
+
+#endif /* _NGX_EVENT_QUIC_MIGRATION_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_openssl_compat.c

@@ -0,0 +1,652 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+
+#if (NGX_QUIC_OPENSSL_COMPAT)
+
+#define NGX_QUIC_COMPAT_RECORD_SIZE          1024
+
+#define NGX_QUIC_COMPAT_SSL_TP_EXT           0x39
+
+#define NGX_QUIC_COMPAT_CLIENT_HANDSHAKE     "CLIENT_HANDSHAKE_TRAFFIC_SECRET"
+#define NGX_QUIC_COMPAT_SERVER_HANDSHAKE     "SERVER_HANDSHAKE_TRAFFIC_SECRET"
+#define NGX_QUIC_COMPAT_CLIENT_APPLICATION   "CLIENT_TRAFFIC_SECRET_0"
+#define NGX_QUIC_COMPAT_SERVER_APPLICATION   "SERVER_TRAFFIC_SECRET_0"
+
+
+typedef struct {
+    ngx_quic_secret_t             secret;
+    ngx_uint_t                    cipher;
+} ngx_quic_compat_keys_t;
+
+
+typedef struct {
+    ngx_log_t                    *log;
+
+    u_char                        type;
+    ngx_str_t                     payload;
+    uint64_t                      number;
+    ngx_quic_compat_keys_t       *keys;
+} ngx_quic_compat_record_t;
+
+
+struct ngx_quic_compat_s {
+    const SSL_QUIC_METHOD        *method;
+
+    enum ssl_encryption_level_t   write_level;
+
+    uint64_t                      read_record;
+    ngx_quic_compat_keys_t        keys;
+
+    ngx_str_t                     tp;
+    ngx_str_t                     ctp;
+};
+
+
+static void ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line);
+static ngx_int_t ngx_quic_compat_set_encryption_secret(ngx_connection_t *c,
+    ngx_quic_compat_keys_t *keys, enum ssl_encryption_level_t level,
+    const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len);
+static void ngx_quic_compat_cleanup_encryption_secret(void *data);
+static int ngx_quic_compat_add_transport_params_callback(SSL *ssl,
+    unsigned int ext_type, unsigned int context, const unsigned char **out,
+    size_t *outlen, X509 *x, size_t chainidx, int *al, void *add_arg);
+static int ngx_quic_compat_parse_transport_params_callback(SSL *ssl,
+    unsigned int ext_type, unsigned int context, const unsigned char *in,
+    size_t inlen, X509 *x, size_t chainidx, int *al, void *parse_arg);
+static void ngx_quic_compat_message_callback(int write_p, int version,
+    int content_type, const void *buf, size_t len, SSL *ssl, void *arg);
+static size_t ngx_quic_compat_create_header(ngx_quic_compat_record_t *rec,
+    u_char *out, ngx_uint_t plain);
+static ngx_int_t ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec,
+    ngx_str_t *res);
+
+
+ngx_int_t
+ngx_quic_compat_init(ngx_conf_t *cf, SSL_CTX *ctx)
+{
+    SSL_CTX_set_keylog_callback(ctx, ngx_quic_compat_keylog_callback);
+
+    if (SSL_CTX_has_client_custom_ext(ctx, NGX_QUIC_COMPAT_SSL_TP_EXT)) {
+        return NGX_OK;
+    }
+
+    if (SSL_CTX_add_custom_ext(ctx, NGX_QUIC_COMPAT_SSL_TP_EXT,
+                               SSL_EXT_CLIENT_HELLO
+                               |SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS,
+                               ngx_quic_compat_add_transport_params_callback,
+                               NULL,
+                               NULL,
+                               ngx_quic_compat_parse_transport_params_callback,
+                               NULL)
+        == 0)
+    {
+        ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
+                      "SSL_CTX_add_custom_ext() failed");
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_quic_compat_keylog_callback(const SSL *ssl, const char *line)
+{
+    u_char                        ch, *p, *start, value;
+    size_t                        n;
+    ngx_uint_t                    write;
+    const SSL_CIPHER             *cipher;
+    ngx_quic_compat_t            *com;
+    ngx_connection_t             *c;
+    ngx_quic_connection_t        *qc;
+    enum ssl_encryption_level_t   level;
+    u_char                        secret[EVP_MAX_MD_SIZE];
+
+    c = ngx_ssl_get_connection(ssl);
+    if (c->type != SOCK_DGRAM) {
+        return;
+    }
+
+    p = (u_char *) line;
+
+    for (start = p; *p && *p != ' '; p++);
+
+    n = p - start;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic compat secret %*s", n, start);
+
+    if (n == sizeof(NGX_QUIC_COMPAT_CLIENT_HANDSHAKE) - 1
+        && ngx_strncmp(start, NGX_QUIC_COMPAT_CLIENT_HANDSHAKE, n) == 0)
+    {
+        level = ssl_encryption_handshake;
+        write = 0;
+
+    } else if (n == sizeof(NGX_QUIC_COMPAT_SERVER_HANDSHAKE) - 1
+               && ngx_strncmp(start, NGX_QUIC_COMPAT_SERVER_HANDSHAKE, n) == 0)
+    {
+        level = ssl_encryption_handshake;
+        write = 1;
+
+    } else if (n == sizeof(NGX_QUIC_COMPAT_CLIENT_APPLICATION) - 1
+               && ngx_strncmp(start, NGX_QUIC_COMPAT_CLIENT_APPLICATION, n)
+                  == 0)
+    {
+        level = ssl_encryption_application;
+        write = 0;
+
+    } else if (n == sizeof(NGX_QUIC_COMPAT_SERVER_APPLICATION) - 1
+               && ngx_strncmp(start, NGX_QUIC_COMPAT_SERVER_APPLICATION, n)
+                   == 0)
+    {
+        level = ssl_encryption_application;
+        write = 1;
+
+    } else {
+        return;
+    }
+
+    if (*p++ == '\0') {
+        return;
+    }
+
+    for ( /* void */ ; *p && *p != ' '; p++);
+
+    if (*p++ == '\0') {
+        return;
+    }
+
+    for (n = 0, start = p; *p; p++) {
+        ch = *p;
+
+        if (ch >= '0' && ch <= '9') {
+            value = ch - '0';
+            goto next;
+        }
+
+        ch = (u_char) (ch | 0x20);
+
+        if (ch >= 'a' && ch <= 'f') {
+            value = ch - 'a' + 10;
+            goto next;
+        }
+
+        ngx_log_error(NGX_LOG_EMERG, c->log, 0,
+                      "invalid OpenSSL QUIC secret format");
+
+        return;
+
+    next:
+
+        if ((p - start) % 2) {
+            secret[n++] += value;
+
+        } else {
+            if (n >= EVP_MAX_MD_SIZE) {
+                ngx_log_error(NGX_LOG_EMERG, c->log, 0,
+                              "too big OpenSSL QUIC secret");
+                return;
+            }
+
+            secret[n] = (value << 4);
+        }
+    }
+
+    qc = ngx_quic_get_connection(c);
+    com = qc->compat;
+    cipher = SSL_get_current_cipher(ssl);
+
+    if (write) {
+        com->method->set_write_secret((SSL *) ssl, level, cipher, secret, n);
+        com->write_level = level;
+
+    } else {
+        com->method->set_read_secret((SSL *) ssl, level, cipher, secret, n);
+        com->read_record = 0;
+
+        if (ngx_quic_compat_set_encryption_secret(c, &com->keys, level,
+                                                  cipher, secret, n)
+            != NGX_OK)
+        {
+            qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        }
+    }
+
+    ngx_explicit_memzero(secret, n);
+}
+
+
+static ngx_int_t
+ngx_quic_compat_set_encryption_secret(ngx_connection_t *c,
+    ngx_quic_compat_keys_t *keys, enum ssl_encryption_level_t level,
+    const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len)
+{
+    ngx_int_t            key_len;
+    ngx_str_t            secret_str;
+    ngx_uint_t           i;
+    ngx_quic_md_t        key;
+    ngx_quic_hkdf_t      seq[2];
+    ngx_quic_secret_t   *peer_secret;
+    ngx_quic_ciphers_t   ciphers;
+    ngx_pool_cleanup_t  *cln;
+
+    peer_secret = &keys->secret;
+
+    keys->cipher = SSL_CIPHER_get_id(cipher);
+
+    key_len = ngx_quic_ciphers(keys->cipher, &ciphers);
+
+    if (key_len == NGX_ERROR) {
+        ngx_ssl_error(NGX_LOG_INFO, c->log, 0, "unexpected cipher");
+        return NGX_ERROR;
+    }
+
+    key.len = key_len;
+
+    peer_secret->iv.len = NGX_QUIC_IV_LEN;
+
+    secret_str.len = secret_len;
+    secret_str.data = (u_char *) secret;
+
+    ngx_quic_hkdf_set(&seq[0], "tls13 key", &key, &secret_str);
+    ngx_quic_hkdf_set(&seq[1], "tls13 iv", &peer_secret->iv, &secret_str);
+
+    for (i = 0; i < (sizeof(seq) / sizeof(seq[0])); i++) {
+        if (ngx_quic_hkdf_expand(&seq[i], ciphers.d, c->log) != NGX_OK) {
+            return NGX_ERROR;
+        }
+    }
+
+    /* register cleanup handler once */
+
+    if (peer_secret->ctx) {
+        ngx_quic_crypto_cleanup(peer_secret);
+
+    } else {
+        cln = ngx_pool_cleanup_add(c->pool, 0);
+        if (cln == NULL) {
+            return NGX_ERROR;
+        }
+
+        cln->handler = ngx_quic_compat_cleanup_encryption_secret;
+        cln->data = peer_secret;
+    }
+
+    if (ngx_quic_crypto_init(ciphers.c, peer_secret, &key, 1, c->log)
+        == NGX_ERROR)
+    {
+        return NGX_ERROR;
+    }
+
+    ngx_explicit_memzero(key.data, key.len);
+
+    return NGX_OK;
+}
+
+
+static void
+ngx_quic_compat_cleanup_encryption_secret(void *data)
+{
+    ngx_quic_secret_t *secret = data;
+
+    ngx_quic_crypto_cleanup(secret);
+}
+
+
+static int
+ngx_quic_compat_add_transport_params_callback(SSL *ssl, unsigned int ext_type,
+    unsigned int context, const unsigned char **out, size_t *outlen, X509 *x,
+    size_t chainidx, int *al, void *add_arg)
+{
+    ngx_connection_t       *c;
+    ngx_quic_compat_t      *com;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl);
+    if (c->type != SOCK_DGRAM) {
+        return 0;
+    }
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic compat add transport params");
+
+    qc = ngx_quic_get_connection(c);
+    com = qc->compat;
+
+    *out = com->tp.data;
+    *outlen = com->tp.len;
+
+    return 1;
+}
+
+
+static int
+ngx_quic_compat_parse_transport_params_callback(SSL *ssl, unsigned int ext_type,
+    unsigned int context, const unsigned char *in, size_t inlen, X509 *x,
+    size_t chainidx, int *al, void *parse_arg)
+{
+    u_char                 *p;
+    ngx_connection_t       *c;
+    ngx_quic_compat_t      *com;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl);
+    if (c->type != SOCK_DGRAM) {
+        return 0;
+    }
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic compat parse transport params");
+
+    qc = ngx_quic_get_connection(c);
+    com = qc->compat;
+
+    p = ngx_pnalloc(c->pool, inlen);
+    if (p == NULL) {
+        return 0;
+    }
+
+    ngx_memcpy(p, in, inlen);
+
+    com->ctp.data = p;
+    com->ctp.len = inlen;
+
+    return 1;
+}
+
+
+int
+SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method)
+{
+    BIO                    *rbio, *wbio;
+    ngx_connection_t       *c;
+    ngx_quic_compat_t      *com;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl);
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic compat set method");
+
+    qc = ngx_quic_get_connection(c);
+
+    qc->compat = ngx_pcalloc(c->pool, sizeof(ngx_quic_compat_t));
+    if (qc->compat == NULL) {
+        return 0;
+    }
+
+    com = qc->compat;
+    com->method = quic_method;
+
+    rbio = BIO_new(BIO_s_mem());
+    if (rbio == NULL) {
+        return 0;
+    }
+
+    wbio = BIO_new(BIO_s_null());
+    if (wbio == NULL) {
+        BIO_free(rbio);
+        return 0;
+    }
+
+    SSL_set_bio(ssl, rbio, wbio);
+
+    SSL_set_msg_callback(ssl, ngx_quic_compat_message_callback);
+
+    /* early data is not supported */
+    SSL_set_max_early_data(ssl, 0);
+
+    return 1;
+}
+
+
+static void
+ngx_quic_compat_message_callback(int write_p, int version, int content_type,
+    const void *buf, size_t len, SSL *ssl, void *arg)
+{
+    ngx_uint_t                    alert;
+    ngx_connection_t             *c;
+    ngx_quic_compat_t            *com;
+    ngx_quic_connection_t        *qc;
+    enum ssl_encryption_level_t   level;
+
+    if (!write_p) {
+        return;
+    }
+
+    c = ngx_ssl_get_connection(ssl);
+    qc = ngx_quic_get_connection(c);
+
+    if (qc == NULL) {
+        /* closing */
+        return;
+    }
+
+    com = qc->compat;
+    level = com->write_level;
+
+    switch (content_type) {
+
+    case SSL3_RT_HANDSHAKE:
+        ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic compat tx level:%d len:%uz", level, len);
+
+        if (com->method->add_handshake_data(ssl, level, buf, len) != 1) {
+            return;
+        }
+
+        break;
+
+    case SSL3_RT_ALERT:
+        if (len >= 2) {
+            alert = ((u_char *) buf)[1];
+
+            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "quic compat level:%d alert:%ui len:%uz",
+                           level, alert, len);
+
+            if (com->method->send_alert(ssl, level, alert) != 1) {
+                return;
+            }
+        }
+
+        break;
+    }
+
+    return;
+}
+
+
+int
+SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level,
+    const uint8_t *data, size_t len)
+{
+    BIO                       *rbio;
+    size_t                     n;
+    u_char                    *p;
+    ngx_str_t                  res;
+    ngx_connection_t          *c;
+    ngx_quic_compat_t         *com;
+    ngx_quic_connection_t     *qc;
+    ngx_quic_compat_record_t   rec;
+    u_char                     in[NGX_QUIC_COMPAT_RECORD_SIZE + 1];
+    u_char                     out[NGX_QUIC_COMPAT_RECORD_SIZE + 1
+                                   + SSL3_RT_HEADER_LENGTH
+                                   + NGX_QUIC_TAG_LEN];
+
+    c = ngx_ssl_get_connection(ssl);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic compat rx level:%d len:%uz", level, len);
+
+    qc = ngx_quic_get_connection(c);
+    com = qc->compat;
+    rbio = SSL_get_rbio(ssl);
+
+    while (len) {
+        ngx_memzero(&rec, sizeof(ngx_quic_compat_record_t));
+
+        rec.type = SSL3_RT_HANDSHAKE;
+        rec.log = c->log;
+        rec.number = com->read_record++;
+        rec.keys = &com->keys;
+
+        if (level == ssl_encryption_initial) {
+            n = ngx_min(len, 65535);
+
+            rec.payload.len = n;
+            rec.payload.data = (u_char *) data;
+
+            ngx_quic_compat_create_header(&rec, out, 1);
+
+            BIO_write(rbio, out, SSL3_RT_HEADER_LENGTH);
+            BIO_write(rbio, data, n);
+
+#if defined(NGX_QUIC_DEBUG_CRYPTO) && defined(NGX_QUIC_DEBUG_PACKETS)
+            ngx_log_debug5(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "quic compat record len:%uz %*xs%*xs",
+                           n + SSL3_RT_HEADER_LENGTH,
+                           (size_t) SSL3_RT_HEADER_LENGTH, out, n, data);
+#endif
+
+        } else {
+            n = ngx_min(len, NGX_QUIC_COMPAT_RECORD_SIZE);
+
+            p = ngx_cpymem(in, data, n);
+            *p++ = SSL3_RT_HANDSHAKE;
+
+            rec.payload.len = p - in;
+            rec.payload.data = in;
+
+            res.data = out;
+
+            if (ngx_quic_compat_create_record(&rec, &res) != NGX_OK) {
+                return 0;
+            }
+
+#if defined(NGX_QUIC_DEBUG_CRYPTO) && defined(NGX_QUIC_DEBUG_PACKETS)
+            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                           "quic compat record len:%uz %xV", res.len, &res);
+#endif
+
+            BIO_write(rbio, res.data, res.len);
+        }
+
+        data += n;
+        len -= n;
+    }
+
+    return 1;
+}
+
+
+static size_t
+ngx_quic_compat_create_header(ngx_quic_compat_record_t *rec, u_char *out,
+    ngx_uint_t plain)
+{
+    u_char  type;
+    size_t  len;
+
+    len = rec->payload.len;
+
+    if (plain) {
+        type = rec->type;
+
+    } else {
+        type = SSL3_RT_APPLICATION_DATA;
+        len += NGX_QUIC_TAG_LEN;
+    }
+
+    out[0] = type;
+    out[1] = 0x03;
+    out[2] = 0x03;
+    out[3] = (len >> 8);
+    out[4] = len;
+
+    return 5;
+}
+
+
+static ngx_int_t
+ngx_quic_compat_create_record(ngx_quic_compat_record_t *rec, ngx_str_t *res)
+{
+    ngx_str_t           ad, out;
+    ngx_quic_secret_t  *secret;
+    u_char              nonce[NGX_QUIC_IV_LEN];
+
+    ad.data = res->data;
+    ad.len = ngx_quic_compat_create_header(rec, ad.data, 0);
+
+    out.len = rec->payload.len + NGX_QUIC_TAG_LEN;
+    out.data = res->data + ad.len;
+
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, rec->log, 0,
+                   "quic compat ad len:%uz %xV", ad.len, &ad);
+#endif
+
+    secret = &rec->keys->secret;
+
+    if (secret->ctx == NULL) {
+        return NGX_ERROR;
+    }
+
+    ngx_memcpy(nonce, secret->iv.data, secret->iv.len);
+    ngx_quic_compute_nonce(nonce, sizeof(nonce), rec->number);
+
+    if (ngx_quic_crypto_seal(secret, &out, nonce, &rec->payload, &ad, rec->log)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
+    res->len = ad.len + out.len;
+
+    return NGX_OK;
+}
+
+
+int
+SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params,
+    size_t params_len)
+{
+    ngx_connection_t       *c;
+    ngx_quic_compat_t      *com;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl);
+    qc = ngx_quic_get_connection(c);
+    com = qc->compat;
+
+    com->tp.len = params_len;
+    com->tp.data = (u_char *) params;
+
+    return 1;
+}
+
+
+void
+SSL_get_peer_quic_transport_params(const SSL *ssl, const uint8_t **out_params,
+    size_t *out_params_len)
+{
+    ngx_connection_t       *c;
+    ngx_quic_compat_t      *com;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl);
+    qc = ngx_quic_get_connection(c);
+    com = qc->compat;
+
+    *out_params = com->ctp.data;
+    *out_params_len = com->ctp.len;
+}
+
+#endif /* NGX_QUIC_OPENSSL_COMPAT */

nginx/src/event/quic/ngx_event_quic_openssl_compat.h

@@ -0,0 +1,51 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_
+#define _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+typedef struct ngx_quic_compat_s  ngx_quic_compat_t;
+
+
+enum ssl_encryption_level_t {
+    ssl_encryption_initial = 0,
+    ssl_encryption_early_data,
+    ssl_encryption_handshake,
+    ssl_encryption_application
+};
+
+
+typedef struct ssl_quic_method_st {
+    int (*set_read_secret)(SSL *ssl, enum ssl_encryption_level_t level,
+                           const SSL_CIPHER *cipher,
+                           const uint8_t *rsecret, size_t secret_len);
+    int (*set_write_secret)(SSL *ssl, enum ssl_encryption_level_t level,
+                            const SSL_CIPHER *cipher,
+                            const uint8_t *wsecret, size_t secret_len);
+    int (*add_handshake_data)(SSL *ssl, enum ssl_encryption_level_t level,
+                              const uint8_t *data, size_t len);
+    int (*flush_flight)(SSL *ssl);
+    int (*send_alert)(SSL *ssl, enum ssl_encryption_level_t level,
+                      uint8_t alert);
+} SSL_QUIC_METHOD;
+
+
+ngx_int_t ngx_quic_compat_init(ngx_conf_t *cf, SSL_CTX *ctx);
+
+int SSL_set_quic_method(SSL *ssl, const SSL_QUIC_METHOD *quic_method);
+int SSL_provide_quic_data(SSL *ssl, enum ssl_encryption_level_t level,
+    const uint8_t *data, size_t len);
+int SSL_set_quic_transport_params(SSL *ssl, const uint8_t *params,
+    size_t params_len);
+void SSL_get_peer_quic_transport_params(const SSL *ssl,
+    const uint8_t **out_params, size_t *out_params_len);
+
+#endif /* _NGX_EVENT_QUIC_OPENSSL_COMPAT_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_output.h

@@ -0,0 +1,40 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_OUTPUT_H_INCLUDED_
+#define _NGX_EVENT_QUIC_OUTPUT_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+ngx_int_t ngx_quic_output(ngx_connection_t *c);
+
+ngx_int_t ngx_quic_negotiate_version(ngx_connection_t *c,
+    ngx_quic_header_t *inpkt);
+
+ngx_int_t ngx_quic_send_stateless_reset(ngx_connection_t *c,
+    ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
+ngx_int_t ngx_quic_send_cc(ngx_connection_t *c);
+ngx_int_t ngx_quic_send_early_cc(ngx_connection_t *c,
+    ngx_quic_header_t *inpkt, ngx_uint_t err, const char *reason);
+
+ngx_int_t ngx_quic_send_retry(ngx_connection_t *c,
+    ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
+ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c, ngx_quic_path_t *path);
+
+ngx_int_t ngx_quic_send_ack(ngx_connection_t *c,
+    ngx_quic_send_ctx_t *ctx);
+ngx_int_t ngx_quic_send_ack_range(ngx_connection_t *c,
+    ngx_quic_send_ctx_t *ctx, uint64_t smallest, uint64_t largest);
+
+ngx_int_t ngx_quic_frame_sendto(ngx_connection_t *c, ngx_quic_frame_t *frame,
+    size_t min, ngx_quic_path_t *path);
+size_t ngx_quic_path_limit(ngx_connection_t *c, ngx_quic_path_t *path,
+    size_t size);
+
+#endif /* _NGX_EVENT_QUIC_OUTPUT_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_protection.h

@@ -0,0 +1,118 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_PROTECTION_H_INCLUDED_
+#define _NGX_EVENT_QUIC_PROTECTION_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+#include <ngx_event_quic_transport.h>
+
+
+/* RFC 5116, 5.1/5.3 and RFC 8439, 2.3/2.5 for all supported ciphers */
+#define NGX_QUIC_IV_LEN               12
+#define NGX_QUIC_TAG_LEN              16
+
+/* largest hash used in TLS is SHA-384 */
+#define NGX_QUIC_MAX_MD_SIZE          48
+
+
+#if (defined OPENSSL_IS_BORINGSSL || defined OPENSSL_IS_AWSLC)
+#define NGX_QUIC_BORINGSSL_EVP_API    1
+#define ngx_quic_cipher_t             EVP_AEAD
+#define ngx_quic_crypto_ctx_t         EVP_AEAD_CTX
+#else
+#define NGX_QUIC_BORINGSSL_EVP_API    0
+#define ngx_quic_cipher_t             EVP_CIPHER
+#define ngx_quic_crypto_ctx_t         EVP_CIPHER_CTX
+#endif
+
+
+typedef struct {
+    size_t                    len;
+    u_char                    data[NGX_QUIC_MAX_MD_SIZE];
+} ngx_quic_md_t;
+
+
+typedef struct {
+    size_t                    len;
+    u_char                    data[NGX_QUIC_IV_LEN];
+} ngx_quic_iv_t;
+
+
+typedef struct {
+    ngx_quic_md_t             secret;
+    ngx_quic_iv_t             iv;
+    ngx_quic_md_t             hp;
+    ngx_quic_crypto_ctx_t    *ctx;
+    EVP_CIPHER_CTX           *hp_ctx;
+} ngx_quic_secret_t;
+
+
+typedef struct {
+    ngx_quic_secret_t         client;
+    ngx_quic_secret_t         server;
+} ngx_quic_secrets_t;
+
+
+struct ngx_quic_keys_s {
+    ngx_quic_secrets_t        secrets[NGX_QUIC_ENCRYPTION_LAST];
+    ngx_quic_secrets_t        next_key;
+    ngx_uint_t                cipher;
+};
+
+
+typedef struct {
+    const ngx_quic_cipher_t  *c;
+    const EVP_CIPHER         *hp;
+    const EVP_MD             *d;
+} ngx_quic_ciphers_t;
+
+
+typedef struct {
+    size_t                    out_len;
+    u_char                   *out;
+
+    size_t                    prk_len;
+    const uint8_t            *prk;
+
+    size_t                    label_len;
+    const u_char             *label;
+} ngx_quic_hkdf_t;
+
+#define ngx_quic_hkdf_set(seq, _label, _out, _prk)                            \
+    (seq)->out_len = (_out)->len; (seq)->out = (_out)->data;                  \
+    (seq)->prk_len = (_prk)->len, (seq)->prk = (_prk)->data,                  \
+    (seq)->label_len = (sizeof(_label) - 1); (seq)->label = (u_char *)(_label);
+
+
+ngx_int_t ngx_quic_keys_set_initial_secret(ngx_quic_keys_t *keys,
+    ngx_str_t *secret, ngx_log_t *log);
+ngx_int_t ngx_quic_keys_set_encryption_secret(ngx_log_t *log,
+    ngx_uint_t is_write, ngx_quic_keys_t *keys, ngx_uint_t level,
+    const SSL_CIPHER *cipher, const uint8_t *secret, size_t secret_len);
+ngx_uint_t ngx_quic_keys_available(ngx_quic_keys_t *keys, ngx_uint_t level,
+    ngx_uint_t is_write);
+void ngx_quic_keys_discard(ngx_quic_keys_t *keys, ngx_uint_t level);
+void ngx_quic_keys_switch(ngx_connection_t *c, ngx_quic_keys_t *keys);
+void ngx_quic_keys_update(ngx_event_t *ev);
+void ngx_quic_keys_cleanup(ngx_quic_keys_t *keys);
+ngx_int_t ngx_quic_encrypt(ngx_quic_header_t *pkt, ngx_str_t *res);
+ngx_int_t ngx_quic_decrypt(ngx_quic_header_t *pkt, uint64_t *largest_pn);
+void ngx_quic_compute_nonce(u_char *nonce, size_t len, uint64_t pn);
+ngx_int_t ngx_quic_ciphers(ngx_uint_t id, ngx_quic_ciphers_t *ciphers);
+ngx_int_t ngx_quic_crypto_init(const ngx_quic_cipher_t *cipher,
+    ngx_quic_secret_t *s, ngx_quic_md_t *key, ngx_int_t enc, ngx_log_t *log);
+ngx_int_t ngx_quic_crypto_seal(ngx_quic_secret_t *s, ngx_str_t *out,
+    const u_char *nonce, ngx_str_t *in, ngx_str_t *ad, ngx_log_t *log);
+void ngx_quic_crypto_cleanup(ngx_quic_secret_t *s);
+ngx_int_t ngx_quic_hkdf_expand(ngx_quic_hkdf_t *hkdf, const EVP_MD *digest,
+    ngx_log_t *log);
+
+
+#endif /* _NGX_EVENT_QUIC_PROTECTION_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_socket.c

@@ -0,0 +1,237 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+
+ngx_int_t
+ngx_quic_open_sockets(ngx_connection_t *c, ngx_quic_connection_t *qc,
+    ngx_quic_header_t *pkt)
+{
+    ngx_quic_socket_t     *qsock, *tmp;
+    ngx_quic_client_id_t  *cid;
+
+    /*
+     * qc->path = NULL
+     *
+     * qc->nclient_ids = 0
+     * qc->nsockets = 0
+     * qc->max_retired_seqnum = 0
+     * qc->client_seqnum = 0
+     */
+
+    ngx_queue_init(&qc->sockets);
+    ngx_queue_init(&qc->free_sockets);
+
+    ngx_queue_init(&qc->paths);
+    ngx_queue_init(&qc->free_paths);
+
+    ngx_queue_init(&qc->client_ids);
+    ngx_queue_init(&qc->free_client_ids);
+
+    qc->tp.original_dcid.len = pkt->odcid.len;
+    qc->tp.original_dcid.data = ngx_pstrdup(c->pool, &pkt->odcid);
+    if (qc->tp.original_dcid.data == NULL) {
+        return NGX_ERROR;
+    }
+
+    /* socket to use for further processing (id auto-generated) */
+    qsock = ngx_quic_create_socket(c, qc);
+    if (qsock == NULL) {
+        return NGX_ERROR;
+    }
+
+    /* socket is listening at new server id */
+    if (ngx_quic_listen(c, qc, qsock) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    qsock->used = 1;
+
+    qc->tp.initial_scid.len = qsock->sid.len;
+    qc->tp.initial_scid.data = ngx_pnalloc(c->pool, qsock->sid.len);
+    if (qc->tp.initial_scid.data == NULL) {
+        goto failed;
+    }
+    ngx_memcpy(qc->tp.initial_scid.data, qsock->sid.id, qsock->sid.len);
+
+    /* for all packets except first, this is set at udp layer */
+    c->udp = &qsock->udp;
+
+    /* ngx_quic_get_connection(c) macro is now usable */
+
+    /* we have a client identified by scid */
+    cid = ngx_quic_create_client_id(c, &pkt->scid, 0, NULL);
+    if (cid == NULL) {
+        goto failed;
+    }
+
+    /* path of the first packet is our initial active path */
+    qc->path = ngx_quic_new_path(c, c->sockaddr, c->socklen, cid);
+    if (qc->path == NULL) {
+        goto failed;
+    }
+
+    qc->path->tag = NGX_QUIC_PATH_ACTIVE;
+
+    if (pkt->validated) {
+        qc->path->validated = 1;
+    }
+
+    ngx_quic_path_dbg(c, "set active", qc->path);
+
+    tmp = ngx_pcalloc(c->pool, sizeof(ngx_quic_socket_t));
+    if (tmp == NULL) {
+        goto failed;
+    }
+
+    tmp->sid.seqnum = NGX_QUIC_UNSET_PN; /* temporary socket */
+
+    ngx_memcpy(tmp->sid.id, pkt->dcid.data, pkt->dcid.len);
+    tmp->sid.len = pkt->dcid.len;
+
+    if (ngx_quic_listen(c, qc, tmp) != NGX_OK) {
+        goto failed;
+    }
+
+    return NGX_OK;
+
+failed:
+
+    ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node);
+    c->udp = NULL;
+
+    return NGX_ERROR;
+}
+
+
+ngx_quic_socket_t *
+ngx_quic_create_socket(ngx_connection_t *c, ngx_quic_connection_t *qc)
+{
+    ngx_queue_t        *q;
+    ngx_quic_socket_t  *sock;
+
+    if (!ngx_queue_empty(&qc->free_sockets)) {
+
+        q = ngx_queue_head(&qc->free_sockets);
+        sock = ngx_queue_data(q, ngx_quic_socket_t, queue);
+
+        ngx_queue_remove(&sock->queue);
+
+        ngx_memzero(sock, sizeof(ngx_quic_socket_t));
+
+    } else {
+
+        sock = ngx_pcalloc(c->pool, sizeof(ngx_quic_socket_t));
+        if (sock == NULL) {
+            return NULL;
+        }
+    }
+
+    sock->sid.len = NGX_QUIC_SERVER_CID_LEN;
+    if (ngx_quic_create_server_id(c, sock->sid.id) != NGX_OK) {
+        return NULL;
+    }
+
+    sock->sid.seqnum = qc->server_seqnum++;
+
+    return sock;
+}
+
+
+void
+ngx_quic_close_socket(ngx_connection_t *c, ngx_quic_socket_t *qsock)
+{
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    ngx_queue_remove(&qsock->queue);
+    ngx_queue_insert_head(&qc->free_sockets, &qsock->queue);
+
+    ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node);
+    qc->nsockets--;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic socket seq:%L closed nsock:%ui",
+                   (int64_t) qsock->sid.seqnum, qc->nsockets);
+}
+
+
+ngx_int_t
+ngx_quic_listen(ngx_connection_t *c, ngx_quic_connection_t *qc,
+    ngx_quic_socket_t *qsock)
+{
+    ngx_str_t              id;
+    ngx_quic_server_id_t  *sid;
+
+    sid = &qsock->sid;
+
+    id.data = sid->id;
+    id.len = sid->len;
+
+    qsock->udp.connection = c;
+    qsock->udp.node.key = ngx_crc32_long(id.data, id.len);
+    qsock->udp.key = id;
+
+    ngx_rbtree_insert(&c->listening->rbtree, &qsock->udp.node);
+
+    ngx_queue_insert_tail(&qc->sockets, &qsock->queue);
+
+    qc->nsockets++;
+    qsock->quic = qc;
+
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic socket seq:%L listening at sid:%xV nsock:%ui",
+                   (int64_t) sid->seqnum, &id, qc->nsockets);
+
+    return NGX_OK;
+}
+
+
+void
+ngx_quic_close_sockets(ngx_connection_t *c)
+{
+    ngx_queue_t            *q;
+    ngx_quic_socket_t      *qsock;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    while (!ngx_queue_empty(&qc->sockets)) {
+        q = ngx_queue_head(&qc->sockets);
+        qsock = ngx_queue_data(q, ngx_quic_socket_t, queue);
+
+        ngx_quic_close_socket(c, qsock);
+    }
+}
+
+
+ngx_quic_socket_t *
+ngx_quic_find_socket(ngx_connection_t *c, uint64_t seqnum)
+{
+    ngx_queue_t            *q;
+    ngx_quic_socket_t      *qsock;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    for (q = ngx_queue_head(&qc->sockets);
+         q != ngx_queue_sentinel(&qc->sockets);
+         q = ngx_queue_next(q))
+    {
+        qsock = ngx_queue_data(q, ngx_quic_socket_t, queue);
+
+        if (qsock->sid.seqnum == seqnum) {
+            return qsock;
+        }
+    }
+
+    return NULL;
+}

nginx/src/event/quic/ngx_event_quic_socket.h

@@ -0,0 +1,28 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_SOCKET_H_INCLUDED_
+#define _NGX_EVENT_QUIC_SOCKET_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+ngx_int_t ngx_quic_open_sockets(ngx_connection_t *c,
+    ngx_quic_connection_t *qc, ngx_quic_header_t *pkt);
+void ngx_quic_close_sockets(ngx_connection_t *c);
+
+ngx_quic_socket_t *ngx_quic_create_socket(ngx_connection_t *c,
+    ngx_quic_connection_t *qc);
+ngx_int_t ngx_quic_listen(ngx_connection_t *c, ngx_quic_connection_t *qc,
+    ngx_quic_socket_t *qsock);
+void ngx_quic_close_socket(ngx_connection_t *c, ngx_quic_socket_t *qsock);
+
+ngx_quic_socket_t *ngx_quic_find_socket(ngx_connection_t *c, uint64_t seqnum);
+
+
+#endif /* _NGX_EVENT_QUIC_SOCKET_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_ssl.c

@@ -0,0 +1,987 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+
+/*
+ * RFC 9000, 7.5.  Cryptographic Message Buffering
+ *
+ * Implementations MUST support buffering at least 4096 bytes of data
+ */
+#define NGX_QUIC_MAX_BUFFERED    65535
+
+
+#if (NGX_QUIC_OPENSSL_API)
+
+static int ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *data, size_t len, size_t *consumed, void *arg);
+static int ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char **data, size_t *bytes_read, void *arg);
+static int ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn,
+    size_t bytes_read, void *arg);
+static int ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t level,
+    int direction, const unsigned char *secret, size_t secret_len, void *arg);
+static int ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *params, size_t params_len, void *arg);
+static int ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert,
+    void *arg);
+
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+
+static ngx_inline ngx_uint_t ngx_quic_map_encryption_level(
+    enum ssl_encryption_level_t ssl_level);
+
+#if (NGX_QUIC_BORINGSSL_API)
+static int ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher,
+    const uint8_t *secret, size_t secret_len);
+static int ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher,
+    const uint8_t *secret, size_t secret_len);
+#else /* NGX_QUIC_QUICTLS_API */
+static int ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const uint8_t *read_secret,
+    const uint8_t *write_secret, size_t secret_len);
+#endif
+
+static int ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const uint8_t *data, size_t len);
+static int ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn);
+static int ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, uint8_t alert);
+
+#endif
+
+static ngx_int_t ngx_quic_handshake(ngx_connection_t *c);
+static ngx_int_t ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level);
+
+
+#if (NGX_QUIC_OPENSSL_API)
+
+static int
+ngx_quic_cbs_send(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *data, size_t len, size_t *consumed, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_chain_t            *out;
+    unsigned int            alpn_len;
+    ngx_quic_frame_t       *frame;
+    const unsigned char    *alpn_data;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_send len:%uz", len);
+
+    qc = ngx_quic_get_connection(c);
+
+    *consumed = 0;
+
+    SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len);
+
+    if (alpn_len == 0) {
+        qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL);
+        qc->error_reason = "missing ALPN extension";
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "quic missing ALPN extension");
+        return 1;
+    }
+
+    if (!qc->client_tp_done) {
+        /* RFC 9001, 8.2.  QUIC Transport Parameters Extension */
+        qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION);
+        qc->error_reason = "missing transport parameters";
+
+        ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                      "missing transport parameters");
+        return 1;
+    }
+
+    ctx = ngx_quic_get_send_ctx(qc, qc->write_level);
+
+    out = ngx_quic_copy_buffer(c, (u_char *) data, len);
+    if (out == NGX_CHAIN_ERROR) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    frame = ngx_quic_alloc_frame(c);
+    if (frame == NULL) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    frame->data = out;
+    frame->level = qc->write_level;
+    frame->type = NGX_QUIC_FT_CRYPTO;
+    frame->u.crypto.offset = ctx->crypto_sent;
+    frame->u.crypto.length = len;
+
+    ctx->crypto_sent += len;
+    *consumed = len;
+
+    ngx_quic_queue_frame(qc, frame);
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_recv_rcd(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char **data, size_t *bytes_read, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_buf_t              *b;
+    ngx_chain_t            *cl;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_recv_rcd");
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, qc->read_level);
+
+    cl = ctx->crypto.chain;
+
+    if (cl == NULL || cl->buf->sync) {
+        *data = NULL;
+        *bytes_read = 0;
+
+    } else {
+        b = cl->buf;
+
+        *data = b->pos;
+        *bytes_read = b->last - b->pos;
+    }
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_release_rcd(ngx_ssl_conn_t *ssl_conn, size_t bytes_read, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_chain_t            *cl;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_release_rcd len:%uz", bytes_read);
+
+    /* already closed on handshake failure */
+
+    qc = ngx_quic_get_connection(c);
+    if (qc == NULL) {
+        return 1;
+    }
+
+    ctx = ngx_quic_get_send_ctx(qc, qc->read_level);
+
+    cl = ngx_quic_read_buffer(c, &ctx->crypto, bytes_read);
+    if (cl == NGX_CHAIN_ERROR) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    ngx_quic_free_chain(c, cl);
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_yield_secret(ngx_ssl_conn_t *ssl_conn, uint32_t ssl_level,
+    int direction, const unsigned char *secret, size_t secret_len, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_uint_t              level;
+    const SSL_CIPHER       *cipher;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_yield_secret() level:%uD", ssl_level);
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic %s secret len:%uz %*xs",
+                   direction ? "write" : "read", secret_len,
+                   secret_len, secret);
+#endif
+
+    qc = ngx_quic_get_connection(c);
+    cipher = SSL_get_current_cipher(ssl_conn);
+
+    switch (ssl_level) {
+    case OSSL_RECORD_PROTECTION_LEVEL_NONE:
+        level = NGX_QUIC_ENCRYPTION_INITIAL;
+        break;
+    case OSSL_RECORD_PROTECTION_LEVEL_EARLY:
+        level = NGX_QUIC_ENCRYPTION_EARLY_DATA;
+        break;
+    case OSSL_RECORD_PROTECTION_LEVEL_HANDSHAKE:
+        level = NGX_QUIC_ENCRYPTION_HANDSHAKE;
+        break;
+    default: /* OSSL_RECORD_PROTECTION_LEVEL_APPLICATION */
+        level = NGX_QUIC_ENCRYPTION_APPLICATION;
+        break;
+    }
+
+    if (ngx_quic_keys_set_encryption_secret(c->log, direction, qc->keys, level,
+                                            cipher, secret, secret_len)
+        != NGX_OK)
+    {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    if (direction) {
+        qc->write_level = level;
+
+    } else {
+        qc->read_level = level;
+    }
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_got_transport_params(ngx_ssl_conn_t *ssl_conn,
+    const unsigned char *params, size_t params_len, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    u_char                 *p, *end;
+    ngx_quic_tp_t           ctp;
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_got_transport_params() len:%uz",
+                   params_len);
+
+    qc = ngx_quic_get_connection(c);
+
+    /* defaults for parameters not sent by client */
+    ngx_memcpy(&ctp, &qc->ctp, sizeof(ngx_quic_tp_t));
+
+    p = (u_char *) params;
+    end = p + params_len;
+
+    if (ngx_quic_parse_transport_params(p, end, &ctp, c->log) != NGX_OK) {
+        qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR;
+        qc->error_reason = "failed to process transport parameters";
+
+        return 1;
+    }
+
+    if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) {
+        return 1;
+    }
+
+    qc->client_tp_done = 1;
+
+    return 1;
+}
+
+
+static int
+ngx_quic_cbs_alert(ngx_ssl_conn_t *ssl_conn, unsigned char alert, void *arg)
+{
+    ngx_connection_t  *c = arg;
+
+    ngx_quic_connection_t  *qc;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_cbs_alert() alert:%d", (int) alert);
+
+    /* already closed on regular shutdown */
+
+    qc = ngx_quic_get_connection(c);
+    if (qc == NULL) {
+        return 1;
+    }
+
+    qc->error = NGX_QUIC_ERR_CRYPTO(alert);
+    qc->error_reason = "handshake failed";
+
+    return 1;
+}
+
+
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+
+
+static ngx_inline ngx_uint_t
+ngx_quic_map_encryption_level(enum ssl_encryption_level_t ssl_level)
+{
+    switch (ssl_level) {
+    case ssl_encryption_initial:
+        return NGX_QUIC_ENCRYPTION_INITIAL;
+    case ssl_encryption_early_data:
+        return NGX_QUIC_ENCRYPTION_EARLY_DATA;
+    case ssl_encryption_handshake:
+        return NGX_QUIC_ENCRYPTION_HANDSHAKE;
+    default: /* ssl_encryption_application */
+        return NGX_QUIC_ENCRYPTION_APPLICATION;
+    }
+}
+
+
+#if (NGX_QUIC_BORINGSSL_API)
+
+static int
+ngx_quic_set_read_secret(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher,
+    const uint8_t *rsecret, size_t secret_len)
+{
+    ngx_uint_t              level;
+    ngx_connection_t       *c;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+    qc = ngx_quic_get_connection(c);
+    level = ngx_quic_map_encryption_level(ssl_level);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_set_read_secret() level:%d", ssl_level);
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic read secret len:%uz %*xs", secret_len,
+                   secret_len, rsecret);
+#endif
+
+    if (ngx_quic_keys_set_encryption_secret(c->log, 0, qc->keys, level,
+                                            cipher, rsecret, secret_len)
+        != NGX_OK)
+    {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+    }
+
+    return 1;
+}
+
+
+static int
+ngx_quic_set_write_secret(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const SSL_CIPHER *cipher,
+    const uint8_t *wsecret, size_t secret_len)
+{
+    ngx_uint_t              level;
+    ngx_connection_t       *c;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+    qc = ngx_quic_get_connection(c);
+    level = ngx_quic_map_encryption_level(ssl_level);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_set_write_secret() level:%d", ssl_level);
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic write secret len:%uz %*xs", secret_len,
+                   secret_len, wsecret);
+#endif
+
+    if (ngx_quic_keys_set_encryption_secret(c->log, 1, qc->keys, level,
+                                            cipher, wsecret, secret_len)
+        != NGX_OK)
+    {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+    }
+
+    return 1;
+}
+
+#else /* NGX_QUIC_QUICTLS_API */
+
+static int
+ngx_quic_set_encryption_secrets(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const uint8_t *rsecret,
+    const uint8_t *wsecret, size_t secret_len)
+{
+    ngx_uint_t              level;
+    ngx_connection_t       *c;
+    const SSL_CIPHER       *cipher;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+    qc = ngx_quic_get_connection(c);
+    level = ngx_quic_map_encryption_level(ssl_level);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_set_encryption_secrets() level:%d",
+                   ssl_level);
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic read secret len:%uz %*xs", secret_len,
+                   secret_len, rsecret);
+#endif
+
+    cipher = SSL_get_current_cipher(ssl_conn);
+
+    if (ngx_quic_keys_set_encryption_secret(c->log, 0, qc->keys, level,
+                                            cipher, rsecret, secret_len)
+        != NGX_OK)
+    {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    if (level == NGX_QUIC_ENCRYPTION_EARLY_DATA) {
+        return 1;
+    }
+
+#ifdef NGX_QUIC_DEBUG_CRYPTO
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic write secret len:%uz %*xs", secret_len,
+                   secret_len, wsecret);
+#endif
+
+    if (ngx_quic_keys_set_encryption_secret(c->log, 1, qc->keys, level,
+                                            cipher, wsecret, secret_len)
+        != NGX_OK)
+    {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+    }
+
+    return 1;
+}
+
+#endif
+
+
+static int
+ngx_quic_add_handshake_data(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, const uint8_t *data, size_t len)
+{
+    u_char                 *p, *end;
+    size_t                  client_params_len;
+    ngx_uint_t              level;
+    ngx_chain_t            *out;
+    unsigned int            alpn_len;
+    const uint8_t          *client_params;
+    ngx_quic_tp_t           ctp;
+    ngx_quic_frame_t       *frame;
+    ngx_connection_t       *c;
+    const unsigned char    *alpn_data;
+    ngx_quic_send_ctx_t    *ctx;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+    qc = ngx_quic_get_connection(c);
+    level = ngx_quic_map_encryption_level(ssl_level);
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_add_handshake_data");
+
+    if (!qc->client_tp_done) {
+        /*
+         * things to do once during handshake: check ALPN and transport
+         * parameters; we want to break handshake if something is wrong
+         * here;
+         */
+
+        SSL_get0_alpn_selected(ssl_conn, &alpn_data, &alpn_len);
+
+        if (alpn_len == 0) {
+            if (qc->error == 0) {
+                qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_NO_APPLICATION_PROTOCOL);
+                qc->error_reason = "missing ALPN extension";
+
+                ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                              "quic missing ALPN extension");
+            }
+
+            return 1;
+        }
+
+        SSL_get_peer_quic_transport_params(ssl_conn, &client_params,
+                                           &client_params_len);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                       "quic SSL_get_peer_quic_transport_params():"
+                       " params_len:%ui", client_params_len);
+
+        if (client_params_len == 0) {
+            /* RFC 9001, 8.2.  QUIC Transport Parameters Extension */
+
+            if (qc->error == 0) {
+                qc->error = NGX_QUIC_ERR_CRYPTO(SSL_AD_MISSING_EXTENSION);
+                qc->error_reason = "missing transport parameters";
+
+                ngx_log_error(NGX_LOG_INFO, c->log, 0,
+                              "missing transport parameters");
+            }
+
+            return 1;
+        }
+
+        p = (u_char *) client_params;
+        end = p + client_params_len;
+
+        /* defaults for parameters not sent by client */
+        ngx_memcpy(&ctp, &qc->ctp, sizeof(ngx_quic_tp_t));
+
+        if (ngx_quic_parse_transport_params(p, end, &ctp, c->log)
+            != NGX_OK)
+        {
+            qc->error = NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR;
+            qc->error_reason = "failed to process transport parameters";
+
+            return 1;
+        }
+
+        if (ngx_quic_apply_transport_params(c, &ctp) != NGX_OK) {
+            return 1;
+        }
+
+        qc->client_tp_done = 1;
+    }
+
+    ctx = ngx_quic_get_send_ctx(qc, level);
+
+    out = ngx_quic_copy_buffer(c, (u_char *) data, len);
+    if (out == NGX_CHAIN_ERROR) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    frame = ngx_quic_alloc_frame(c);
+    if (frame == NULL) {
+        qc->error = NGX_QUIC_ERR_INTERNAL_ERROR;
+        return 1;
+    }
+
+    frame->data = out;
+    frame->level = level;
+    frame->type = NGX_QUIC_FT_CRYPTO;
+    frame->u.crypto.offset = ctx->crypto_sent;
+    frame->u.crypto.length = len;
+
+    ctx->crypto_sent += len;
+
+    ngx_quic_queue_frame(qc, frame);
+
+    return 1;
+}
+
+
+static int
+ngx_quic_flush_flight(ngx_ssl_conn_t *ssl_conn)
+{
+#if (NGX_DEBUG)
+    ngx_connection_t  *c;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_flush_flight()");
+#endif
+    return 1;
+}
+
+
+static int
+ngx_quic_send_alert(ngx_ssl_conn_t *ssl_conn,
+    enum ssl_encryption_level_t ssl_level, uint8_t alert)
+{
+    ngx_connection_t       *c;
+    ngx_quic_connection_t  *qc;
+
+    c = ngx_ssl_get_connection(ssl_conn);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic ngx_quic_send_alert() level:%d alert:%d",
+                   ssl_level, (int) alert);
+
+    /* already closed on regular shutdown */
+
+    qc = ngx_quic_get_connection(c);
+    if (qc == NULL) {
+        return 1;
+    }
+
+    qc->error = NGX_QUIC_ERR_CRYPTO(alert);
+    qc->error_reason = "handshake failed";
+
+    return 1;
+}
+
+#endif
+
+
+ngx_int_t
+ngx_quic_handle_crypto_frame(ngx_connection_t *c, ngx_quic_header_t *pkt,
+    ngx_quic_frame_t *frame)
+{
+    uint64_t                  last;
+    ngx_quic_send_ctx_t      *ctx;
+    ngx_quic_connection_t    *qc;
+    ngx_quic_crypto_frame_t  *f;
+
+    qc = ngx_quic_get_connection(c);
+
+    if (!ngx_quic_keys_available(qc->keys, pkt->level, 0)) {
+        return NGX_OK;
+    }
+
+    ctx = ngx_quic_get_send_ctx(qc, pkt->level);
+    f = &frame->u.crypto;
+
+    /* no overflow since both values are 62-bit */
+    last = f->offset + f->length;
+
+    if (last > ctx->crypto.offset + NGX_QUIC_MAX_BUFFERED) {
+        qc->error = NGX_QUIC_ERR_CRYPTO_BUFFER_EXCEEDED;
+        return NGX_ERROR;
+    }
+
+    if (last <= ctx->crypto.offset) {
+        if (pkt->level == NGX_QUIC_ENCRYPTION_INITIAL) {
+            /* speeding up handshake completion */
+
+            if (!ngx_queue_empty(&ctx->sent)) {
+                ngx_quic_resend_frames(c, ctx);
+
+                ctx = ngx_quic_get_send_ctx(qc, NGX_QUIC_ENCRYPTION_HANDSHAKE);
+                while (!ngx_queue_empty(&ctx->sent)) {
+                    ngx_quic_resend_frames(c, ctx);
+                }
+            }
+        }
+
+        return NGX_OK;
+    }
+
+    if (ngx_quic_write_buffer(c, &ctx->crypto, frame->data, f->length,
+                              f->offset)
+        == NGX_CHAIN_ERROR)
+    {
+        return NGX_ERROR;
+    }
+
+    if (ngx_quic_crypto_provide(c, pkt->level) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    return ngx_quic_handshake(c);
+}
+
+
+static ngx_int_t
+ngx_quic_handshake(ngx_connection_t *c)
+{
+    int                     n, sslerr;
+    ngx_ssl_conn_t         *ssl_conn;
+    ngx_quic_frame_t       *frame;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    ssl_conn = c->ssl->connection;
+
+    n = SSL_do_handshake(ssl_conn);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_do_handshake: %d", n);
+
+    if (n <= 0) {
+        sslerr = SSL_get_error(ssl_conn, n);
+
+        ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d",
+                       sslerr);
+
+        if (c->ssl->handshake_rejected) {
+            ngx_connection_error(c, 0, "handshake rejected");
+            ERR_clear_error();
+            return NGX_ERROR;
+        }
+
+        if (qc->error) {
+            ngx_connection_error(c, 0, "SSL_do_handshake() failed");
+            ERR_clear_error();
+            return NGX_ERROR;
+        }
+
+        if (sslerr != SSL_ERROR_WANT_READ) {
+            ngx_ssl_connection_error(c, sslerr, 0, "SSL_do_handshake() failed");
+            return NGX_ERROR;
+        }
+    }
+
+    if (qc->error) {
+        ngx_connection_error(c, 0, "SSL_do_handshake() failed");
+        return NGX_ERROR;
+    }
+
+    if (!SSL_is_init_finished(ssl_conn)) {
+        if (ngx_quic_keys_available(qc->keys, NGX_QUIC_ENCRYPTION_EARLY_DATA, 0)
+            && qc->client_tp_done)
+        {
+            if (ngx_quic_init_streams(c) != NGX_OK) {
+                return NGX_ERROR;
+            }
+        }
+
+        return NGX_OK;
+    }
+
+#if (NGX_DEBUG)
+    ngx_ssl_handshake_log(c);
+#endif
+
+    c->ssl->handshaked = 1;
+
+    frame = ngx_quic_alloc_frame(c);
+    if (frame == NULL) {
+        return NGX_ERROR;
+    }
+
+    frame->level = NGX_QUIC_ENCRYPTION_APPLICATION;
+    frame->type = NGX_QUIC_FT_HANDSHAKE_DONE;
+    ngx_quic_queue_frame(qc, frame);
+
+    if (qc->conf->retry) {
+        if (ngx_quic_send_new_token(c, qc->path) != NGX_OK) {
+            return NGX_ERROR;
+        }
+    }
+
+    /*
+     * RFC 9001, 9.5.  Header Protection Timing Side Channels
+     *
+     * Generating next keys before a key update is received.
+     */
+
+    ngx_post_event(&qc->key_update, &ngx_posted_events);
+
+    /*
+     * RFC 9001, 4.9.2.  Discarding Handshake Keys
+     *
+     * An endpoint MUST discard its Handshake keys
+     * when the TLS handshake is confirmed.
+     */
+    ngx_quic_discard_ctx(c, NGX_QUIC_ENCRYPTION_HANDSHAKE);
+
+    ngx_quic_discover_path_mtu(c, qc->path);
+
+    /* start accepting clients on negotiated number of server ids */
+    if (ngx_quic_create_sockets(c) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (ngx_quic_init_streams(c) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_quic_crypto_provide(ngx_connection_t *c, ngx_uint_t level)
+{
+#if (NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API)
+
+    ngx_buf_t                    *b;
+    ngx_chain_t                  *out, *cl;
+    ngx_quic_send_ctx_t          *ctx;
+    ngx_quic_connection_t        *qc;
+    enum ssl_encryption_level_t   ssl_level;
+
+    qc = ngx_quic_get_connection(c);
+    ctx = ngx_quic_get_send_ctx(qc, level);
+
+    out = ngx_quic_read_buffer(c, &ctx->crypto, (uint64_t) -1);
+    if (out == NGX_CHAIN_ERROR) {
+        return NGX_ERROR;
+    }
+
+    switch (level) {
+    case NGX_QUIC_ENCRYPTION_INITIAL:
+        ssl_level = ssl_encryption_initial;
+        break;
+    case NGX_QUIC_ENCRYPTION_EARLY_DATA:
+        ssl_level = ssl_encryption_early_data;
+        break;
+    case NGX_QUIC_ENCRYPTION_HANDSHAKE:
+        ssl_level = ssl_encryption_handshake;
+        break;
+    default: /* NGX_QUIC_ENCRYPTION_APPLICATION */
+        ssl_level = ssl_encryption_application;
+        break;
+    }
+
+    for (cl = out; cl; cl = cl->next) {
+        b = cl->buf;
+
+        if (!SSL_provide_quic_data(c->ssl->connection, ssl_level, b->pos,
+                                   b->last - b->pos))
+        {
+            ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                          "SSL_provide_quic_data() failed");
+            return NGX_ERROR;
+        }
+    }
+
+    ngx_quic_free_chain(c, out);
+
+#endif
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_init_connection(ngx_connection_t *c)
+{
+    u_char                 *p;
+    size_t                  clen;
+    ssize_t                 len;
+    ngx_str_t               dcid;
+    ngx_ssl_conn_t         *ssl_conn;
+    ngx_quic_socket_t      *qsock;
+    ngx_quic_connection_t  *qc;
+
+#if (NGX_QUIC_OPENSSL_API)
+    static const OSSL_DISPATCH  qtdis[] = {
+
+        { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND,
+          (void (*)(void)) ngx_quic_cbs_send },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD,
+          (void (*)(void)) ngx_quic_cbs_recv_rcd },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD,
+          (void (*)(void)) ngx_quic_cbs_release_rcd },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET,
+          (void (*)(void)) ngx_quic_cbs_yield_secret },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS,
+          (void (*)(void)) ngx_quic_cbs_got_transport_params },
+
+        { OSSL_FUNC_SSL_QUIC_TLS_ALERT,
+          (void (*)(void)) ngx_quic_cbs_alert },
+
+        { 0, NULL }
+    };
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+    static SSL_QUIC_METHOD  quic_method;
+#endif
+
+    qc = ngx_quic_get_connection(c);
+
+    if (ngx_ssl_create_connection(qc->conf->ssl, c, 0) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    c->ssl->no_wait_shutdown = 1;
+
+    ssl_conn = c->ssl->connection;
+
+#if (NGX_QUIC_OPENSSL_API)
+
+    if (SSL_set_quic_tls_cbs(ssl_conn, qtdis, c) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_tls_cbs() failed");
+        return NGX_ERROR;
+    }
+
+    if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) {
+        SSL_set_quic_tls_early_data_enabled(ssl_conn, 1);
+    }
+
+#else /* NGX_QUIC_BORINGSSL_API || NGX_QUIC_QUICTLS_API */
+
+    if (!quic_method.send_alert) {
+#if (NGX_QUIC_BORINGSSL_API)
+        quic_method.set_read_secret = ngx_quic_set_read_secret;
+        quic_method.set_write_secret = ngx_quic_set_write_secret;
+#else
+        quic_method.set_encryption_secrets = ngx_quic_set_encryption_secrets;
+#endif
+        quic_method.add_handshake_data = ngx_quic_add_handshake_data;
+        quic_method.flush_flight = ngx_quic_flush_flight;
+        quic_method.send_alert = ngx_quic_send_alert;
+    }
+
+    if (SSL_set_quic_method(ssl_conn, &quic_method) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_method() failed");
+        return NGX_ERROR;
+    }
+
+#if (NGX_QUIC_QUICTLS_API)
+    if (SSL_CTX_get_max_early_data(qc->conf->ssl->ctx)) {
+        SSL_set_quic_early_data_enabled(ssl_conn, 1);
+    }
+#endif
+
+#endif
+
+    qsock = ngx_quic_get_socket(c);
+
+    dcid.data = qsock->sid.id;
+    dcid.len = qsock->sid.len;
+
+    if (ngx_quic_new_sr_token(c, &dcid, qc->conf->sr_token_key, qc->tp.sr_token)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
+    len = ngx_quic_create_transport_params(NULL, NULL, &qc->tp, &clen);
+    /* always succeeds */
+
+    p = ngx_pnalloc(c->pool, len);
+    if (p == NULL) {
+        return NGX_ERROR;
+    }
+
+    len = ngx_quic_create_transport_params(p, p + len, &qc->tp, NULL);
+    if (len < 0) {
+        return NGX_ERROR;
+    }
+
+#ifdef NGX_QUIC_DEBUG_PACKETS
+    ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic transport parameters len:%uz %*xs", len, len, p);
+#endif
+
+#if (NGX_QUIC_OPENSSL_API)
+    if (SSL_set_quic_tls_transport_params(ssl_conn, p, len) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_tls_transport_params() failed");
+        return NGX_ERROR;
+    }
+#else
+    if (SSL_set_quic_transport_params(ssl_conn, p, len) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_transport_params() failed");
+        return NGX_ERROR;
+    }
+#endif
+
+#if (defined OPENSSL_IS_BORINGSSL || defined OPENSSL_IS_AWSLC)
+    if (SSL_set_quic_early_data_context(ssl_conn, p, clen) == 0) {
+        ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+                      "quic SSL_set_quic_early_data_context() failed");
+        return NGX_ERROR;
+    }
+#endif
+
+    return NGX_OK;
+}

nginx/src/event/quic/ngx_event_quic_ssl.h

@@ -0,0 +1,19 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_SSL_H_INCLUDED_
+#define _NGX_EVENT_QUIC_SSL_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+ngx_int_t ngx_quic_init_connection(ngx_connection_t *c);
+
+ngx_int_t ngx_quic_handle_crypto_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_frame_t *frame);
+
+#endif /* _NGX_EVENT_QUIC_SSL_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_streams.h

@@ -0,0 +1,44 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_STREAMS_H_INCLUDED_
+#define _NGX_EVENT_QUIC_STREAMS_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+ngx_int_t ngx_quic_handle_stream_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_frame_t *frame);
+void ngx_quic_handle_stream_ack(ngx_connection_t *c,
+    ngx_quic_frame_t *f);
+ngx_int_t ngx_quic_handle_max_data_frame(ngx_connection_t *c,
+    ngx_quic_max_data_frame_t *f);
+ngx_int_t ngx_quic_handle_streams_blocked_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_streams_blocked_frame_t *f);
+ngx_int_t ngx_quic_handle_data_blocked_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_data_blocked_frame_t *f);
+ngx_int_t ngx_quic_handle_stream_data_blocked_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_stream_data_blocked_frame_t *f);
+ngx_int_t ngx_quic_handle_max_stream_data_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_max_stream_data_frame_t *f);
+ngx_int_t ngx_quic_handle_reset_stream_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_reset_stream_frame_t *f);
+ngx_int_t ngx_quic_handle_stop_sending_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_stop_sending_frame_t *f);
+ngx_int_t ngx_quic_handle_max_streams_frame(ngx_connection_t *c,
+    ngx_quic_header_t *pkt, ngx_quic_max_streams_frame_t *f);
+
+ngx_int_t ngx_quic_init_streams(ngx_connection_t *c);
+void ngx_quic_rbtree_insert_stream(ngx_rbtree_node_t *temp,
+    ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
+ngx_quic_stream_t *ngx_quic_find_stream(ngx_rbtree_t *rbtree,
+    uint64_t id);
+ngx_int_t ngx_quic_close_streams(ngx_connection_t *c,
+    ngx_quic_connection_t *qc);
+
+#endif /* _NGX_EVENT_QUIC_STREAMS_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_tokens.c

@@ -0,0 +1,265 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+
+ngx_int_t
+ngx_quic_new_sr_token(ngx_connection_t *c, ngx_str_t *cid, u_char *secret,
+    u_char *token)
+{
+    ngx_str_t  tmp;
+    u_char     buf[NGX_QUIC_SR_KEY_LEN + sizeof(ngx_uint_t)];
+
+    ngx_memcpy(buf, secret, NGX_QUIC_SR_KEY_LEN);
+    ngx_memcpy(buf + NGX_QUIC_SR_KEY_LEN, &ngx_worker, sizeof(ngx_uint_t));
+
+    tmp.data = buf;
+    tmp.len = sizeof(buf);
+
+    if (ngx_quic_derive_key(c->log, "sr_token_key", &tmp, cid, token,
+                            NGX_QUIC_SR_TOKEN_LEN)
+        != NGX_OK)
+    {
+        return NGX_ERROR;
+    }
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic stateless reset token %*xs",
+                    (size_t) NGX_QUIC_SR_TOKEN_LEN, token);
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_new_token(ngx_log_t *log, struct sockaddr *sockaddr,
+    socklen_t socklen, u_char *key, ngx_str_t *token, ngx_str_t *odcid,
+    time_t exp, ngx_uint_t is_retry)
+{
+    int                len, iv_len;
+    u_char            *p, *iv;
+    EVP_CIPHER_CTX    *ctx;
+    const EVP_CIPHER  *cipher;
+
+    u_char             in[NGX_QUIC_MAX_TOKEN_SIZE];
+
+    ngx_quic_address_hash(sockaddr, socklen, !is_retry, NULL, 0, in);
+
+    p = in + 20;
+
+    p = ngx_cpymem(p, &exp, sizeof(time_t));
+
+    *p++ = is_retry ? 1 : 0;
+
+    if (odcid) {
+        *p++ = odcid->len;
+        p = ngx_cpymem(p, odcid->data, odcid->len);
+
+    } else {
+        *p++ = 0;
+    }
+
+    len = p - in;
+
+    cipher = EVP_aes_256_gcm();
+    iv_len = NGX_QUIC_AES_256_GCM_IV_LEN;
+
+    if ((size_t) (iv_len + len + NGX_QUIC_AES_256_GCM_TAG_LEN) > token->len) {
+        ngx_log_error(NGX_LOG_ALERT, log, 0, "quic token buffer is too small");
+        return NGX_ERROR;
+    }
+
+    ctx = EVP_CIPHER_CTX_new();
+    if (ctx == NULL) {
+        return NGX_ERROR;
+    }
+
+    iv = token->data;
+
+    if (RAND_bytes(iv, iv_len) <= 0
+        || !EVP_EncryptInit_ex(ctx, cipher, NULL, key, iv))
+    {
+        EVP_CIPHER_CTX_free(ctx);
+        return NGX_ERROR;
+    }
+
+    token->len = iv_len;
+
+    if (EVP_EncryptUpdate(ctx, token->data + token->len, &len, in, len) != 1) {
+        EVP_CIPHER_CTX_free(ctx);
+        return NGX_ERROR;
+    }
+
+    token->len += len;
+
+    if (EVP_EncryptFinal_ex(ctx, token->data + token->len, &len) <= 0) {
+        EVP_CIPHER_CTX_free(ctx);
+        return NGX_ERROR;
+    }
+
+    token->len += len;
+
+    if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG,
+                            NGX_QUIC_AES_256_GCM_TAG_LEN,
+                            token->data + token->len)
+        == 0)
+    {
+        EVP_CIPHER_CTX_free(ctx);
+        return NGX_ERROR;
+    }
+
+    token->len += NGX_QUIC_AES_256_GCM_TAG_LEN;
+
+    EVP_CIPHER_CTX_free(ctx);
+
+#ifdef NGX_QUIC_DEBUG_PACKETS
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0,
+                   "quic new token len:%uz %xV", token->len, token);
+#endif
+
+    return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_validate_token(ngx_connection_t *c, u_char *key,
+    ngx_quic_header_t *pkt)
+{
+    int                len, tlen, iv_len;
+    u_char            *iv, *p;
+    time_t             now, exp;
+    size_t             total;
+    ngx_str_t          odcid;
+    EVP_CIPHER_CTX    *ctx;
+    const EVP_CIPHER  *cipher;
+
+    u_char             addr_hash[20];
+    u_char             tdec[NGX_QUIC_MAX_TOKEN_SIZE];
+
+#if NGX_SUPPRESS_WARN
+    ngx_str_null(&odcid);
+#endif
+
+    /* Retry token or NEW_TOKEN in a previous connection */
+
+    cipher = EVP_aes_256_gcm();
+    iv = pkt->token.data;
+    iv_len = NGX_QUIC_AES_256_GCM_IV_LEN;
+
+    /* sanity checks */
+
+    if (pkt->token.len < (size_t) iv_len + NGX_QUIC_AES_256_GCM_TAG_LEN) {
+        goto garbage;
+    }
+
+    if (pkt->token.len > (size_t) iv_len + NGX_QUIC_MAX_TOKEN_SIZE
+                         + NGX_QUIC_AES_256_GCM_TAG_LEN)
+    {
+        goto garbage;
+    }
+
+    ctx = EVP_CIPHER_CTX_new();
+    if (ctx == NULL) {
+        return NGX_ERROR;
+    }
+
+    if (!EVP_DecryptInit_ex(ctx, cipher, NULL, key, iv)) {
+        EVP_CIPHER_CTX_free(ctx);
+        return NGX_ERROR;
+    }
+
+    p = pkt->token.data + iv_len;
+    len = pkt->token.len - iv_len - NGX_QUIC_AES_256_GCM_TAG_LEN;
+
+    if (EVP_DecryptUpdate(ctx, tdec, &tlen, p, len) != 1) {
+        EVP_CIPHER_CTX_free(ctx);
+        goto garbage;
+    }
+    total = tlen;
+
+    if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG,
+                            NGX_QUIC_AES_256_GCM_TAG_LEN, p + len)
+        == 0)
+    {
+        EVP_CIPHER_CTX_free(ctx);
+        goto garbage;
+    }
+
+    if (EVP_DecryptFinal_ex(ctx, tdec + tlen, &tlen) <= 0) {
+        EVP_CIPHER_CTX_free(ctx);
+        goto garbage;
+    }
+    total += tlen;
+
+    EVP_CIPHER_CTX_free(ctx);
+
+    if (total < (20 + sizeof(time_t) + 2)) {
+        goto garbage;
+    }
+
+    p = tdec + 20;
+
+    ngx_memcpy(&exp, p, sizeof(time_t));
+    p += sizeof(time_t);
+
+    pkt->retried = (*p++ == 1);
+
+    ngx_quic_address_hash(c->sockaddr, c->socklen, !pkt->retried, NULL, 0,
+                          addr_hash);
+
+    if (ngx_memcmp(tdec, addr_hash, 20) != 0) {
+        goto bad_token;
+    }
+
+    odcid.len = *p++;
+    if (odcid.len) {
+        if (odcid.len > NGX_QUIC_MAX_CID_LEN) {
+            goto bad_token;
+        }
+
+        if ((size_t)(tdec + total - p) < odcid.len) {
+            goto bad_token;
+        }
+
+        odcid.data = p;
+    }
+
+    now = ngx_time();
+
+    if (now > exp) {
+        ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic expired token");
+        return NGX_DECLINED;
+    }
+
+    if (odcid.len) {
+        pkt->odcid.len = odcid.len;
+        pkt->odcid.data = pkt->odcid_buf;
+        ngx_memcpy(pkt->odcid.data, odcid.data, odcid.len);
+
+    } else {
+        pkt->odcid = pkt->dcid;
+    }
+
+    pkt->validated = 1;
+
+    return NGX_OK;
+
+garbage:
+
+    ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic garbage token");
+
+    return NGX_ABORT;
+
+bad_token:
+
+    ngx_log_error(NGX_LOG_INFO, c->log, 0, "quic invalid token");
+
+    return NGX_DECLINED;
+}

nginx/src/event/quic/ngx_event_quic_tokens.h

@@ -0,0 +1,34 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_TOKENS_H_INCLUDED_
+#define _NGX_EVENT_QUIC_TOKENS_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+#define NGX_QUIC_MAX_TOKEN_SIZE              64
+    /* SHA-1(addr)=20 + sizeof(time_t) + retry(1) + odcid.len(1) + odcid */
+
+#define NGX_QUIC_AES_256_GCM_IV_LEN          12
+#define NGX_QUIC_AES_256_GCM_TAG_LEN         16
+
+#define NGX_QUIC_TOKEN_BUF_SIZE             (NGX_QUIC_AES_256_GCM_IV_LEN      \
+                                             + NGX_QUIC_MAX_TOKEN_SIZE        \
+                                             + NGX_QUIC_AES_256_GCM_TAG_LEN)
+
+
+ngx_int_t ngx_quic_new_sr_token(ngx_connection_t *c, ngx_str_t *cid,
+    u_char *secret, u_char *token);
+ngx_int_t ngx_quic_new_token(ngx_log_t *log, struct sockaddr *sockaddr,
+    socklen_t socklen, u_char *key, ngx_str_t *token, ngx_str_t *odcid,
+    time_t expires, ngx_uint_t is_retry);
+ngx_int_t ngx_quic_validate_token(ngx_connection_t *c,
+    u_char *key, ngx_quic_header_t *pkt);
+
+#endif /* _NGX_EVENT_QUIC_TOKENS_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_transport.h

@@ -0,0 +1,398 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_TRANSPORT_H_INCLUDED_
+#define _NGX_EVENT_QUIC_TRANSPORT_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+/*
+ * RFC 9000, 17.2.  Long Header Packets
+ *           17.3.  Short Header Packets
+ *
+ * QUIC flags in first byte
+ */
+#define NGX_QUIC_PKT_LONG       0x80  /* header form */
+#define NGX_QUIC_PKT_FIXED_BIT  0x40
+#define NGX_QUIC_PKT_TYPE       0x30  /* in long packet */
+#define NGX_QUIC_PKT_KPHASE     0x04  /* in short packet */
+
+#define ngx_quic_long_pkt(flags)  ((flags) & NGX_QUIC_PKT_LONG)
+#define ngx_quic_short_pkt(flags)  (((flags) & NGX_QUIC_PKT_LONG) == 0)
+
+/* Long packet types */
+#define NGX_QUIC_PKT_INITIAL    0x00
+#define NGX_QUIC_PKT_ZRTT       0x10
+#define NGX_QUIC_PKT_HANDSHAKE  0x20
+#define NGX_QUIC_PKT_RETRY      0x30
+
+#define ngx_quic_pkt_in(flags)                                                \
+    (((flags) & NGX_QUIC_PKT_TYPE) == NGX_QUIC_PKT_INITIAL)
+#define ngx_quic_pkt_zrtt(flags)                                              \
+    (((flags) & NGX_QUIC_PKT_TYPE) == NGX_QUIC_PKT_ZRTT)
+#define ngx_quic_pkt_hs(flags)                                                \
+    (((flags) & NGX_QUIC_PKT_TYPE) == NGX_QUIC_PKT_HANDSHAKE)
+#define ngx_quic_pkt_retry(flags)                                             \
+    (((flags) & NGX_QUIC_PKT_TYPE) == NGX_QUIC_PKT_RETRY)
+
+#define ngx_quic_pkt_rb_mask(flags)                                           \
+    (ngx_quic_long_pkt(flags) ? 0x0C : 0x18)
+#define ngx_quic_pkt_hp_mask(flags)                                           \
+    (ngx_quic_long_pkt(flags) ? 0x0F : 0x1F)
+
+#define ngx_quic_level_name(lvl)                                              \
+    (lvl == NGX_QUIC_ENCRYPTION_APPLICATION) ? "app"                          \
+        : (lvl == NGX_QUIC_ENCRYPTION_INITIAL) ? "init"                       \
+            : (lvl == NGX_QUIC_ENCRYPTION_HANDSHAKE) ? "hs" : "early"
+
+#define NGX_QUIC_MAX_CID_LEN                             20
+#define NGX_QUIC_SERVER_CID_LEN                          NGX_QUIC_MAX_CID_LEN
+
+/* 12.4.  Frames and Frame Types */
+#define NGX_QUIC_FT_PADDING                              0x00
+#define NGX_QUIC_FT_PING                                 0x01
+#define NGX_QUIC_FT_ACK                                  0x02
+#define NGX_QUIC_FT_ACK_ECN                              0x03
+#define NGX_QUIC_FT_RESET_STREAM                         0x04
+#define NGX_QUIC_FT_STOP_SENDING                         0x05
+#define NGX_QUIC_FT_CRYPTO                               0x06
+#define NGX_QUIC_FT_NEW_TOKEN                            0x07
+#define NGX_QUIC_FT_STREAM                               0x08
+#define NGX_QUIC_FT_STREAM1                              0x09
+#define NGX_QUIC_FT_STREAM2                              0x0A
+#define NGX_QUIC_FT_STREAM3                              0x0B
+#define NGX_QUIC_FT_STREAM4                              0x0C
+#define NGX_QUIC_FT_STREAM5                              0x0D
+#define NGX_QUIC_FT_STREAM6                              0x0E
+#define NGX_QUIC_FT_STREAM7                              0x0F
+#define NGX_QUIC_FT_MAX_DATA                             0x10
+#define NGX_QUIC_FT_MAX_STREAM_DATA                      0x11
+#define NGX_QUIC_FT_MAX_STREAMS                          0x12
+#define NGX_QUIC_FT_MAX_STREAMS2                         0x13
+#define NGX_QUIC_FT_DATA_BLOCKED                         0x14
+#define NGX_QUIC_FT_STREAM_DATA_BLOCKED                  0x15
+#define NGX_QUIC_FT_STREAMS_BLOCKED                      0x16
+#define NGX_QUIC_FT_STREAMS_BLOCKED2                     0x17
+#define NGX_QUIC_FT_NEW_CONNECTION_ID                    0x18
+#define NGX_QUIC_FT_RETIRE_CONNECTION_ID                 0x19
+#define NGX_QUIC_FT_PATH_CHALLENGE                       0x1A
+#define NGX_QUIC_FT_PATH_RESPONSE                        0x1B
+#define NGX_QUIC_FT_CONNECTION_CLOSE                     0x1C
+#define NGX_QUIC_FT_CONNECTION_CLOSE_APP                 0x1D
+#define NGX_QUIC_FT_HANDSHAKE_DONE                       0x1E
+
+#define NGX_QUIC_FT_LAST  NGX_QUIC_FT_HANDSHAKE_DONE
+
+/* 22.5.  QUIC Transport Error Codes Registry */
+#define NGX_QUIC_ERR_NO_ERROR                            0x00
+#define NGX_QUIC_ERR_INTERNAL_ERROR                      0x01
+#define NGX_QUIC_ERR_CONNECTION_REFUSED                  0x02
+#define NGX_QUIC_ERR_FLOW_CONTROL_ERROR                  0x03
+#define NGX_QUIC_ERR_STREAM_LIMIT_ERROR                  0x04
+#define NGX_QUIC_ERR_STREAM_STATE_ERROR                  0x05
+#define NGX_QUIC_ERR_FINAL_SIZE_ERROR                    0x06
+#define NGX_QUIC_ERR_FRAME_ENCODING_ERROR                0x07
+#define NGX_QUIC_ERR_TRANSPORT_PARAMETER_ERROR           0x08
+#define NGX_QUIC_ERR_CONNECTION_ID_LIMIT_ERROR           0x09
+#define NGX_QUIC_ERR_PROTOCOL_VIOLATION                  0x0A
+#define NGX_QUIC_ERR_INVALID_TOKEN                       0x0B
+#define NGX_QUIC_ERR_APPLICATION_ERROR                   0x0C
+#define NGX_QUIC_ERR_CRYPTO_BUFFER_EXCEEDED              0x0D
+#define NGX_QUIC_ERR_KEY_UPDATE_ERROR                    0x0E
+#define NGX_QUIC_ERR_AEAD_LIMIT_REACHED                  0x0F
+#define NGX_QUIC_ERR_NO_VIABLE_PATH                      0x10
+
+#define NGX_QUIC_ERR_CRYPTO_ERROR                       0x100
+
+#define NGX_QUIC_ERR_CRYPTO(e)  (NGX_QUIC_ERR_CRYPTO_ERROR + (e))
+
+
+/* 22.3.  QUIC Transport Parameters Registry */
+#define NGX_QUIC_TP_ORIGINAL_DCID                        0x00
+#define NGX_QUIC_TP_MAX_IDLE_TIMEOUT                     0x01
+#define NGX_QUIC_TP_SR_TOKEN                             0x02
+#define NGX_QUIC_TP_MAX_UDP_PAYLOAD_SIZE                 0x03
+#define NGX_QUIC_TP_INITIAL_MAX_DATA                     0x04
+#define NGX_QUIC_TP_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL   0x05
+#define NGX_QUIC_TP_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE  0x06
+#define NGX_QUIC_TP_INITIAL_MAX_STREAM_DATA_UNI          0x07
+#define NGX_QUIC_TP_INITIAL_MAX_STREAMS_BIDI             0x08
+#define NGX_QUIC_TP_INITIAL_MAX_STREAMS_UNI              0x09
+#define NGX_QUIC_TP_ACK_DELAY_EXPONENT                   0x0A
+#define NGX_QUIC_TP_MAX_ACK_DELAY                        0x0B
+#define NGX_QUIC_TP_DISABLE_ACTIVE_MIGRATION             0x0C
+#define NGX_QUIC_TP_PREFERRED_ADDRESS                    0x0D
+#define NGX_QUIC_TP_ACTIVE_CONNECTION_ID_LIMIT           0x0E
+#define NGX_QUIC_TP_INITIAL_SCID                         0x0F
+#define NGX_QUIC_TP_RETRY_SCID                           0x10
+
+#define NGX_QUIC_CID_LEN_MIN                                8
+#define NGX_QUIC_CID_LEN_MAX                               20
+
+#define NGX_QUIC_MAX_RANGES                                10
+
+
+typedef struct {
+    uint64_t                                    gap;
+    uint64_t                                    range;
+} ngx_quic_ack_range_t;
+
+
+typedef struct {
+    uint64_t                                    largest;
+    uint64_t                                    delay;
+    uint64_t                                    range_count;
+    uint64_t                                    first_range;
+    uint64_t                                    ect0;
+    uint64_t                                    ect1;
+    uint64_t                                    ce;
+    uint64_t                                    ranges_length;
+} ngx_quic_ack_frame_t;
+
+
+typedef struct {
+    uint64_t                                    seqnum;
+    uint64_t                                    retire;
+    uint8_t                                     len;
+    u_char                                      cid[NGX_QUIC_CID_LEN_MAX];
+    u_char                                      srt[NGX_QUIC_SR_TOKEN_LEN];
+} ngx_quic_new_conn_id_frame_t;
+
+
+typedef struct {
+    uint64_t                                    length;
+} ngx_quic_new_token_frame_t;
+
+/*
+ * common layout for CRYPTO and STREAM frames;
+ * conceptually, CRYPTO frame is also a stream
+ * frame lacking some properties
+ */
+typedef struct {
+    uint64_t                                    offset;
+    uint64_t                                    length;
+} ngx_quic_ordered_frame_t;
+
+typedef ngx_quic_ordered_frame_t  ngx_quic_crypto_frame_t;
+
+
+typedef struct {
+    /* initial fields same as in ngx_quic_ordered_frame_t */
+    uint64_t                                    offset;
+    uint64_t                                    length;
+
+    uint64_t                                    stream_id;
+    unsigned                                    off:1;
+    unsigned                                    len:1;
+    unsigned                                    fin:1;
+} ngx_quic_stream_frame_t;
+
+
+typedef struct {
+    uint64_t                                    max_data;
+} ngx_quic_max_data_frame_t;
+
+
+typedef struct {
+    uint64_t                                    error_code;
+    uint64_t                                    frame_type;
+    ngx_str_t                                   reason;
+} ngx_quic_close_frame_t;
+
+
+typedef struct {
+    uint64_t                                    id;
+    uint64_t                                    error_code;
+    uint64_t                                    final_size;
+} ngx_quic_reset_stream_frame_t;
+
+
+typedef struct {
+    uint64_t                                    id;
+    uint64_t                                    error_code;
+} ngx_quic_stop_sending_frame_t;
+
+
+typedef struct {
+    uint64_t                                    limit;
+    ngx_uint_t                                  bidi;  /* unsigned: bidi:1 */
+} ngx_quic_streams_blocked_frame_t;
+
+
+typedef struct {
+    uint64_t                                    limit;
+    ngx_uint_t                                  bidi;  /* unsigned: bidi:1 */
+} ngx_quic_max_streams_frame_t;
+
+
+typedef struct {
+    uint64_t                                    id;
+    uint64_t                                    limit;
+} ngx_quic_max_stream_data_frame_t;
+
+
+typedef struct {
+    uint64_t                                    limit;
+} ngx_quic_data_blocked_frame_t;
+
+
+typedef struct {
+    uint64_t                                    id;
+    uint64_t                                    limit;
+} ngx_quic_stream_data_blocked_frame_t;
+
+
+typedef struct {
+    uint64_t                                    sequence_number;
+} ngx_quic_retire_cid_frame_t;
+
+
+typedef struct {
+    u_char                                      data[8];
+} ngx_quic_path_challenge_frame_t;
+
+
+typedef struct ngx_quic_frame_s                 ngx_quic_frame_t;
+
+struct ngx_quic_frame_s {
+    ngx_uint_t                                  type;
+    ngx_uint_t                                  level;
+    ngx_queue_t                                 queue;
+    uint64_t                                    pnum;
+    size_t                                      plen;
+    ngx_msec_t                                  send_time;
+    ssize_t                                     len;
+    unsigned                                    need_ack:1;
+    unsigned                                    pkt_need_ack:1;
+    unsigned                                    ignore_congestion:1;
+    unsigned                                    ignore_loss:1;
+
+    ngx_chain_t                                *data;
+    union {
+        ngx_quic_ack_frame_t                    ack;
+        ngx_quic_crypto_frame_t                 crypto;
+        ngx_quic_ordered_frame_t                ord;
+        ngx_quic_new_conn_id_frame_t            ncid;
+        ngx_quic_new_token_frame_t              token;
+        ngx_quic_stream_frame_t                 stream;
+        ngx_quic_max_data_frame_t               max_data;
+        ngx_quic_close_frame_t                  close;
+        ngx_quic_reset_stream_frame_t           reset_stream;
+        ngx_quic_stop_sending_frame_t           stop_sending;
+        ngx_quic_streams_blocked_frame_t        streams_blocked;
+        ngx_quic_max_streams_frame_t            max_streams;
+        ngx_quic_max_stream_data_frame_t        max_stream_data;
+        ngx_quic_data_blocked_frame_t           data_blocked;
+        ngx_quic_stream_data_blocked_frame_t    stream_data_blocked;
+        ngx_quic_retire_cid_frame_t             retire_cid;
+        ngx_quic_path_challenge_frame_t         path_challenge;
+        ngx_quic_path_challenge_frame_t         path_response;
+    } u;
+};
+
+
+typedef struct {
+    ngx_log_t                                  *log;
+    ngx_quic_path_t                            *path;
+
+    ngx_quic_keys_t                            *keys;
+
+    ngx_msec_t                                  received;
+    uint64_t                                    number;
+    uint8_t                                     num_len;
+    uint32_t                                    trunc;
+    uint8_t                                     flags;
+    uint32_t                                    version;
+    ngx_str_t                                   token;
+    ngx_uint_t                                  level;
+    ngx_uint_t                                  error;
+
+    /* filled in by parser */
+    ngx_buf_t                                  *raw;   /* udp datagram */
+
+    u_char                                     *data;  /* quic packet */
+    size_t                                      len;
+
+    /* cleartext fields */
+    ngx_str_t                                   odcid; /* retry packet tag */
+    u_char                                      odcid_buf[NGX_QUIC_MAX_CID_LEN];
+    ngx_str_t                                   dcid;
+    ngx_str_t                                   scid;
+    uint64_t                                    pn;
+    u_char                                     *plaintext;
+    ngx_str_t                                   payload; /* decrypted data */
+
+    unsigned                                    need_ack:1;
+    unsigned                                    key_phase:1;
+    unsigned                                    key_update:1;
+    unsigned                                    parsed:1;
+    unsigned                                    decrypted:1;
+    unsigned                                    validated:1;
+    unsigned                                    retried:1;
+    unsigned                                    first:1;
+    unsigned                                    rebound:1;
+    unsigned                                    path_challenged:1;
+} ngx_quic_header_t;
+
+
+typedef struct {
+    ngx_msec_t                 max_idle_timeout;
+    ngx_msec_t                 max_ack_delay;
+
+    size_t                     max_udp_payload_size;
+    size_t                     initial_max_data;
+    size_t                     initial_max_stream_data_bidi_local;
+    size_t                     initial_max_stream_data_bidi_remote;
+    size_t                     initial_max_stream_data_uni;
+    ngx_uint_t                 initial_max_streams_bidi;
+    ngx_uint_t                 initial_max_streams_uni;
+    ngx_uint_t                 ack_delay_exponent;
+    ngx_uint_t                 active_connection_id_limit;
+    ngx_flag_t                 disable_active_migration;
+
+    ngx_str_t                  original_dcid;
+    ngx_str_t                  initial_scid;
+    ngx_str_t                  retry_scid;
+    u_char                     sr_token[NGX_QUIC_SR_TOKEN_LEN];
+
+    /* TODO */
+    void                      *preferred_address;
+} ngx_quic_tp_t;
+
+
+ngx_int_t ngx_quic_parse_packet(ngx_quic_header_t *pkt);
+
+size_t ngx_quic_create_version_negotiation(ngx_quic_header_t *pkt, u_char *out);
+
+size_t ngx_quic_payload_size(ngx_quic_header_t *pkt, size_t pkt_len);
+
+size_t ngx_quic_create_header(ngx_quic_header_t *pkt, u_char *out,
+    u_char **pnp);
+
+size_t ngx_quic_create_retry_itag(ngx_quic_header_t *pkt, u_char *out,
+    u_char **start);
+
+ssize_t ngx_quic_parse_frame(ngx_quic_header_t *pkt, u_char *start, u_char *end,
+    ngx_quic_frame_t *frame);
+ssize_t ngx_quic_create_frame(u_char *p, ngx_quic_frame_t *f);
+
+ssize_t ngx_quic_parse_ack_range(ngx_log_t *log, u_char *start,
+    u_char *end, uint64_t *gap, uint64_t *range);
+size_t ngx_quic_create_ack_range(u_char *p, uint64_t gap, uint64_t range);
+
+ngx_int_t ngx_quic_init_transport_params(ngx_quic_tp_t *tp,
+    ngx_quic_conf_t *qcf);
+ngx_int_t ngx_quic_parse_transport_params(u_char *p, u_char *end,
+    ngx_quic_tp_t *tp, ngx_log_t *log);
+ssize_t ngx_quic_create_transport_params(u_char *p, u_char *end,
+    ngx_quic_tp_t *tp, size_t *clen);
+
+void ngx_quic_dcid_encode_key(u_char *dcid, uint64_t key);
+
+#endif /* _NGX_EVENT_QUIC_TRANSPORT_H_INCLUDED_ */

nginx/src/event/quic/ngx_event_quic_udp.c

@@ -0,0 +1,420 @@
+
+/*
+ * Copyright (C) Roman Arutyunyan
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>
+
+
+static void ngx_quic_close_accepted_connection(ngx_connection_t *c);
+static ngx_connection_t *ngx_quic_lookup_connection(ngx_listening_t *ls,
+    ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen);
+
+
+void
+ngx_quic_recvmsg(ngx_event_t *ev)
+{
+    ssize_t             n;
+    ngx_str_t           key;
+    ngx_buf_t           buf;
+    ngx_log_t          *log;
+    ngx_err_t           err;
+    socklen_t           socklen, local_socklen;
+    ngx_event_t        *rev, *wev;
+    struct iovec        iov[1];
+    struct msghdr       msg;
+    ngx_sockaddr_t      sa, lsa;
+    struct sockaddr    *sockaddr, *local_sockaddr;
+    ngx_listening_t    *ls;
+    ngx_event_conf_t   *ecf;
+    ngx_connection_t   *c, *lc;
+    ngx_quic_socket_t  *qsock;
+    static u_char       buffer[NGX_QUIC_MAX_UDP_PAYLOAD_SIZE];
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+    u_char             msg_control[CMSG_SPACE(sizeof(ngx_addrinfo_t))];
+#endif
+
+    if (ev->timedout) {
+        if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) {
+            return;
+        }
+
+        ev->timedout = 0;
+    }
+
+    ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
+
+    if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) {
+        ev->available = ecf->multi_accept;
+    }
+
+    lc = ev->data;
+    ls = lc->listening;
+    ev->ready = 0;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+                   "quic recvmsg on %V, ready: %d",
+                   &ls->addr_text, ev->available);
+
+    do {
+        ngx_memzero(&msg, sizeof(struct msghdr));
+
+        iov[0].iov_base = (void *) buffer;
+        iov[0].iov_len = sizeof(buffer);
+
+        msg.msg_name = &sa;
+        msg.msg_namelen = sizeof(ngx_sockaddr_t);
+        msg.msg_iov = iov;
+        msg.msg_iovlen = 1;
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+        if (ls->wildcard) {
+            msg.msg_control = &msg_control;
+            msg.msg_controllen = sizeof(msg_control);
+
+            ngx_memzero(&msg_control, sizeof(msg_control));
+        }
+#endif
+
+        n = recvmsg(lc->fd, &msg, 0);
+
+        if (n == -1) {
+            err = ngx_socket_errno;
+
+            if (err == NGX_EAGAIN) {
+                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err,
+                               "quic recvmsg() not ready");
+                return;
+            }
+
+            ngx_log_error(NGX_LOG_ALERT, ev->log, err, "quic recvmsg() failed");
+
+            return;
+        }
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+        if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
+            ngx_log_error(NGX_LOG_ALERT, ev->log, 0,
+                          "quic recvmsg() truncated data");
+            continue;
+        }
+#endif
+
+        sockaddr = msg.msg_name;
+        socklen = msg.msg_namelen;
+
+        if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
+            socklen = sizeof(ngx_sockaddr_t);
+        }
+
+#if (NGX_HAVE_UNIX_DOMAIN)
+
+        if (sockaddr->sa_family == AF_UNIX) {
+            struct sockaddr_un *saun = (struct sockaddr_un *) sockaddr;
+
+            if (socklen <= (socklen_t) offsetof(struct sockaddr_un, sun_path)
+                || saun->sun_path[0] == '\0')
+            {
+                ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0,
+                               "unbound unix socket");
+                goto next;
+            }
+        }
+
+#endif
+
+        local_sockaddr = ls->sockaddr;
+        local_socklen = ls->socklen;
+
+#if (NGX_HAVE_ADDRINFO_CMSG)
+
+        if (ls->wildcard) {
+            struct cmsghdr  *cmsg;
+
+            ngx_memcpy(&lsa, local_sockaddr, local_socklen);
+            local_sockaddr = &lsa.sockaddr;
+
+            for (cmsg = CMSG_FIRSTHDR(&msg);
+                 cmsg != NULL;
+                 cmsg = CMSG_NXTHDR(&msg, cmsg))
+            {
+                if (ngx_get_srcaddr_cmsg(cmsg, local_sockaddr) == NGX_OK) {
+                    break;
+                }
+            }
+        }
+
+#endif
+
+        if (ngx_quic_get_packet_dcid(ev->log, buffer, n, &key) != NGX_OK) {
+            goto next;
+        }
+
+        c = ngx_quic_lookup_connection(ls, &key, local_sockaddr, local_socklen);
+
+        if (c) {
+
+#if (NGX_DEBUG)
+            if (c->log->log_level & NGX_LOG_DEBUG_EVENT) {
+                ngx_log_handler_pt  handler;
+
+                handler = c->log->handler;
+                c->log->handler = NULL;
+
+                ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                               "quic recvmsg: fd:%d n:%z", c->fd, n);
+
+                c->log->handler = handler;
+            }
+#endif
+
+            ngx_memzero(&buf, sizeof(ngx_buf_t));
+
+            buf.pos = buffer;
+            buf.last = buffer + n;
+            buf.start = buf.pos;
+            buf.end = buffer + sizeof(buffer);
+
+            qsock = ngx_quic_get_socket(c);
+
+            ngx_memcpy(&qsock->sockaddr, sockaddr, socklen);
+            qsock->socklen = socklen;
+
+            c->udp->buffer = &buf;
+
+            rev = c->read;
+            rev->ready = 1;
+            rev->active = 0;
+
+            rev->handler(rev);
+
+            if (c->udp) {
+                c->udp->buffer = NULL;
+            }
+
+            rev->ready = 0;
+            rev->active = 1;
+
+            goto next;
+        }
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1);
+#endif
+
+        ngx_accept_disabled = ngx_cycle->connection_n / 8
+                              - ngx_cycle->free_connection_n;
+
+        c = ngx_get_connection(lc->fd, ev->log);
+        if (c == NULL) {
+            return;
+        }
+
+        c->shared = 1;
+        c->type = SOCK_DGRAM;
+        c->socklen = socklen;
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_active, 1);
+#endif
+
+        c->pool = ngx_create_pool(ls->pool_size, ev->log);
+        if (c->pool == NULL) {
+            ngx_quic_close_accepted_connection(c);
+            return;
+        }
+
+        c->sockaddr = ngx_palloc(c->pool, NGX_SOCKADDRLEN);
+        if (c->sockaddr == NULL) {
+            ngx_quic_close_accepted_connection(c);
+            return;
+        }
+
+        ngx_memcpy(c->sockaddr, sockaddr, socklen);
+
+        log = ngx_palloc(c->pool, sizeof(ngx_log_t));
+        if (log == NULL) {
+            ngx_quic_close_accepted_connection(c);
+            return;
+        }
+
+        *log = ls->log;
+
+        c->log = log;
+        c->pool->log = log;
+        c->listening = ls;
+
+        if (local_sockaddr == &lsa.sockaddr) {
+            local_sockaddr = ngx_palloc(c->pool, local_socklen);
+            if (local_sockaddr == NULL) {
+                ngx_quic_close_accepted_connection(c);
+                return;
+            }
+
+            ngx_memcpy(local_sockaddr, &lsa, local_socklen);
+        }
+
+        c->local_sockaddr = local_sockaddr;
+        c->local_socklen = local_socklen;
+
+        c->buffer = ngx_create_temp_buf(c->pool, n);
+        if (c->buffer == NULL) {
+            ngx_quic_close_accepted_connection(c);
+            return;
+        }
+
+        c->buffer->last = ngx_cpymem(c->buffer->last, buffer, n);
+
+        rev = c->read;
+        wev = c->write;
+
+        rev->active = 1;
+        wev->ready = 1;
+
+        rev->log = log;
+        wev->log = log;
+
+        /*
+         * TODO: MT: - ngx_atomic_fetch_add()
+         *             or protection by critical section or light mutex
+         *
+         * TODO: MP: - allocated in a shared memory
+         *           - ngx_atomic_fetch_add()
+         *             or protection by critical section or light mutex
+         */
+
+        c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1);
+
+        c->start_time = ngx_current_msec;
+
+#if (NGX_STAT_STUB)
+        (void) ngx_atomic_fetch_add(ngx_stat_handled, 1);
+#endif
+
+        if (ls->addr_ntop) {
+            c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len);
+            if (c->addr_text.data == NULL) {
+                ngx_quic_close_accepted_connection(c);
+                return;
+            }
+
+            c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen,
+                                             c->addr_text.data,
+                                             ls->addr_text_max_len, 0);
+            if (c->addr_text.len == 0) {
+                ngx_quic_close_accepted_connection(c);
+                return;
+            }
+        }
+
+#if (NGX_DEBUG)
+        {
+        ngx_str_t  addr;
+        u_char     text[NGX_SOCKADDR_STRLEN];
+
+        ngx_debug_accepted_connection(ecf, c);
+
+        if (log->log_level & NGX_LOG_DEBUG_EVENT) {
+            addr.data = text;
+            addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text,
+                                     NGX_SOCKADDR_STRLEN, 1);
+
+            ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0,
+                           "*%uA quic recvmsg: %V fd:%d n:%z",
+                           c->number, &addr, c->fd, n);
+        }
+
+        }
+#endif
+
+        log->data = NULL;
+        log->handler = NULL;
+
+        ls->handler(c);
+
+    next:
+
+        if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
+            ev->available -= n;
+        }
+
+    } while (ev->available);
+}
+
+
+static void
+ngx_quic_close_accepted_connection(ngx_connection_t *c)
+{
+    ngx_free_connection(c);
+
+    c->fd = (ngx_socket_t) -1;
+
+    if (c->pool) {
+        ngx_destroy_pool(c->pool);
+    }
+
+#if (NGX_STAT_STUB)
+    (void) ngx_atomic_fetch_add(ngx_stat_active, -1);
+#endif
+}
+
+
+static ngx_connection_t *
+ngx_quic_lookup_connection(ngx_listening_t *ls, ngx_str_t *key,
+    struct sockaddr *local_sockaddr, socklen_t local_socklen)
+{
+    uint32_t            hash;
+    ngx_int_t           rc;
+    ngx_connection_t   *c;
+    ngx_rbtree_node_t  *node, *sentinel;
+    ngx_quic_socket_t  *qsock;
+
+    if (key->len == 0) {
+        return NULL;
+    }
+
+    node = ls->rbtree.root;
+    sentinel = ls->rbtree.sentinel;
+    hash = ngx_crc32_long(key->data, key->len);
+
+    while (node != sentinel) {
+
+        if (hash < node->key) {
+            node = node->left;
+            continue;
+        }
+
+        if (hash > node->key) {
+            node = node->right;
+            continue;
+        }
+
+        /* hash == node->key */
+
+        qsock = (ngx_quic_socket_t *) node;
+
+        rc = ngx_memn2cmp(key->data, qsock->sid.id, key->len, qsock->sid.len);
+
+        c = qsock->udp.connection;
+
+        if (rc == 0 && ls->wildcard) {
+            rc = ngx_cmp_sockaddr(local_sockaddr, local_socklen,
+                                  c->local_sockaddr, c->local_socklen, 1);
+        }
+
+        if (rc == 0) {
+            c->udp = &qsock->udp;
+            return c;
+        }
+
+        node = (rc < 0) ? node->left : node->right;
+    }
+
+    return NULL;
+}