QUIC: reusable mode for main connection.

The connection is automatically switched to this mode by transport layer when
there are no non-cancelable streams.  Currently, cancelable streams are
HTTP/3 encoder/decoder/control streams.

src/event/quic/ngx_event_quic.c

@@ -341,6 +341,8 @@ ngx_quic_new_connection(ngx_connection_t *c, ngx_quic_conf_t *conf,
         return NULL;
     }
 
+    ngx_reusable_connection(c, 1);
+
     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
                    "quic connection created");
 
@@ -420,7 +422,7 @@ ngx_quic_input_handler(ngx_event_t *rev)
     if (c->close) {
         qc->error = NGX_QUIC_ERR_NO_ERROR;
         qc->error_reason = "graceful shutdown";
-        ngx_quic_close_connection(c, NGX_OK);
+        ngx_quic_close_connection(c, NGX_ERROR);
         return;
     }
 
@@ -603,12 +605,17 @@ ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
     ngx_quic_connection_t  *qc;
 
     qc = ngx_quic_get_connection(c);
+
+    if (qc->closing) {
+        return;
+    }
+
     qc->error = err;
     qc->error_reason = reason;
     qc->error_app = 1;
     qc->error_ftype = 0;
 
-    ngx_quic_close_connection(c, NGX_OK);
+    ngx_post_event(&qc->close, &ngx_posted_events);
 }
 
 
@@ -630,20 +637,13 @@ ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err,
 static void
 ngx_quic_close_handler(ngx_event_t *ev)
 {
-    ngx_connection_t       *c;
+    ngx_connection_t  *c;
-    ngx_quic_connection_t  *qc;
 
     ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "quic close handler");
 
     c = ev->data;
-    qc = ngx_quic_get_connection(c);
 
-    if (qc->closing) {
+    ngx_quic_close_connection(c, NGX_OK);
-        ngx_quic_close_connection(c, NGX_OK);
-
-    } else if (qc->shutdown) {
-        ngx_quic_shutdown_quic(c);
-    }
 }
 
 
@@ -1428,31 +1428,10 @@ ngx_quic_push_handler(ngx_event_t *ev)
 void
 ngx_quic_shutdown_quic(ngx_connection_t *c)
 {
-    ngx_rbtree_t           *tree;
-    ngx_rbtree_node_t      *node;
-    ngx_quic_stream_t      *qs;
     ngx_quic_connection_t  *qc;
 
-    qc = ngx_quic_get_connection(c);
+    if (c->reusable) {
-
+        qc = ngx_quic_get_connection(c);
-    if (qc->closing) {
+        ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason);
-        return;
     }
-
-    tree = &qc->streams.tree;
-
-    if (tree->root != tree->sentinel) {
-        for (node = ngx_rbtree_min(tree->root, tree->sentinel);
-             node;
-             node = ngx_rbtree_next(tree, node))
-        {
-            qs = (ngx_quic_stream_t *) node;
-
-            if (!qs->cancelable) {
-                return;
-            }
-        }
-    }
-
-    ngx_quic_finalize_connection(c, qc->shutdown_code, qc->shutdown_reason);
 }

src/event/quic/ngx_event_quic.h

@@ -113,6 +113,7 @@ void ngx_quic_shutdown_connection(ngx_connection_t *c, ngx_uint_t err,
     const char *reason);
 ngx_int_t ngx_quic_reset_stream(ngx_connection_t *c, ngx_uint_t err);
 ngx_int_t ngx_quic_shutdown_stream(ngx_connection_t *c, int how);
+void ngx_quic_cancelable_stream(ngx_connection_t *c);
 ngx_int_t ngx_quic_handle_read_event(ngx_event_t *rev, ngx_uint_t flags);
 ngx_int_t ngx_quic_handle_write_event(ngx_event_t *wev, size_t lowat);
 ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len,

src/event/quic/ngx_event_quic_streams.c

@@ -33,6 +33,7 @@ static ngx_chain_t *ngx_quic_stream_send_chain(ngx_connection_t *c,
 static ngx_int_t ngx_quic_stream_flush(ngx_quic_stream_t *qs);
 static void ngx_quic_stream_cleanup_handler(void *data);
 static ngx_int_t ngx_quic_close_stream(ngx_quic_stream_t *qs);
+static ngx_int_t ngx_quic_can_shutdown(ngx_connection_t *c);
 static ngx_int_t ngx_quic_control_flow(ngx_quic_stream_t *qs, uint64_t last);
 static ngx_int_t ngx_quic_update_flow(ngx_quic_stream_t *qs, uint64_t last);
 static ngx_int_t ngx_quic_update_max_stream_data(ngx_quic_stream_t *qs);
@@ -51,6 +52,10 @@ ngx_quic_open_stream(ngx_connection_t *c, ngx_uint_t bidi)
     pc = c->quic ? c->quic->parent : c;
     qc = ngx_quic_get_connection(pc);
 
+    if (qc->closing) {
+        return NULL;
+    }
+
     if (bidi) {
         if (qc->streams.server_streams_bidi
             >= qc->streams.server_max_streams_bidi)
@@ -161,13 +166,10 @@ ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc)
     ngx_pool_t         *pool;
     ngx_queue_t        *q;
     ngx_rbtree_t       *tree;
+    ngx_connection_t   *sc;
     ngx_rbtree_node_t  *node;
     ngx_quic_stream_t  *qs;
 
-#if (NGX_DEBUG)
-    ngx_uint_t          ns;
-#endif
-
     while (!ngx_queue_empty(&qc->streams.uninitialized)) {
         q = ngx_queue_head(&qc->streams.uninitialized);
         ngx_queue_remove(q);
@@ -185,34 +187,34 @@ ngx_quic_close_streams(ngx_connection_t *c, ngx_quic_connection_t *qc)
         return NGX_OK;
     }
 
-#if (NGX_DEBUG)
-    ns = 0;
-#endif
-
     node = ngx_rbtree_min(tree->root, tree->sentinel);
 
     while (node) {
         qs = (ngx_quic_stream_t *) node;
         node = ngx_rbtree_next(tree, node);
+        sc = qs->connection;
 
         qs->recv_state = NGX_QUIC_STREAM_RECV_RESET_RECVD;
         qs->send_state = NGX_QUIC_STREAM_SEND_RESET_SENT;
 
-        if (qs->connection == NULL) {
+        if (sc == NULL) {
             ngx_quic_close_stream(qs);
             continue;
         }
 
-        ngx_quic_set_event(qs->connection->read);
+        ngx_quic_set_event(sc->read);
-        ngx_quic_set_event(qs->connection->write);
+        ngx_quic_set_event(sc->write);
 
-#if (NGX_DEBUG)
+        sc->close = 1;
-        ns++;
+        sc->read->handler(sc->read);
-#endif
     }
 
-    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+    if (tree->root == tree->sentinel) {
-                   "quic connection has %ui active streams", ns);
+        return NGX_OK;
+    }
+
+    ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "quic connection has active streams");
 
     return NGX_AGAIN;
 }
@@ -587,6 +589,7 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id)
 {
     ngx_log_t              *log;
     ngx_pool_t             *pool;
+    ngx_uint_t              reusable;
     ngx_queue_t            *q;
     ngx_connection_t       *sc;
     ngx_quic_stream_t      *qs;
@@ -639,10 +642,14 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id)
     *log = *c->log;
     pool->log = log;
 
+    reusable = c->reusable;
+    ngx_reusable_connection(c, 0);
+
     sc = ngx_get_connection(c->fd, log);
     if (sc == NULL) {
         ngx_destroy_pool(pool);
         ngx_queue_insert_tail(&qc->streams.free, &qs->queue);
+        ngx_reusable_connection(c, reusable);
         return NULL;
     }
 
@@ -712,6 +719,7 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id)
         ngx_close_connection(sc);
         ngx_destroy_pool(pool);
         ngx_queue_insert_tail(&qc->streams.free, &qs->queue);
+        ngx_reusable_connection(c, reusable);
         return NULL;
     }
 
@@ -724,6 +732,31 @@ ngx_quic_create_stream(ngx_connection_t *c, uint64_t id)
 }
 
 
+void
+ngx_quic_cancelable_stream(ngx_connection_t *c)
+{
+    ngx_connection_t       *pc;
+    ngx_quic_stream_t      *qs;
+    ngx_quic_connection_t  *qc;
+
+    qs = c->quic;
+    pc = qs->parent;
+    qc = ngx_quic_get_connection(pc);
+
+    if (!qs->cancelable) {
+        qs->cancelable = 1;
+
+        if (ngx_quic_can_shutdown(pc) == NGX_OK) {
+            ngx_reusable_connection(pc, 1);
+
+            if (qc->shutdown) {
+                ngx_quic_shutdown_quic(pc);
+            }
+        }
+    }
+}
+
+
 static void
 ngx_quic_empty_handler(ngx_event_t *ev)
 {
@@ -1056,10 +1089,43 @@ ngx_quic_close_stream(ngx_quic_stream_t *qs)
         ngx_quic_queue_frame(qc, frame);
     }
 
-    if (qc->shutdown) {
+    if (!pc->reusable && ngx_quic_can_shutdown(pc) == NGX_OK) {
-        ngx_post_event(&qc->close, &ngx_posted_events);
+        ngx_reusable_connection(pc, 1);
     }
 
+    if (qc->shutdown) {
+        ngx_quic_shutdown_quic(pc);
+    }
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_quic_can_shutdown(ngx_connection_t *c)
+{
+    ngx_rbtree_t           *tree;
+    ngx_rbtree_node_t      *node;
+    ngx_quic_stream_t      *qs;
+    ngx_quic_connection_t  *qc;
+
+    qc = ngx_quic_get_connection(c);
+
+    tree = &qc->streams.tree;
+
+    if (tree->root != tree->sentinel) {
+        for (node = ngx_rbtree_min(tree->root, tree->sentinel);
+             node;
+             node = ngx_rbtree_next(tree, node))
+        {
+            qs = (ngx_quic_stream_t *) node;
+
+            if (!qs->cancelable) {
+                return NGX_DECLINED;
+            }
+        }
+     }
+
     return NGX_OK;
 }
 

src/http/v3/ngx_http_v3_uni.c

@@ -52,7 +52,7 @@ ngx_http_v3_init_uni_stream(ngx_connection_t *c)
         return;
     }
 
-    c->quic->cancelable = 1;
+    ngx_quic_cancelable_stream(c);
 
     us = ngx_pcalloc(c->pool, sizeof(ngx_http_v3_uni_stream_t));
     if (us == NULL) {
@@ -182,6 +182,11 @@ ngx_http_v3_uni_read_handler(ngx_event_t *rev)
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 read handler");
 
+    if (c->close) {
+        ngx_http_v3_close_uni_stream(c);
+        return;
+    }
+
     ngx_memzero(&b, sizeof(ngx_buf_t));
 
     while (rev->ready) {
@@ -262,6 +267,11 @@ ngx_http_v3_uni_dummy_read_handler(ngx_event_t *rev)
 
     ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http3 dummy read handler");
 
+    if (c->close) {
+        ngx_http_v3_close_uni_stream(c);
+        return;
+    }
+
     if (rev->ready) {
         if (c->recv(c, &ch, 1) != 0) {
             ngx_http_v3_finalize_connection(c, NGX_HTTP_V3_ERR_NO_ERROR, NULL);
@@ -404,7 +414,7 @@ ngx_http_v3_get_uni_stream(ngx_connection_t *c, ngx_uint_t type)
         goto failed;
     }
 
-    sc->quic->cancelable = 1;
+    ngx_quic_cancelable_stream(sc);
 
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
                    "http3 create uni stream, type:%ui", type);