bugfix: improve HTTP/3 SSL Lua callback yield handling.

Author: swananan

.travis.yml

@@ -137,7 +137,7 @@ script:
   - cd lua-cjson/ && make -j$JOBS && sudo make install && cd ..
   #- if [ -n "$PCRE2_VER" ]; then tar zxf download-cache/pcre2-$PCRE2_VER.tar.gz; cd pcre2-$PCRE2_VER/; ./configure --prefix=$PCRE2_PREFIX --enable-jit --enable-utf > build.log 2>&1 || (cat build.log && exit 1); make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1); sudo PATH=$PATH make install > build.log 2>&1 || (cat build.log && exit 1); cd ..; fi
   #- if [ -n "$OPENSSL_VER" ]; then tar zxf download-cache/openssl-$OPENSSL_VER.tar.gz; cd openssl-$OPENSSL_VER/; patch -p1 < ../../openresty/patches/openssl-$OPENSSL_PATCH_VER-sess_set_get_cb_yield.patch; ./config shared enable-ssl3 enable-ssl3-method -g --prefix=$OPENSSL_PREFIX --libdir=lib -DPURIFY > build.log 2>&1 || (cat build.log && exit 1); make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1); sudo make PATH=$PATH install_sw > build.log 2>&1 || (cat build.log && exit 1); cd ..; fi
-  - if [ -n "$BORINGSSL" ]; then sudo rm -fr /usr/local/openresty/openssl3/ && sudo tar -C /usr/local/openresty/openssl3 -xf boringssl-20230902-x64-focal.tar.gz --strip-components=1; fi
+  - if [ -n "$BORINGSSL" ]; then sudo rm -fr /usr/local/openresty/openssl3/ && sudo mkdir -p /usr/local/openresty/openssl3 && sudo tar -C /usr/local/openresty/openssl3 -xf boringssl-20230902-x64-focal.tar.gz --strip-components=1; fi
   - export NGX_BUILD_CC=$CC
   - sh util/build-without-ssl.sh $NGINX_VERSION > build.log 2>&1 || (cat build.log && exit 1)
   - sh util/build-with-dd.sh $NGINX_VERSION > build.log 2>&1 || (cat build.log && exit 1)

src/ngx_http_lua_ssl_certby.c

@@ -346,7 +346,6 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data)
 
     return -1;
 
-#if 1
 failed:
 
     if (r && r->pool) {
@@ -358,15 +357,14 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data)
     }
 
     return 0;
-#endif
 }
 
 
 static void
 ngx_http_lua_ssl_cert_done(void *data)
 {
-    ngx_connection_t                *c;
-    ngx_http_lua_ssl_ctx_t          *cctx = data;
+    ngx_connection_t        *c;
+    ngx_http_lua_ssl_ctx_t  *cctx = data;
 
     dd("lua ssl cert done");
 
@@ -387,6 +385,12 @@ ngx_http_lua_ssl_cert_done(void *data)
     c->log->action = "SSL handshaking";
 
     ngx_post_event(c->write, &ngx_posted_events);
+
+#if (NGX_HTTP_V3) && OPENSSL_VERSION_NUMBER >= 0x1000205fL
+#   if (NGX_QUIC_OPENSSL_COMPAT)
+    ngx_http_lua_resume_quic_ssl_handshake(c);
+#   endif
+#endif
 }
 
 

src/ngx_http_lua_ssl_client_helloby.c

@@ -341,7 +341,6 @@ ngx_http_lua_ssl_client_hello_handler(ngx_ssl_conn_t *ssl_conn,
 
     return -1;
 
-#if 1
 failed:
 
     if (r && r->pool) {
@@ -353,15 +352,14 @@ ngx_http_lua_ssl_client_hello_handler(ngx_ssl_conn_t *ssl_conn,
     }
 
     return 0;
-#endif
 }
 
 
 static void
 ngx_http_lua_ssl_client_hello_done(void *data)
 {
-    ngx_connection_t                *c;
-    ngx_http_lua_ssl_ctx_t          *cctx = data;
+    ngx_connection_t        *c;
+    ngx_http_lua_ssl_ctx_t  *cctx = data;
 
     dd("lua ssl client hello done");
 
@@ -382,6 +380,12 @@ ngx_http_lua_ssl_client_hello_done(void *data)
     c->log->action = "SSL handshaking";
 
     ngx_post_event(c->write, &ngx_posted_events);
+
+#if (NGX_HTTP_V3) && defined(SSL_ERROR_WANT_CLIENT_HELLO_CB)
+#   if (NGX_QUIC_OPENSSL_COMPAT)
+    ngx_http_lua_resume_quic_ssl_handshake(c);
+#   endif
+#endif
 }
 
 
@@ -666,6 +670,7 @@ int
 ngx_http_lua_ffi_ssl_get_client_hello_ext_present(ngx_http_request_t *r,
     int **extensions, size_t *extensions_len, char **err)
 {
+#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
     ngx_ssl_conn_t          *ssl_conn;
     int                      got_extensions;
     size_t                   ext_len;
@@ -684,7 +689,6 @@ ngx_http_lua_ffi_ssl_get_client_hello_ext_present(ngx_http_request_t *r,
         return NGX_ERROR;
     }
 
-#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
     got_extensions = SSL_client_hello_get1_extensions_present(ssl_conn,
                                                            &ext_out, &ext_len);
     if (!got_extensions || !ext_out || !ext_len) {
@@ -710,6 +714,7 @@ ngx_http_lua_ffi_ssl_get_client_hello_ext_present(ngx_http_request_t *r,
 int ngx_http_lua_ffi_ssl_get_client_hello_ciphers(ngx_http_request_t *r,
     unsigned short *ciphers,  size_t ciphers_size, char **err)
 {
+#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
     int                      i;
     size_t                   ciphers_cnt;
     size_t                   ciphersuites_bytes;
@@ -727,8 +732,6 @@ int ngx_http_lua_ffi_ssl_get_client_hello_ciphers(ngx_http_request_t *r,
         return NGX_ERROR;
     }
 
-
-#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
     ciphersuites_bytes = SSL_client_hello_get0_ciphers(ssl_conn, &ciphers_raw);
 
     if (ciphersuites_bytes == 0) {

src/ngx_http_lua_util.c

@@ -49,6 +49,10 @@
 #if (NGX_THREADS)
 #include "ngx_http_lua_worker_thread.h"
 #endif
+#if (NGX_HTTP_V3)
+#include <ngx_event_quic.h>
+#include <ngx_event_quic_connection.h>
+#endif
 
 
 #if 1
@@ -3826,6 +3830,18 @@ ngx_http_lua_close_fake_connection(ngx_connection_t *c)
     c->read->closed = 1;
     c->write->closed = 1;
 
+    /* When destroying the pool, the registered clean callbacks will be
+     * executed. If the ngx_connection_t is freed before these callbacks are
+     * run, and a new ngx_connection_t is created within a clean callback,
+     * it is possible for the freed ngx_connection_t to be reused again.
+     * If this reused ngx_connection_t is destroyed again within the clean
+     * callback logic, it may result in other clean callbacks holding a
+     * ngx_connection_t that has already been destroyed.
+     */
+    if (pool) {
+        ngx_destroy_pool(pool);
+    }
+
     /* we temporarily use a valid fd (0) to make ngx_free_connection happy */
 
     c->fd = 0;
@@ -3841,10 +3857,6 @@ ngx_http_lua_close_fake_connection(ngx_connection_t *c)
     if (ngx_cycle->files) {
         ngx_cycle->files[0] = saved_c;
     }
-
-    if (pool) {
-        ngx_destroy_pool(pool);
-    }
 }
 
 
@@ -4561,4 +4573,63 @@ ngx_http_lua_ffi_bypass_if_checks(ngx_http_request_t *r)
     r->disable_not_modified = 1;
 }
 
+
+#if (NGX_HTTP_V3)
+void
+ngx_http_lua_resume_quic_ssl_handshake(ngx_connection_t *c)
+{
+    ngx_int_t        rc, sslerr;
+    ngx_ssl_conn_t  *ssl_conn;
+
+    if (c == NULL || c->ssl == NULL || c->ssl->connection == NULL) {
+        return;
+    }
+
+    if (!c->udp || c->read == NULL
+        || c->read->handler != ngx_quic_input_handler)
+    {
+        return;
+    }
+
+    ssl_conn = c->ssl->connection;
+
+    /* Openresty ssl lua scripts are triggered by registering callbacks into
+     * openssl. If a lua script calls a yield api during execution, openssl's
+     * SSL_do_handshake will return a specific error code. The application
+     * should not treat these codes as fatal. The lua script will resume on
+     * yield-related events until it finishes. After completion,
+     * SSL_do_handshake should be called again to advance openssl's state
+     * machine.
+     *
+     * Note that nginx quic and openresty ssl lua scripts are independent. When
+     * openresty ssl lua runs, the client is waiting for a server response, so
+     * nginx quic does not affect the execution of the lua script. After the lua
+     * script finishes, SSL_do_handshake is called again, and nginx quic's
+     * registered callback continues the handshake.
+     */
+    rc = SSL_do_handshake(ssl_conn);
+    sslerr = SSL_get_error(ssl_conn, rc);
+
+    if (rc <= 0 && sslerr != SSL_ERROR_WANT_READ
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+        && sslerr != SSL_ERROR_WANT_X509_LOOKUP
+#endif
+#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
+        && sslerr != SSL_ERROR_WANT_CLIENT_HELLO_CB
+#endif
+    ) {
+        /* If a fatal error occurs or lua script exits with error during quic
+         * handshake, the quic connection will be closed immediately.
+         */
+        ngx_quic_close_connection(c, NGX_ERROR);
+
+    } else {
+        ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
+                       "lua resuming quic ssl handshake: rc %d, err %d, will "
+                       "continue driving handshake or next lua script phase",
+                       rc, sslerr);
+    }
+}
+#endif
+
 /* vi:set ft=c ts=4 sw=4 et fdm=marker: */

src/ngx_http_lua_util.h

@@ -267,6 +267,10 @@ ngx_addr_t *ngx_http_lua_parse_addr(lua_State *L, u_char *text, size_t len);
 
 size_t ngx_http_lua_escape_log(u_char *dst, u_char *src, size_t size);
 
+#if (NGX_HTTP_V3)
+void ngx_http_lua_resume_quic_ssl_handshake(ngx_connection_t *c);
+#endif
+
 
 static ngx_inline void
 ngx_http_lua_init_ctx(ngx_http_request_t *r, ngx_http_lua_ctx_t *ctx)