diff --git a/configure.ac b/configure.ac index 537a51ff99c..2430a317442 100644 --- a/configure.ac +++ b/configure.ac @@ -2225,6 +2225,19 @@ then AM_CFLAGS="$AM_CFLAGS -DSHOW_SECRETS -DHAVE_SECRET_CALLBACK -DWOLFSSL_SSLKEYLOGFILE -DWOLFSSL_KEYLOG_EXPORT_WARNED" fi +# TLS receive read-ahead: when enabled at runtime via wolfSSL_set_read_ahead(), +# the record-header read pulls in up to a full record in one recv(), avoiding a +# second syscall for the body. Opt-in (default: disabled). +AC_ARG_ENABLE([readahead], + [AS_HELP_STRING([--enable-readahead],[Enable TLS receive read-ahead support, activated at runtime with wolfSSL_set_read_ahead() (default: disabled)])], + [ ENABLED_READAHEAD=$enableval ], + [ ENABLED_READAHEAD=no ] + ) +if test "$ENABLED_READAHEAD" = "yes" +then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_TLS_READ_AHEAD" +fi + # TLS v1.3 Draft 18 (Note: only final TLS v1.3 supported, here for backwards build compatibility) AC_ARG_ENABLE([tls13-draft18], [AS_HELP_STRING([--enable-tls13-draft18],[Enable wolfSSL TLS v1.3 Draft 18 (default: disabled)])], @@ -13303,6 +13316,7 @@ echo " * Dual alg cert support: $ENABLED_DUAL_ALG_CERTS" echo " * ERR Queues per Thread: $ENABLED_ERRORQUEUEPERTHREAD" echo " * rwlock: $ENABLED_RWLOCK" echo " * keylog export: $ENABLED_KEYLOG_EXPORT" +echo " * TLS receive read-ahead: $ENABLED_READAHEAD" echo " * AutoSAR : $ENABLED_AUTOSAR" echo " * ML-KEM standalone: $ENABLED_MLKEM_STANDALONE" echo " * PQ/T hybrids: $ENABLED_PQC_HYBRIDS" diff --git a/doc/dox_comments/header_files/ssl.h b/doc/dox_comments/header_files/ssl.h index 59f44c3a54f..cd3c3c88629 100644 --- a/doc/dox_comments/header_files/ssl.h +++ b/doc/dox_comments/header_files/ssl.h @@ -5486,6 +5486,21 @@ int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); \ingroup Setup \brief This function sets the read ahead flag in the WOLFSSL_CTX structure. + When enabled, the record-header read pulls in up to a full TLS record in a + single recv(), so the body (and any following buffered records) is obtained + without a second syscall. The flag only changes I/O behaviour when the + library is built with read-ahead support (--enable-readahead / + WOLFSSL_TLS_READ_AHEAD); otherwise it is stored but inert. + + \note With read-ahead enabled, undecrypted data can remain buffered + internally while the socket has no more data to read. wolfSSL_has_pending() + reports whether any such data is buffered, but a non-zero return does not + guarantee a full record is available: wolfSSL_read() may still return + WANT_READ when only a partial record is buffered. Event-driven applications + should therefore drain the connection by calling wolfSSL_read() until it + returns WANT_READ (or an error) before returning to select()/poll(), rather + than looping on wolfSSL_has_pending() alone; otherwise buffered data could + be missed or the loop could spin. \return SSL_SUCCESS If ctx read ahead flag set. \return SSL_FAILURE If ctx is NULL then SSL_FAILURE is returned. @@ -5506,9 +5521,88 @@ int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); \sa wolfSSL_CTX_new \sa wolfSSL_CTX_free \sa wolfSSL_CTX_get_read_ahead + \sa wolfSSL_has_pending */ int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v); +/*! + \ingroup Setup + + \brief This function sets the read-ahead receive window size for contexts + created from this WOLFSSL_CTX. When read-ahead is enabled + (wolfSSL_CTX_set_read_ahead()), a single recv() pulls in up to \p len bytes: + - \p len of 0 resets the window to the one-record default. This is also + the window a freshly created context already carries, so the record body + is read together with its header without a second syscall. + - A \p len larger than one record lets a single recv() coalesce several + back-to-back records, one syscall instead of one per record. + - A \p len smaller than one record caps the receive buffer's footprint + when the peer's records are known to be small (e.g. 4 KB), saving heap + versus the one-record default. + \p len is only a speculative read window, not a hard limit: a record larger + than \p len is still received correctly, with the input buffer grown to the + record's actual size on demand (costing an extra syscall and reallocation + for that record) and then reallocated back down to the window once the + oversized record is consumed, so the retained footprint stays bounded by + \p len rather than by the largest record seen. The setting only affects I/O + when the library is built with read-ahead support (--enable-readahead / + WOLFSSL_TLS_READ_AHEAD); otherwise it is stored but inert. This is the + wolfSSL equivalent of OpenSSL's SSL_CTX_set_default_read_buffer_len(); unlike + OpenSSL it returns a status code (callers that ignore the return remain + source-compatible) and it honours sizes below one record, whereas OpenSSL + only ever enlarges the buffer. A \p len above WOLFSSL_MAX_READ_AHEAD_SZ + (16 MB) is clamped to that maximum. Because 0 and oversized values are both + normalised, wolfSSL_CTX_get_default_read_buffer_len() reports the effective + window (the one-record default rather than 0 when unset), not the raw + argument. + + \note This is a memory-vs-syscall trade-off. While read-ahead is enabled the + input buffer is retained (bounded to \p len) for the connection's lifetime, + so a large \p len multiplied by many concurrent connections is persistent + memory, while a small \p len bounds per-connection footprint at the cost of + more syscalls for records that exceed it. + + \return SSL_SUCCESS If the buffer length was set. + \return SSL_FAILURE If ctx is NULL. + + \param ctx WOLFSSL_CTX structure to set the read-ahead buffer length on. + \param len read-ahead coalescing buffer size in bytes (0 = one record). + + _Example_ + \code + WOLFSSL_CTX* ctx; + // setup ctx + wolfSSL_CTX_set_read_ahead(ctx, 1); + // coalesce up to four max-size records per recv() + wolfSSL_CTX_set_default_read_buffer_len(ctx, 4 * 16384); + \endcode + + \sa wolfSSL_CTX_set_read_ahead + \sa wolfSSL_set_default_read_buffer_len + \sa wolfSSL_has_pending +*/ +int wolfSSL_CTX_set_default_read_buffer_len(WOLFSSL_CTX* ctx, size_t len); + +/*! + \ingroup Setup + + \brief This function sets the read-ahead coalescing buffer size on a single + WOLFSSL session, overriding the value inherited from its WOLFSSL_CTX. See + wolfSSL_CTX_set_default_read_buffer_len() for the full description, the + one-record default, and the memory-vs-syscall trade-off. + + \return SSL_SUCCESS If the buffer length was set. + \return SSL_FAILURE If ssl is NULL. + + \param ssl WOLFSSL structure to set the read-ahead buffer length on. + \param len read-ahead coalescing buffer size in bytes (0 = one record). + + \sa wolfSSL_CTX_set_default_read_buffer_len + \sa wolfSSL_set_read_ahead + \sa wolfSSL_has_pending +*/ +int wolfSSL_set_default_read_buffer_len(WOLFSSL* ssl, size_t len); + /*! \ingroup Setup diff --git a/examples/benchmark/tls_bench.c b/examples/benchmark/tls_bench.c index a3f3675730e..72a6382c0aa 100644 --- a/examples/benchmark/tls_bench.c +++ b/examples/benchmark/tls_bench.c @@ -1100,6 +1100,13 @@ static int bench_tls_client(info_t* info) #endif wolfSSL_SetIOReadCtx(cli_ssl, info); wolfSSL_SetIOWriteCtx(cli_ssl, info); +#if defined(WOLFSSL_TLS_READ_AHEAD) && !defined(NO_FILESYSTEM) && \ + !defined(NO_STDIO_FILESYSTEM) + /* Optional A/B benchmark toggle for TLS receive read-ahead. Gated on + * filesystem support since XGETENV is only defined there. */ + if (XGETENV("WOLF_BENCH_READ_AHEAD") != NULL) + wolfSSL_set_read_ahead(cli_ssl, 1); +#endif #if !defined(SINGLE_THREADED) && defined(WOLFSSL_DTLS) /* synchronize with server */ @@ -1557,6 +1564,13 @@ static int bench_tls_server(info_t* info) wolfSSL_SetIOReadCtx(srv_ssl, info); wolfSSL_SetIOWriteCtx(srv_ssl, info); +#if defined(WOLFSSL_TLS_READ_AHEAD) && !defined(NO_FILESYSTEM) && \ + !defined(NO_STDIO_FILESYSTEM) + /* Optional A/B benchmark toggle for TLS receive read-ahead. Gated on + * filesystem support since XGETENV is only defined there. */ + if (XGETENV("WOLF_BENCH_READ_AHEAD") != NULL) + wolfSSL_set_read_ahead(srv_ssl, 1); +#endif #ifndef NO_DH wolfSSL_SetTmpDH(srv_ssl, dhp, sizeof(dhp), dhg, sizeof(dhg)); #endif diff --git a/examples/client/client.c b/examples/client/client.c index 64958f2091e..338c7c9234d 100644 --- a/examples/client/client.c +++ b/examples/client/client.c @@ -790,6 +790,15 @@ static int ClientBenchmarkThroughput(WOLFSSL_CTX* ctx, char* host, word16 port, if (ssl == NULL) err_sys("unable to get SSL object"); +#if defined(WOLFSSL_TLS_READ_AHEAD) && !defined(NO_FILESYSTEM) && \ + !defined(NO_STDIO_FILESYSTEM) + /* Optional A/B toggle: enable TLS receive read-ahead for the throughput + * benchmark when WOLF_BENCH_READ_AHEAD is set in the environment. Gated on + * filesystem support since XGETENV is only defined there. */ + if (XGETENV("WOLF_BENCH_READ_AHEAD") != NULL) + wolfSSL_set_read_ahead(ssl, 1); +#endif + tcp_connect(&sockfd, host, port, dtlsUDP, dtlsSCTP, ssl); if (wolfSSL_set_fd(ssl, sockfd) != WOLFSSL_SUCCESS) { err_sys("error in setting fd"); diff --git a/src/internal.c b/src/internal.c index 0ea1b04f2b7..b13a97529b4 100644 --- a/src/internal.c +++ b/src/internal.c @@ -2650,6 +2650,15 @@ int InitSSL_Ctx(WOLFSSL_CTX* ctx, WOLFSSL_METHOD* method, void* heap) } ctx->timeout = WOLFSSL_SESSION_TIMEOUT; +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) + /* Default the read-ahead window to one full record. Contexts (and the + * WOLFSSL objects that inherit it) then always carry a concrete window, so + * the receive path uses ssl->readAheadSz directly without a per-read + * fallback. A caller override replaces it; passing 0 resets it to this + * default (see wolfSSL_CTX_set_default_read_buffer_len()). */ + ctx->readAheadSz = WOLFSSL_READ_AHEAD_SZ; +#endif + #ifdef WOLFSSL_DTLS if (method->version.major == DTLS_MAJOR) { ctx->minDowngrade = WOLFSSL_MIN_DTLS_DOWNGRADE; @@ -7509,8 +7518,9 @@ int SetSSL_CTX(WOLFSSL* ssl, WOLFSSL_CTX* ctx, int writeDup) ssl->ConnectFilter_arg = ctx->ConnectFilter_arg; #endif -#ifdef OPENSSL_EXTRA +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) ssl->readAhead = ctx->readAhead; + ssl->readAheadSz = ctx->readAheadSz; #endif #if defined(OPENSSL_EXTRA) && !defined(NO_BIO) /* Don't change recv callback if currently using BIO's */ @@ -11619,6 +11629,41 @@ void ShrinkInputBuffer(WOLFSSL* ssl, int forcedFree) ssl->buffers.clearOutputBuffer.length > 0)) return; +#ifdef WOLFSSL_TLS_READ_AHEAD + /* While read-ahead is enabled, retain a dynamic input buffer sized to the + * configured window rather than shrinking all the way back to the static + * buffer, so the speculative over-read is a bounded, mostly one-time + * allocation instead of per-record churn. A forced free (connection + * teardown) still reclaims everything. */ + if (!forcedFree && ssl->readAhead) { + /* Already within the window: keep the buffer as-is. */ + if (ssl->buffers.inputBuffer.bufferSize <= ssl->readAheadSz) + return; + + /* The buffer grew past the window to receive an oversized record. When + * the window needs a dynamic buffer, reallocate down to it so the + * retained footprint tracks the window, not the largest record seen; + * when the window fits in the static buffer, fall through and shrink to + * static below. + * + * GrowInputBuffer(newBytes, usedLength) resizes the input buffer to + * newBytes + usedLength while preserving the usedLength bytes still + * pending, so requesting (readAheadSz - usedLength) new bytes yields a + * buffer of exactly readAheadSz. usedLength is <= STATIC_BUFFER_LEN < + * readAheadSz here (guaranteed above), so the subtraction stays + * positive. */ + if (ssl->readAheadSz > STATIC_BUFFER_LEN) { + if (GrowInputBuffer(ssl, (int)ssl->readAheadSz - usedLength, + usedLength) != 0) { + /* Realloc failed: keep the current (larger) buffer rather than + * dropping the buffered data. */ + WOLFSSL_MSG("read-ahead buffer shrink failed, keeping buffer"); + } + return; + } + } +#endif + WOLFSSL_MSG("Shrinking input buffer"); if (!forcedFree && usedLength > 0) { @@ -23189,12 +23234,16 @@ static int DoAlert(WOLFSSL* ssl, byte* input, word32* inOutIdx, int* type) return level; } -static int GetInputData(WOLFSSL *ssl, word32 size) +static int GetInputData_ex(WOLFSSL *ssl, word32 size, word32 readAhead) { int inSz; int maxLength; int usedLength; int dtlsExtra = 0; + int extra = 0; +#ifndef WOLFSSL_TLS_READ_AHEAD + (void)readAhead; +#endif if (ssl->options.disableRead) return WC_NO_ERR_TRACE(WANT_READ); @@ -23231,10 +23280,27 @@ static int GetInputData(WOLFSSL *ssl, word32 size) } inSz = (int)(size - (word32)usedLength); /* from last partial read */ + +#ifdef WOLFSSL_TLS_READ_AHEAD + /* Request more than the minimum so that a single recv() can also pull + * in the record body (and possibly following records), avoiding a + * second syscall. 'size' remains the loop-termination minimum, so a + * blocking socket never waits for read-ahead bytes the peer may not + * send. Mirrors the DTLS dtlsExtra over-read above. + * + * The buffer is grown once to hold a full record and then kept (see the + * ssl->readAhead guard in ShrinkInputBuffer), so this is a one-time + * allocation per connection, not per-record churn. */ + if (readAhead > size) { + extra = (int)(readAhead - size); + inSz += extra; + } +#endif } if (inSz > maxLength) { - if (GrowInputBuffer(ssl, (int)(size + (word32)dtlsExtra), usedLength) < 0) + if (GrowInputBuffer(ssl, + (int)(size + (word32)dtlsExtra + (word32)extra), usedLength) < 0) return MEMORY_E; } @@ -23292,6 +23358,11 @@ static int GetInputData(WOLFSSL *ssl, word32 size) return 0; } +static int GetInputData(WOLFSSL *ssl, word32 size) +{ + return GetInputData_ex(ssl, size, 0); +} + #if defined(HAVE_ENCRYPT_THEN_MAC) && !defined(WOLFSSL_AEAD_ONLY) static WC_INLINE int VerifyMacEnc(WOLFSSL* ssl, const byte* input, word32 msgSz, int content) @@ -24123,7 +24194,28 @@ static int DoProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) /* get header or return error */ if (!ssl->options.dtls) { - if ((ret = GetInputData(ssl, (word32)readSz)) < 0) + word32 readAheadSz = 0; + #ifdef WOLFSSL_TLS_READ_AHEAD + /* When read-ahead is enabled, request more than the record + * header in a single recv() so the body (and possibly following + * records) can be pulled in without a second syscall. The window + * size is configurable via + * wolfSSL_CTX_set_default_read_buffer_len(): + * - 0 (unset) requests one full record, the sensible default. + * - A larger value lets one recv() coalesce several back-to-back + * records. + * - A smaller (sub-record) value caps the receive buffer's + * footprint when the peer's records are known to be small. + * The value is only a speculative read window: a record larger + * than it is still received correctly, as the buffer is grown to + * the record's actual size on demand. */ + if (ssl->readAhead) { + /* readAheadSz is always concrete (defaulted to + * WOLFSSL_READ_AHEAD_SZ at CTX init, never 0). */ + readAheadSz = ssl->readAheadSz; + } + #endif + if ((ret = GetInputData_ex(ssl, (word32)readSz, readAheadSz)) < 0) return ret; } else { #ifdef WOLFSSL_DTLS @@ -24848,6 +24940,16 @@ static int DoProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) ssl->options.serverState == SERVER_FINISHED_COMPLETE && ssl->options.handShakeState != HANDSHAKE_DONE))) +#endif +#ifdef WOLFSSL_TLS_READ_AHEAD + /* With read-ahead, more than one record may be buffered. If + * application data was just decrypted, return it now so it + * is delivered to the caller before any following buffered + * record (e.g. a close_notify alert) is processed, which + * would otherwise discard the pending app data. The + * remaining records stay buffered for the next call. */ + || (ssl->curRL.type == application_data && + ssl->buffers.clearOutputBuffer.length > 0) #endif ) { /* Shrink input buffer when we successfully finish record diff --git a/src/ssl.c b/src/ssl.c index f5a309552c1..cb76547e4ab 100644 --- a/src/ssl.c +++ b/src/ssl.c @@ -2829,7 +2829,21 @@ int wolfSSL_has_pending(const WOLFSSL* ssl) if (ssl == NULL) return WOLFSSL_FAILURE; - return ssl->buffers.clearOutputBuffer.length > 0; + if (ssl->buffers.clearOutputBuffer.length > 0) + return 1; + +#ifdef WOLFSSL_TLS_READ_AHEAD + /* Read-ahead can leave undecrypted data buffered while the socket itself + * has no more data. This may be a complete record or only a partial one + * (e.g. a coalesced read that pulled a record plus the head of the next), + * so a non-zero return does not guarantee wolfSSL_read() will yield + * application data without another socket read. Report it so a + * select()/poll() loop keeps draining until wolfSSL_read() reports + * WANT_READ, instead of stalling on buffered data. */ + if (ssl->buffers.inputBuffer.length > ssl->buffers.inputBuffer.idx) + return 1; +#endif + return 0; } #ifndef WOLFSSL_LEANPSK @@ -10847,6 +10861,9 @@ long wolfSSL_CTX_sess_timeouts(WOLFSSL_CTX* ctx) } #endif +#endif /* OPENSSL_EXTRA */ + +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) int wolfSSL_get_read_ahead(const WOLFSSL* ssl) { if (ssl == NULL) { @@ -10890,7 +10907,80 @@ int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v) return WOLFSSL_SUCCESS; } +/* Set the read-ahead receive window size. When read-ahead is enabled, a single + * recv() pulls in up to this many bytes: + * - 0 (the default) requests one full record. + * - A value larger than one record lets a single recv() coalesce several + * back-to-back records. + * - A value smaller than one record is honoured as-is, capping the receive + * buffer's footprint when the peer's records are known to be small. + * 'len' is only a speculative window: a record larger than it is still received + * correctly, with the buffer grown on demand. Effective only when the library + * is built with read-ahead support (WOLFSSL_TLS_READ_AHEAD). Returns int rather + * than OpenSSL's void to match wolfSSL's other read-ahead setters; callers that + * ignore the return remain source-compatible. */ +int wolfSSL_CTX_set_default_read_buffer_len(WOLFSSL_CTX* ctx, size_t len) +{ + if (ctx == NULL) { + return WOLFSSL_FAILURE; + } + + /* 0 selects the default one-record window (matches OpenSSL, where 0 means + * "use the built-in default"). Otherwise clamp to a sane maximum so the + * stored window cannot overflow the signed arithmetic in the receive path + * (see WOLFSSL_MAX_READ_AHEAD_SZ). */ + if (len == 0) { + len = WOLFSSL_READ_AHEAD_SZ; + } + else if (len > (size_t)WOLFSSL_MAX_READ_AHEAD_SZ) { + len = (size_t)WOLFSSL_MAX_READ_AHEAD_SZ; + } + ctx->readAheadSz = (word32)len; + + return WOLFSSL_SUCCESS; +} + +int wolfSSL_set_default_read_buffer_len(WOLFSSL* ssl, size_t len) +{ + if (ssl == NULL) { + return WOLFSSL_FAILURE; + } + /* 0 selects the default one-record window (matches OpenSSL, where 0 means + * "use the built-in default"). Otherwise clamp to a sane maximum so the + * stored window cannot overflow the signed arithmetic in the receive path + * (see WOLFSSL_MAX_READ_AHEAD_SZ). */ + if (len == 0) { + len = WOLFSSL_READ_AHEAD_SZ; + } + else if (len > (size_t)WOLFSSL_MAX_READ_AHEAD_SZ) { + len = (size_t)WOLFSSL_MAX_READ_AHEAD_SZ; + } + ssl->readAheadSz = (word32)len; + + return WOLFSSL_SUCCESS; +} + +long wolfSSL_CTX_get_default_read_buffer_len(WOLFSSL_CTX* ctx) +{ + if (ctx == NULL) { + return WOLFSSL_FAILURE; + } + + return (long)ctx->readAheadSz; +} + +long wolfSSL_get_default_read_buffer_len(const WOLFSSL* ssl) +{ + if (ssl == NULL) { + return WOLFSSL_FAILURE; + } + + return (long)ssl->readAheadSz; +} +#endif /* OPENSSL_EXTRA || WOLFSSL_TLS_READ_AHEAD */ + +#ifdef OPENSSL_EXTRA long wolfSSL_CTX_set_tlsext_opaque_prf_input_callback_arg(WOLFSSL_CTX* ctx, void* arg) { diff --git a/tests/api.c b/tests/api.c index 238ea673976..a3d705df6d2 100644 --- a/tests/api.c +++ b/tests/api.c @@ -35695,6 +35695,354 @@ static int test_wolfSSL_shutdown_pending_data_uaf(void) return EXPECT_RESULT(); } +#if defined(WOLFSSL_TLS_READ_AHEAD) && \ + defined(HAVE_MANUAL_MEMIO_TESTS_DEPENDENCIES) && \ + !defined(WOLFSSL_NO_TLS12) && !defined(NO_RSA) +/* Per-test recv context: counts callbacks and forwards to the memio reader. + * 'coalesce' makes one callback return as many queued records as fit, to + * emulate TCP coalescing several TLS records into a single recv(). */ +struct test_read_ahead_ctx { + struct test_memio_ctx* memio; + int count; + int coalesce; +}; + +static int test_read_ahead_recv_cb(WOLFSSL *ssl, char *buf, int sz, void *ctx) +{ + struct test_read_ahead_ctx* rc = (struct test_read_ahead_ctx*)ctx; + int total = 0; + int ret; + + rc->count++; + do { + ret = test_memio_read_cb(ssl, buf + total, sz - total, rc->memio); + if (ret <= 0) + break; + total += ret; + } while (rc->coalesce && total < sz); + + if (total > 0) + return total; + return ret; +} + +/* Read one application data record on the client, returning the number of + * recv() callback invocations it took. */ +static int test_wolfSSL_read_ahead_one(int enableReadAhead, int *recvCalls) +{ + EXPECT_DECLS; + WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL; + WOLFSSL *ssl_c = NULL, *ssl_s = NULL; + struct test_memio_ctx test_ctx; + struct test_read_ahead_ctx recv_ctx; + /* Use a multi-KB record so the test exercises a realistically sized record + * (larger than a minimal read-ahead window) rather than a tiny one. */ + byte msg[4000]; + byte reply[4000]; + + XMEMSET(msg, 'A', sizeof(msg)); + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&recv_ctx, 0, sizeof(recv_ctx)); + recv_ctx.memio = &test_ctx; + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_2_client_method, wolfTLSv1_2_server_method), 0); + + if (enableReadAhead) + ExpectIntEQ(wolfSSL_set_read_ahead(ssl_c, 1), WOLFSSL_SUCCESS); + + ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 10, NULL), 0); + + /* Server sends a single application data record. */ + ExpectIntEQ(wolfSSL_write(ssl_s, msg, (int)sizeof(msg)), (int)sizeof(msg)); + + /* Count the recv() callbacks used while the client reads that record. */ + wolfSSL_SSLSetIORecv(ssl_c, test_read_ahead_recv_cb); + wolfSSL_SetIOReadCtx(ssl_c, &recv_ctx); + XMEMSET(reply, 0, sizeof(reply)); + ExpectIntEQ(wolfSSL_read(ssl_c, reply, (int)sizeof(reply)), (int)sizeof(msg)); + ExpectIntEQ(XMEMCMP(reply, msg, sizeof(msg)), 0); + + if (recvCalls != NULL) + *recvCalls = recv_ctx.count; + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + + return EXPECT_RESULT(); +} + +static int test_wolfSSL_read_ahead(void) +{ + EXPECT_DECLS; + int callsOff = 0; + int callsOn = 0; + + ExpectIntEQ(test_wolfSSL_read_ahead_one(0, &callsOff), TEST_SUCCESS); + ExpectIntEQ(test_wolfSSL_read_ahead_one(1, &callsOn), TEST_SUCCESS); + + /* Without read-ahead the record header and body are fetched with separate + * recv() calls; with read-ahead the whole record arrives in a single + * recv(), so the body read issues no syscall. */ + ExpectIntGE(callsOff, 2); + ExpectIntEQ(callsOn, 1); + ExpectIntGT(callsOff, callsOn); + + return EXPECT_RESULT(); +} + +/* When read-ahead coalesces an application data record together with a + * following record (here a close_notify), the app data must still be delivered + * before the connection-close is reported, and the buffered record must be + * visible to wolfSSL_has_pending(). */ +static int test_wolfSSL_read_ahead_coalesced(void) +{ + EXPECT_DECLS; + WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL; + WOLFSSL *ssl_c = NULL, *ssl_s = NULL; + struct test_memio_ctx test_ctx; + struct test_read_ahead_ctx recv_ctx; + char msg[] = "hello wolfssl read ahead"; + char reply[64]; + + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&recv_ctx, 0, sizeof(recv_ctx)); + recv_ctx.memio = &test_ctx; + recv_ctx.coalesce = 1; + + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_2_client_method, wolfTLSv1_2_server_method), 0); + ExpectIntEQ(wolfSSL_set_read_ahead(ssl_c, 1), WOLFSSL_SUCCESS); + ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 10, NULL), 0); + + /* Queue two records back-to-back: app data, then close_notify. */ + ExpectIntEQ(wolfSSL_write(ssl_s, msg, (int)sizeof(msg)), (int)sizeof(msg)); + wolfSSL_shutdown(ssl_s); + + wolfSSL_SSLSetIORecv(ssl_c, test_read_ahead_recv_cb); + wolfSSL_SetIOReadCtx(ssl_c, &recv_ctx); + + /* First read returns the app data, not the connection close. */ + XMEMSET(reply, 0, sizeof(reply)); + ExpectIntEQ(wolfSSL_read(ssl_c, reply, (int)sizeof(reply)), (int)sizeof(msg)); + ExpectIntEQ(XMEMCMP(reply, msg, sizeof(msg)), 0); + + /* The close_notify record was pulled in by read-ahead and is still + * buffered, so has_pending must report it even though no socket read + * happened. */ + ExpectIntEQ(wolfSSL_has_pending(ssl_c), 1); + + /* Next read consumes the buffered close_notify and reports the close. */ + ExpectIntEQ(wolfSSL_read(ssl_c, reply, (int)sizeof(reply)), 0); + ExpectIntEQ(wolfSSL_get_error(ssl_c, 0), WOLFSSL_ERROR_ZERO_RETURN); + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + + return EXPECT_RESULT(); +} + +/* Max per-record payload the helper can drive; actual record size is a + * parameter. Kept so several records stay under the 64 KB memio buffer. */ +#define TEST_READ_AHEAD_MAX_REC 16000 + +/* Drive numRec records of recSz bytes through the client with a configurable + * read-ahead window. Reports how many recv() callbacks it took to drain them + * (recvCalls) and the input buffer's retained size afterwards (bufSize). + * bufLen == 0 leaves the one-record default in place. */ +static int test_wolfSSL_read_ahead_buffer_len_run(word32 bufLen, int recSz, + int numRec, int *recvCalls, word32 *bufSize) +{ + EXPECT_DECLS; + WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL; + WOLFSSL *ssl_c = NULL, *ssl_s = NULL; + struct test_memio_ctx test_ctx; + struct test_read_ahead_ctx recv_ctx; + byte msg[TEST_READ_AHEAD_MAX_REC]; + byte reply[TEST_READ_AHEAD_MAX_REC]; + int i; + + XMEMSET(msg, 'A', sizeof(msg)); + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&recv_ctx, 0, sizeof(recv_ctx)); + recv_ctx.memio = &test_ctx; + recv_ctx.coalesce = 1; + + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_2_client_method, wolfTLSv1_2_server_method), 0); + ExpectIntEQ(wolfSSL_set_read_ahead(ssl_c, 1), WOLFSSL_SUCCESS); + if (bufLen > 0) { + ExpectIntEQ(wolfSSL_set_default_read_buffer_len(ssl_c, bufLen), + WOLFSSL_SUCCESS); + /* Values within range are stored as given (sub-record sizes are + * honoured, not rounded up to a record). Clamping of oversized values + * is covered by test_wolfSSL_read_ahead_ctx_inherit. */ + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(ssl_c), (long)bufLen); + } + ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 10, NULL), 0); + + /* Server queues several records back-to-back before the client reads. */ + for (i = 0; i < numRec; i++) { + ExpectIntEQ(wolfSSL_write(ssl_s, msg, recSz), recSz); + } + + wolfSSL_SSLSetIORecv(ssl_c, test_read_ahead_recv_cb); + wolfSSL_SetIOReadCtx(ssl_c, &recv_ctx); + + for (i = 0; i < numRec; i++) { + XMEMSET(reply, 0, sizeof(reply)); + ExpectIntEQ(wolfSSL_read(ssl_c, reply, recSz), recSz); + ExpectIntEQ(XMEMCMP(reply, msg, (size_t)recSz), 0); + } + + if (recvCalls != NULL) + *recvCalls = recv_ctx.count; + /* Capture the retained input buffer size while the session is still alive; + * read-ahead keeps the grown buffer rather than shrinking per record. */ + if (bufSize != NULL && ssl_c != NULL) + *bufSize = ssl_c->buffers.inputBuffer.bufferSize; + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + + return EXPECT_RESULT(); +} + +static int test_wolfSSL_read_ahead_buffer_len(void) +{ + EXPECT_DECLS; + int callsDefault = 0; + int callsLarge = 0; + word32 bufDefault = 0; + word32 bufSmall = 0; + word32 bufShrunk = 0; + + /* Coalescing: with three near-max records, a window spanning all of them + * pulls every record in fewer recv() callbacks than the one-record default + * (which can only hold one such record at a time). */ + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(0, TEST_READ_AHEAD_MAX_REC, + 3, &callsDefault, NULL), TEST_SUCCESS); + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run( + 3 * TEST_READ_AHEAD_MAX_REC + 4096, TEST_READ_AHEAD_MAX_REC, 3, + &callsLarge, NULL), TEST_SUCCESS); + ExpectIntGT(callsDefault, callsLarge); + + /* Footprint: with small records, a sub-record window is honoured (not + * clamped up to a full record), so the retained input buffer is much + * smaller than the one-record default while data is still read correctly. */ + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(0, 1024, 4, NULL, + &bufDefault), TEST_SUCCESS); + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(2048, 1024, 4, NULL, + &bufSmall), TEST_SUCCESS); + /* The footprint comparison only holds when the input buffer actually goes + * dynamic, i.e. the static buffer is smaller than the configured window. + * With LARGE_STATIC_BUFFERS (or a large user STATIC_BUFFER_LEN) the buffer + * never grows and both runs retain STATIC_BUFFER_LEN, so skip the size + * checks there; the runs above still verify the data is read correctly. + * STATIC_BUFFER_LEN resolves to an enum constant, so this is a runtime (not + * preprocessor) guard that the compiler folds away. */ + if (STATIC_BUFFER_LEN < 2048) { + ExpectIntGT(bufDefault, bufSmall); + /* The 2 KB window retains roughly a 2 KB buffer, below one TLS record. */ + ExpectIntLE(bufSmall, 4096); + } + + /* Shrink-back: an 8 KB record exceeds the 2 KB window, growing the buffer to + * receive it, but read-ahead reallocates back down to the window afterwards, + * so the retained buffer tracks the window (~2 KB) rather than the record. */ + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(2048, 8000, 2, NULL, + &bufShrunk), TEST_SUCCESS); + if (STATIC_BUFFER_LEN < 2048) { + ExpectIntLE(bufShrunk, 4096); + } + + return EXPECT_RESULT(); +} + +/* Cover the CTX-level read-buffer-len setter/getter, CTX->SSL inheritance via + * SetSSL_CTX(), NULL-argument handling for all four accessors, and the + * oversized-window clamp. */ +static int test_wolfSSL_read_ahead_ctx_inherit(void) +{ + EXPECT_DECLS; + WOLFSSL_CTX* ctx_c = NULL; + WOLFSSL* ssl_c = NULL; + + /* NULL arguments must fail cleanly, not crash. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(NULL, 4096), + WOLFSSL_FAILURE); + ExpectIntEQ(wolfSSL_set_default_read_buffer_len(NULL, 4096), + WOLFSSL_FAILURE); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(NULL), WOLFSSL_FAILURE); + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(NULL), WOLFSSL_FAILURE); + + ExpectNotNull(ctx_c = wolfSSL_CTX_new(wolfTLSv1_2_client_method())); + + /* A fresh context carries the one-record default window, not 0. */ + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), + (long)WOLFSSL_READ_AHEAD_SZ); + + /* CTX-level setter/getter round-trip. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, 8192), + WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), 8192); + + /* 0 resets to the one-record default rather than storing 0. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, 0), + WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), + (long)WOLFSSL_READ_AHEAD_SZ); + + /* A window above the maximum is clamped to WOLFSSL_MAX_READ_AHEAD_SZ, not + * wrapped around by the size_t->word32 store. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, + (size_t)WOLFSSL_MAX_READ_AHEAD_SZ + 4096), WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), + (long)WOLFSSL_MAX_READ_AHEAD_SZ); + + /* Set a plain value and confirm a freshly created WOLFSSL inherits it + * through SetSSL_CTX(). */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, 8192), + WOLFSSL_SUCCESS); + ExpectNotNull(ssl_c = wolfSSL_new(ctx_c)); + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(ssl_c), 8192); + + /* The session-level setter overrides the inherited value. */ + ExpectIntEQ(wolfSSL_set_default_read_buffer_len(ssl_c, 2048), + WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(ssl_c), 2048); + + wolfSSL_free(ssl_c); + wolfSSL_CTX_free(ctx_c); + + return EXPECT_RESULT(); +} + +#undef TEST_READ_AHEAD_MAX_REC +#else +static int test_wolfSSL_read_ahead(void) +{ + return TEST_SKIPPED; +} +static int test_wolfSSL_read_ahead_coalesced(void) +{ + return TEST_SKIPPED; +} +static int test_wolfSSL_read_ahead_buffer_len(void) +{ + return TEST_SKIPPED; +} +static int test_wolfSSL_read_ahead_ctx_inherit(void) +{ + return TEST_SKIPPED; +} +#endif + static int test_wolfSSL_inject(void) { EXPECT_DECLS; @@ -37716,6 +38064,10 @@ TEST_CASE testCases[] = { TEST_DECL(test_wolfSSL_SendUserCanceled), TEST_DECL(test_wolfSSL_SSLDisableRead), TEST_DECL(test_wolfSSL_shutdown_pending_data_uaf), + TEST_DECL(test_wolfSSL_read_ahead), + TEST_DECL(test_wolfSSL_read_ahead_coalesced), + TEST_DECL(test_wolfSSL_read_ahead_buffer_len), + TEST_DECL(test_wolfSSL_read_ahead_ctx_inherit), TEST_DECL(test_wolfSSL_inject), TEST_DECL(test_ocsp_status_callback), TEST_DECL(test_ocsp_basic_verify), diff --git a/wolfssl/internal.h b/wolfssl/internal.h index d54496e0f9c..94ec917f6ff 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -2375,6 +2375,29 @@ enum { #define STATIC_BUFFER_LEN RECORD_HEADER_SZ #endif +/* Default read-ahead window: when read-ahead is enabled the record header read + * requests up to a full record's worth of data in a single recv() so the body + * (and possibly following records) can be pulled in without a second syscall. + * Sized to one maximum TLS record (MAX_RECORD_SIZE, not the buffer-sizing + * RECORD_SIZE which may be small) so the whole record is captured. Defined + * unconditionally so the setters and CTX init can reference it as the default + * window even when read-ahead I/O is not built. */ +#ifndef WOLFSSL_READ_AHEAD_SZ +#define WOLFSSL_READ_AHEAD_SZ (RECORD_HEADER_SZ + MAX_RECORD_SIZE + \ + COMP_EXTRA + MTU_EXTRA + MAX_MSG_EXTRA) +#endif + +/* Upper bound for a caller-configured read-ahead window + * (wolfSSL_CTX/SSL_set_default_read_buffer_len()). The window feeds signed int + * arithmetic in GetInputData_ex(); bounding it well below INT_MAX ensures a + * large caller-supplied size can never overflow that arithmetic to a negative + * value (which would skip GrowInputBuffer() and drive an oversized recv()). + * 16 MB is far above any realistic coalescing window. Defined unconditionally + * so the setters can clamp even when read-ahead I/O is not built. */ +#ifndef WOLFSSL_MAX_READ_AHEAD_SZ +#define WOLFSSL_MAX_READ_AHEAD_SZ (16 * 1024 * 1024) +#endif + typedef struct { ALIGN16 byte staticBuffer[STATIC_BUFFER_LEN]; byte* buffer; /* place holder for static or dynamic buffer */ @@ -4225,8 +4248,16 @@ struct WOLFSSL_CTX { WOLFSSL_X509_STORE x509_store; /* points to ctx->cm */ WOLFSSL_X509_STORE* x509_store_pt; /* take ownership of external store */ #endif -#if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) || defined(WOLFSSL_WPAS_SMALL) +#if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) || \ + defined(WOLFSSL_WPAS_SMALL) || defined(WOLFSSL_TLS_READ_AHEAD) byte readAhead; +#endif +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) + /* Read-ahead coalescing buffer size. 0 = use one record (default). See + * wolfSSL_CTX_set_default_read_buffer_len(). */ + word32 readAheadSz; +#endif +#if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) || defined(WOLFSSL_WPAS_SMALL) void* userPRFArg; /* passed to prf callback */ #endif #ifdef HAVE_EX_DATA @@ -6282,8 +6313,12 @@ struct WOLFSSL { defined(OPENSSL_ALL) unsigned long peerVerifyRet; #endif -#ifdef OPENSSL_EXTRA +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) byte readAhead; + /* Read-ahead coalescing buffer size; 0 = one record (default). */ + word32 readAheadSz; +#endif +#ifdef OPENSSL_EXTRA #ifdef HAVE_PK_CALLBACKS void* loggingCtx; /* logging callback argument */ #endif diff --git a/wolfssl/openssl/ssl.h b/wolfssl/openssl/ssl.h index cdc8ea64adb..70028632cbb 100644 --- a/wolfssl/openssl/ssl.h +++ b/wolfssl/openssl/ssl.h @@ -1438,6 +1438,8 @@ typedef WOLFSSL_SRTP_PROTECTION_PROFILE SRTP_PROTECTION_PROFILE; #define SSL_set_read_ahead wolfSSL_set_read_ahead #define SSL_CTX_get_read_ahead wolfSSL_CTX_get_read_ahead #define SSL_CTX_set_read_ahead wolfSSL_CTX_set_read_ahead +#define SSL_CTX_set_default_read_buffer_len wolfSSL_CTX_set_default_read_buffer_len +#define SSL_set_default_read_buffer_len wolfSSL_set_default_read_buffer_len #define SSL_CTX_set_tlsext_status_arg wolfSSL_CTX_set_tlsext_status_arg #define SSL_CTX_set_tlsext_opaque_prf_input_callback_arg \ wolfSSL_CTX_set_tlsext_opaque_prf_input_callback_arg diff --git a/wolfssl/ssl.h b/wolfssl/ssl.h index b5a0fb4a07a..0b0ab164d52 100644 --- a/wolfssl/ssl.h +++ b/wolfssl/ssl.h @@ -2718,10 +2718,6 @@ WOLFSSL_API long wolfSSL_CTX_sess_set_cache_size(WOLFSSL_CTX* ctx, long sz); WOLFSSL_API long wolfSSL_CTX_sess_get_cache_size(WOLFSSL_CTX* ctx); WOLFSSL_API long wolfSSL_CTX_get_session_cache_mode(WOLFSSL_CTX* ctx); -WOLFSSL_API int wolfSSL_get_read_ahead(const WOLFSSL* ssl); -WOLFSSL_API int wolfSSL_set_read_ahead(WOLFSSL* ssl, int v); -WOLFSSL_API int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); -WOLFSSL_API int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v); WOLFSSL_API long wolfSSL_CTX_set_tlsext_opaque_prf_input_callback_arg( WOLFSSL_CTX* ctx, void* arg); WOLFSSL_API int wolfSSL_CTX_add_client_CA(WOLFSSL_CTX* ctx, WOLFSSL_X509* x509); @@ -2754,6 +2750,23 @@ WOLFSSL_API long wolfSSL_get_verify_result(const WOLFSSL *ssl); WOLFSSL_API void* wolfSSL_get_app_data( const WOLFSSL *ssl); #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */ +/* Read-ahead control is part of the OpenSSL compatibility layer, and is also + * exposed when TLS read-ahead support is built without that layer. Guard must + * match the definitions in ssl.c (which use the readAhead/readAheadSz struct + * members available only under these macros), so it deliberately excludes + * OPENSSL_EXTRA_X509_SMALL. */ +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) +WOLFSSL_API int wolfSSL_get_read_ahead(const WOLFSSL* ssl); +WOLFSSL_API int wolfSSL_set_read_ahead(WOLFSSL* ssl, int v); +WOLFSSL_API int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); +WOLFSSL_API int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v); +WOLFSSL_API int wolfSSL_CTX_set_default_read_buffer_len(WOLFSSL_CTX* ctx, + size_t len); +WOLFSSL_API int wolfSSL_set_default_read_buffer_len(WOLFSSL* ssl, size_t len); +WOLFSSL_API long wolfSSL_CTX_get_default_read_buffer_len(WOLFSSL_CTX* ctx); +WOLFSSL_API long wolfSSL_get_default_read_buffer_len(const WOLFSSL* ssl); +#endif + #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL) || \ defined(HAVE_WEBSERVER) || defined(HAVE_MEMCACHED)