From 2499c7d66943d93c0ad83e25eec806d5adf89da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Frauenschl=C3=A4ger?= Date: Fri, 19 Jun 2026 08:34:30 +0200 Subject: [PATCH] Add TLS receive read-ahead support Add WOLFSSL_TLS_READ_AHEAD (--enable-readahead), toggled at runtime via wolfSSL_set_read_ahead(). When enabled, the record-header read pulls a full record in one recv() so the body arrives without a second syscall. The receive window is configurable with wolfSSL_CTX/SSL_set_default_read_buffer_len() (OpenSSL-compatible): 0 keeps the one-record default, a larger value coalesces several records per recv(), a smaller value caps the per-connection buffer footprint. Records exceeding the window are still received correctly, the buffer grows on demand and is reallocated back down to the window afterwards so the retained footprint stays bounded. Includes docs, API tests, and a benchmark toggle. --- configure.ac | 14 ++ doc/dox_comments/header_files/ssl.h | 94 ++++++++ examples/benchmark/tls_bench.c | 14 ++ examples/client/client.c | 9 + src/internal.c | 110 ++++++++- src/ssl.c | 92 +++++++- tests/api.c | 352 ++++++++++++++++++++++++++++ wolfssl/internal.h | 39 ++- wolfssl/openssl/ssl.h | 2 + wolfssl/ssl.h | 21 +- 10 files changed, 736 insertions(+), 11 deletions(-) diff --git a/configure.ac b/configure.ac index 537a51ff99c..2430a317442 100644 --- a/configure.ac +++ b/configure.ac @@ -2225,6 +2225,19 @@ then AM_CFLAGS="$AM_CFLAGS -DSHOW_SECRETS -DHAVE_SECRET_CALLBACK -DWOLFSSL_SSLKEYLOGFILE -DWOLFSSL_KEYLOG_EXPORT_WARNED" fi +# TLS receive read-ahead: when enabled at runtime via wolfSSL_set_read_ahead(), +# the record-header read pulls in up to a full record in one recv(), avoiding a +# second syscall for the body. Opt-in (default: disabled). +AC_ARG_ENABLE([readahead], + [AS_HELP_STRING([--enable-readahead],[Enable TLS receive read-ahead support, activated at runtime with wolfSSL_set_read_ahead() (default: disabled)])], + [ ENABLED_READAHEAD=$enableval ], + [ ENABLED_READAHEAD=no ] + ) +if test "$ENABLED_READAHEAD" = "yes" +then + AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_TLS_READ_AHEAD" +fi + # TLS v1.3 Draft 18 (Note: only final TLS v1.3 supported, here for backwards build compatibility) AC_ARG_ENABLE([tls13-draft18], [AS_HELP_STRING([--enable-tls13-draft18],[Enable wolfSSL TLS v1.3 Draft 18 (default: disabled)])], @@ -13303,6 +13316,7 @@ echo " * Dual alg cert support: $ENABLED_DUAL_ALG_CERTS" echo " * ERR Queues per Thread: $ENABLED_ERRORQUEUEPERTHREAD" echo " * rwlock: $ENABLED_RWLOCK" echo " * keylog export: $ENABLED_KEYLOG_EXPORT" +echo " * TLS receive read-ahead: $ENABLED_READAHEAD" echo " * AutoSAR : $ENABLED_AUTOSAR" echo " * ML-KEM standalone: $ENABLED_MLKEM_STANDALONE" echo " * PQ/T hybrids: $ENABLED_PQC_HYBRIDS" diff --git a/doc/dox_comments/header_files/ssl.h b/doc/dox_comments/header_files/ssl.h index 59f44c3a54f..cd3c3c88629 100644 --- a/doc/dox_comments/header_files/ssl.h +++ b/doc/dox_comments/header_files/ssl.h @@ -5486,6 +5486,21 @@ int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); \ingroup Setup \brief This function sets the read ahead flag in the WOLFSSL_CTX structure. + When enabled, the record-header read pulls in up to a full TLS record in a + single recv(), so the body (and any following buffered records) is obtained + without a second syscall. The flag only changes I/O behaviour when the + library is built with read-ahead support (--enable-readahead / + WOLFSSL_TLS_READ_AHEAD); otherwise it is stored but inert. + + \note With read-ahead enabled, undecrypted data can remain buffered + internally while the socket has no more data to read. wolfSSL_has_pending() + reports whether any such data is buffered, but a non-zero return does not + guarantee a full record is available: wolfSSL_read() may still return + WANT_READ when only a partial record is buffered. Event-driven applications + should therefore drain the connection by calling wolfSSL_read() until it + returns WANT_READ (or an error) before returning to select()/poll(), rather + than looping on wolfSSL_has_pending() alone; otherwise buffered data could + be missed or the loop could spin. \return SSL_SUCCESS If ctx read ahead flag set. \return SSL_FAILURE If ctx is NULL then SSL_FAILURE is returned. @@ -5506,9 +5521,88 @@ int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); \sa wolfSSL_CTX_new \sa wolfSSL_CTX_free \sa wolfSSL_CTX_get_read_ahead + \sa wolfSSL_has_pending */ int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v); +/*! + \ingroup Setup + + \brief This function sets the read-ahead receive window size for contexts + created from this WOLFSSL_CTX. When read-ahead is enabled + (wolfSSL_CTX_set_read_ahead()), a single recv() pulls in up to \p len bytes: + - \p len of 0 resets the window to the one-record default. This is also + the window a freshly created context already carries, so the record body + is read together with its header without a second syscall. + - A \p len larger than one record lets a single recv() coalesce several + back-to-back records, one syscall instead of one per record. + - A \p len smaller than one record caps the receive buffer's footprint + when the peer's records are known to be small (e.g. 4 KB), saving heap + versus the one-record default. + \p len is only a speculative read window, not a hard limit: a record larger + than \p len is still received correctly, with the input buffer grown to the + record's actual size on demand (costing an extra syscall and reallocation + for that record) and then reallocated back down to the window once the + oversized record is consumed, so the retained footprint stays bounded by + \p len rather than by the largest record seen. The setting only affects I/O + when the library is built with read-ahead support (--enable-readahead / + WOLFSSL_TLS_READ_AHEAD); otherwise it is stored but inert. This is the + wolfSSL equivalent of OpenSSL's SSL_CTX_set_default_read_buffer_len(); unlike + OpenSSL it returns a status code (callers that ignore the return remain + source-compatible) and it honours sizes below one record, whereas OpenSSL + only ever enlarges the buffer. A \p len above WOLFSSL_MAX_READ_AHEAD_SZ + (16 MB) is clamped to that maximum. Because 0 and oversized values are both + normalised, wolfSSL_CTX_get_default_read_buffer_len() reports the effective + window (the one-record default rather than 0 when unset), not the raw + argument. + + \note This is a memory-vs-syscall trade-off. While read-ahead is enabled the + input buffer is retained (bounded to \p len) for the connection's lifetime, + so a large \p len multiplied by many concurrent connections is persistent + memory, while a small \p len bounds per-connection footprint at the cost of + more syscalls for records that exceed it. + + \return SSL_SUCCESS If the buffer length was set. + \return SSL_FAILURE If ctx is NULL. + + \param ctx WOLFSSL_CTX structure to set the read-ahead buffer length on. + \param len read-ahead coalescing buffer size in bytes (0 = one record). + + _Example_ + \code + WOLFSSL_CTX* ctx; + // setup ctx + wolfSSL_CTX_set_read_ahead(ctx, 1); + // coalesce up to four max-size records per recv() + wolfSSL_CTX_set_default_read_buffer_len(ctx, 4 * 16384); + \endcode + + \sa wolfSSL_CTX_set_read_ahead + \sa wolfSSL_set_default_read_buffer_len + \sa wolfSSL_has_pending +*/ +int wolfSSL_CTX_set_default_read_buffer_len(WOLFSSL_CTX* ctx, size_t len); + +/*! + \ingroup Setup + + \brief This function sets the read-ahead coalescing buffer size on a single + WOLFSSL session, overriding the value inherited from its WOLFSSL_CTX. See + wolfSSL_CTX_set_default_read_buffer_len() for the full description, the + one-record default, and the memory-vs-syscall trade-off. + + \return SSL_SUCCESS If the buffer length was set. + \return SSL_FAILURE If ssl is NULL. + + \param ssl WOLFSSL structure to set the read-ahead buffer length on. + \param len read-ahead coalescing buffer size in bytes (0 = one record). + + \sa wolfSSL_CTX_set_default_read_buffer_len + \sa wolfSSL_set_read_ahead + \sa wolfSSL_has_pending +*/ +int wolfSSL_set_default_read_buffer_len(WOLFSSL* ssl, size_t len); + /*! \ingroup Setup diff --git a/examples/benchmark/tls_bench.c b/examples/benchmark/tls_bench.c index a3f3675730e..72a6382c0aa 100644 --- a/examples/benchmark/tls_bench.c +++ b/examples/benchmark/tls_bench.c @@ -1100,6 +1100,13 @@ static int bench_tls_client(info_t* info) #endif wolfSSL_SetIOReadCtx(cli_ssl, info); wolfSSL_SetIOWriteCtx(cli_ssl, info); +#if defined(WOLFSSL_TLS_READ_AHEAD) && !defined(NO_FILESYSTEM) && \ + !defined(NO_STDIO_FILESYSTEM) + /* Optional A/B benchmark toggle for TLS receive read-ahead. Gated on + * filesystem support since XGETENV is only defined there. */ + if (XGETENV("WOLF_BENCH_READ_AHEAD") != NULL) + wolfSSL_set_read_ahead(cli_ssl, 1); +#endif #if !defined(SINGLE_THREADED) && defined(WOLFSSL_DTLS) /* synchronize with server */ @@ -1557,6 +1564,13 @@ static int bench_tls_server(info_t* info) wolfSSL_SetIOReadCtx(srv_ssl, info); wolfSSL_SetIOWriteCtx(srv_ssl, info); +#if defined(WOLFSSL_TLS_READ_AHEAD) && !defined(NO_FILESYSTEM) && \ + !defined(NO_STDIO_FILESYSTEM) + /* Optional A/B benchmark toggle for TLS receive read-ahead. Gated on + * filesystem support since XGETENV is only defined there. */ + if (XGETENV("WOLF_BENCH_READ_AHEAD") != NULL) + wolfSSL_set_read_ahead(srv_ssl, 1); +#endif #ifndef NO_DH wolfSSL_SetTmpDH(srv_ssl, dhp, sizeof(dhp), dhg, sizeof(dhg)); #endif diff --git a/examples/client/client.c b/examples/client/client.c index 64958f2091e..338c7c9234d 100644 --- a/examples/client/client.c +++ b/examples/client/client.c @@ -790,6 +790,15 @@ static int ClientBenchmarkThroughput(WOLFSSL_CTX* ctx, char* host, word16 port, if (ssl == NULL) err_sys("unable to get SSL object"); +#if defined(WOLFSSL_TLS_READ_AHEAD) && !defined(NO_FILESYSTEM) && \ + !defined(NO_STDIO_FILESYSTEM) + /* Optional A/B toggle: enable TLS receive read-ahead for the throughput + * benchmark when WOLF_BENCH_READ_AHEAD is set in the environment. Gated on + * filesystem support since XGETENV is only defined there. */ + if (XGETENV("WOLF_BENCH_READ_AHEAD") != NULL) + wolfSSL_set_read_ahead(ssl, 1); +#endif + tcp_connect(&sockfd, host, port, dtlsUDP, dtlsSCTP, ssl); if (wolfSSL_set_fd(ssl, sockfd) != WOLFSSL_SUCCESS) { err_sys("error in setting fd"); diff --git a/src/internal.c b/src/internal.c index 0ea1b04f2b7..b13a97529b4 100644 --- a/src/internal.c +++ b/src/internal.c @@ -2650,6 +2650,15 @@ int InitSSL_Ctx(WOLFSSL_CTX* ctx, WOLFSSL_METHOD* method, void* heap) } ctx->timeout = WOLFSSL_SESSION_TIMEOUT; +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) + /* Default the read-ahead window to one full record. Contexts (and the + * WOLFSSL objects that inherit it) then always carry a concrete window, so + * the receive path uses ssl->readAheadSz directly without a per-read + * fallback. A caller override replaces it; passing 0 resets it to this + * default (see wolfSSL_CTX_set_default_read_buffer_len()). */ + ctx->readAheadSz = WOLFSSL_READ_AHEAD_SZ; +#endif + #ifdef WOLFSSL_DTLS if (method->version.major == DTLS_MAJOR) { ctx->minDowngrade = WOLFSSL_MIN_DTLS_DOWNGRADE; @@ -7509,8 +7518,9 @@ int SetSSL_CTX(WOLFSSL* ssl, WOLFSSL_CTX* ctx, int writeDup) ssl->ConnectFilter_arg = ctx->ConnectFilter_arg; #endif -#ifdef OPENSSL_EXTRA +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) ssl->readAhead = ctx->readAhead; + ssl->readAheadSz = ctx->readAheadSz; #endif #if defined(OPENSSL_EXTRA) && !defined(NO_BIO) /* Don't change recv callback if currently using BIO's */ @@ -11619,6 +11629,41 @@ void ShrinkInputBuffer(WOLFSSL* ssl, int forcedFree) ssl->buffers.clearOutputBuffer.length > 0)) return; +#ifdef WOLFSSL_TLS_READ_AHEAD + /* While read-ahead is enabled, retain a dynamic input buffer sized to the + * configured window rather than shrinking all the way back to the static + * buffer, so the speculative over-read is a bounded, mostly one-time + * allocation instead of per-record churn. A forced free (connection + * teardown) still reclaims everything. */ + if (!forcedFree && ssl->readAhead) { + /* Already within the window: keep the buffer as-is. */ + if (ssl->buffers.inputBuffer.bufferSize <= ssl->readAheadSz) + return; + + /* The buffer grew past the window to receive an oversized record. When + * the window needs a dynamic buffer, reallocate down to it so the + * retained footprint tracks the window, not the largest record seen; + * when the window fits in the static buffer, fall through and shrink to + * static below. + * + * GrowInputBuffer(newBytes, usedLength) resizes the input buffer to + * newBytes + usedLength while preserving the usedLength bytes still + * pending, so requesting (readAheadSz - usedLength) new bytes yields a + * buffer of exactly readAheadSz. usedLength is <= STATIC_BUFFER_LEN < + * readAheadSz here (guaranteed above), so the subtraction stays + * positive. */ + if (ssl->readAheadSz > STATIC_BUFFER_LEN) { + if (GrowInputBuffer(ssl, (int)ssl->readAheadSz - usedLength, + usedLength) != 0) { + /* Realloc failed: keep the current (larger) buffer rather than + * dropping the buffered data. */ + WOLFSSL_MSG("read-ahead buffer shrink failed, keeping buffer"); + } + return; + } + } +#endif + WOLFSSL_MSG("Shrinking input buffer"); if (!forcedFree && usedLength > 0) { @@ -23189,12 +23234,16 @@ static int DoAlert(WOLFSSL* ssl, byte* input, word32* inOutIdx, int* type) return level; } -static int GetInputData(WOLFSSL *ssl, word32 size) +static int GetInputData_ex(WOLFSSL *ssl, word32 size, word32 readAhead) { int inSz; int maxLength; int usedLength; int dtlsExtra = 0; + int extra = 0; +#ifndef WOLFSSL_TLS_READ_AHEAD + (void)readAhead; +#endif if (ssl->options.disableRead) return WC_NO_ERR_TRACE(WANT_READ); @@ -23231,10 +23280,27 @@ static int GetInputData(WOLFSSL *ssl, word32 size) } inSz = (int)(size - (word32)usedLength); /* from last partial read */ + +#ifdef WOLFSSL_TLS_READ_AHEAD + /* Request more than the minimum so that a single recv() can also pull + * in the record body (and possibly following records), avoiding a + * second syscall. 'size' remains the loop-termination minimum, so a + * blocking socket never waits for read-ahead bytes the peer may not + * send. Mirrors the DTLS dtlsExtra over-read above. + * + * The buffer is grown once to hold a full record and then kept (see the + * ssl->readAhead guard in ShrinkInputBuffer), so this is a one-time + * allocation per connection, not per-record churn. */ + if (readAhead > size) { + extra = (int)(readAhead - size); + inSz += extra; + } +#endif } if (inSz > maxLength) { - if (GrowInputBuffer(ssl, (int)(size + (word32)dtlsExtra), usedLength) < 0) + if (GrowInputBuffer(ssl, + (int)(size + (word32)dtlsExtra + (word32)extra), usedLength) < 0) return MEMORY_E; } @@ -23292,6 +23358,11 @@ static int GetInputData(WOLFSSL *ssl, word32 size) return 0; } +static int GetInputData(WOLFSSL *ssl, word32 size) +{ + return GetInputData_ex(ssl, size, 0); +} + #if defined(HAVE_ENCRYPT_THEN_MAC) && !defined(WOLFSSL_AEAD_ONLY) static WC_INLINE int VerifyMacEnc(WOLFSSL* ssl, const byte* input, word32 msgSz, int content) @@ -24123,7 +24194,28 @@ static int DoProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) /* get header or return error */ if (!ssl->options.dtls) { - if ((ret = GetInputData(ssl, (word32)readSz)) < 0) + word32 readAheadSz = 0; + #ifdef WOLFSSL_TLS_READ_AHEAD + /* When read-ahead is enabled, request more than the record + * header in a single recv() so the body (and possibly following + * records) can be pulled in without a second syscall. The window + * size is configurable via + * wolfSSL_CTX_set_default_read_buffer_len(): + * - 0 (unset) requests one full record, the sensible default. + * - A larger value lets one recv() coalesce several back-to-back + * records. + * - A smaller (sub-record) value caps the receive buffer's + * footprint when the peer's records are known to be small. + * The value is only a speculative read window: a record larger + * than it is still received correctly, as the buffer is grown to + * the record's actual size on demand. */ + if (ssl->readAhead) { + /* readAheadSz is always concrete (defaulted to + * WOLFSSL_READ_AHEAD_SZ at CTX init, never 0). */ + readAheadSz = ssl->readAheadSz; + } + #endif + if ((ret = GetInputData_ex(ssl, (word32)readSz, readAheadSz)) < 0) return ret; } else { #ifdef WOLFSSL_DTLS @@ -24848,6 +24940,16 @@ static int DoProcessReplyEx(WOLFSSL* ssl, int allowSocketErr) ssl->options.serverState == SERVER_FINISHED_COMPLETE && ssl->options.handShakeState != HANDSHAKE_DONE))) +#endif +#ifdef WOLFSSL_TLS_READ_AHEAD + /* With read-ahead, more than one record may be buffered. If + * application data was just decrypted, return it now so it + * is delivered to the caller before any following buffered + * record (e.g. a close_notify alert) is processed, which + * would otherwise discard the pending app data. The + * remaining records stay buffered for the next call. */ + || (ssl->curRL.type == application_data && + ssl->buffers.clearOutputBuffer.length > 0) #endif ) { /* Shrink input buffer when we successfully finish record diff --git a/src/ssl.c b/src/ssl.c index f5a309552c1..cb76547e4ab 100644 --- a/src/ssl.c +++ b/src/ssl.c @@ -2829,7 +2829,21 @@ int wolfSSL_has_pending(const WOLFSSL* ssl) if (ssl == NULL) return WOLFSSL_FAILURE; - return ssl->buffers.clearOutputBuffer.length > 0; + if (ssl->buffers.clearOutputBuffer.length > 0) + return 1; + +#ifdef WOLFSSL_TLS_READ_AHEAD + /* Read-ahead can leave undecrypted data buffered while the socket itself + * has no more data. This may be a complete record or only a partial one + * (e.g. a coalesced read that pulled a record plus the head of the next), + * so a non-zero return does not guarantee wolfSSL_read() will yield + * application data without another socket read. Report it so a + * select()/poll() loop keeps draining until wolfSSL_read() reports + * WANT_READ, instead of stalling on buffered data. */ + if (ssl->buffers.inputBuffer.length > ssl->buffers.inputBuffer.idx) + return 1; +#endif + return 0; } #ifndef WOLFSSL_LEANPSK @@ -10847,6 +10861,9 @@ long wolfSSL_CTX_sess_timeouts(WOLFSSL_CTX* ctx) } #endif +#endif /* OPENSSL_EXTRA */ + +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) int wolfSSL_get_read_ahead(const WOLFSSL* ssl) { if (ssl == NULL) { @@ -10890,7 +10907,80 @@ int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v) return WOLFSSL_SUCCESS; } +/* Set the read-ahead receive window size. When read-ahead is enabled, a single + * recv() pulls in up to this many bytes: + * - 0 (the default) requests one full record. + * - A value larger than one record lets a single recv() coalesce several + * back-to-back records. + * - A value smaller than one record is honoured as-is, capping the receive + * buffer's footprint when the peer's records are known to be small. + * 'len' is only a speculative window: a record larger than it is still received + * correctly, with the buffer grown on demand. Effective only when the library + * is built with read-ahead support (WOLFSSL_TLS_READ_AHEAD). Returns int rather + * than OpenSSL's void to match wolfSSL's other read-ahead setters; callers that + * ignore the return remain source-compatible. */ +int wolfSSL_CTX_set_default_read_buffer_len(WOLFSSL_CTX* ctx, size_t len) +{ + if (ctx == NULL) { + return WOLFSSL_FAILURE; + } + + /* 0 selects the default one-record window (matches OpenSSL, where 0 means + * "use the built-in default"). Otherwise clamp to a sane maximum so the + * stored window cannot overflow the signed arithmetic in the receive path + * (see WOLFSSL_MAX_READ_AHEAD_SZ). */ + if (len == 0) { + len = WOLFSSL_READ_AHEAD_SZ; + } + else if (len > (size_t)WOLFSSL_MAX_READ_AHEAD_SZ) { + len = (size_t)WOLFSSL_MAX_READ_AHEAD_SZ; + } + ctx->readAheadSz = (word32)len; + + return WOLFSSL_SUCCESS; +} + +int wolfSSL_set_default_read_buffer_len(WOLFSSL* ssl, size_t len) +{ + if (ssl == NULL) { + return WOLFSSL_FAILURE; + } + /* 0 selects the default one-record window (matches OpenSSL, where 0 means + * "use the built-in default"). Otherwise clamp to a sane maximum so the + * stored window cannot overflow the signed arithmetic in the receive path + * (see WOLFSSL_MAX_READ_AHEAD_SZ). */ + if (len == 0) { + len = WOLFSSL_READ_AHEAD_SZ; + } + else if (len > (size_t)WOLFSSL_MAX_READ_AHEAD_SZ) { + len = (size_t)WOLFSSL_MAX_READ_AHEAD_SZ; + } + ssl->readAheadSz = (word32)len; + + return WOLFSSL_SUCCESS; +} + +long wolfSSL_CTX_get_default_read_buffer_len(WOLFSSL_CTX* ctx) +{ + if (ctx == NULL) { + return WOLFSSL_FAILURE; + } + + return (long)ctx->readAheadSz; +} + +long wolfSSL_get_default_read_buffer_len(const WOLFSSL* ssl) +{ + if (ssl == NULL) { + return WOLFSSL_FAILURE; + } + + return (long)ssl->readAheadSz; +} +#endif /* OPENSSL_EXTRA || WOLFSSL_TLS_READ_AHEAD */ + +#ifdef OPENSSL_EXTRA long wolfSSL_CTX_set_tlsext_opaque_prf_input_callback_arg(WOLFSSL_CTX* ctx, void* arg) { diff --git a/tests/api.c b/tests/api.c index 238ea673976..a3d705df6d2 100644 --- a/tests/api.c +++ b/tests/api.c @@ -35695,6 +35695,354 @@ static int test_wolfSSL_shutdown_pending_data_uaf(void) return EXPECT_RESULT(); } +#if defined(WOLFSSL_TLS_READ_AHEAD) && \ + defined(HAVE_MANUAL_MEMIO_TESTS_DEPENDENCIES) && \ + !defined(WOLFSSL_NO_TLS12) && !defined(NO_RSA) +/* Per-test recv context: counts callbacks and forwards to the memio reader. + * 'coalesce' makes one callback return as many queued records as fit, to + * emulate TCP coalescing several TLS records into a single recv(). */ +struct test_read_ahead_ctx { + struct test_memio_ctx* memio; + int count; + int coalesce; +}; + +static int test_read_ahead_recv_cb(WOLFSSL *ssl, char *buf, int sz, void *ctx) +{ + struct test_read_ahead_ctx* rc = (struct test_read_ahead_ctx*)ctx; + int total = 0; + int ret; + + rc->count++; + do { + ret = test_memio_read_cb(ssl, buf + total, sz - total, rc->memio); + if (ret <= 0) + break; + total += ret; + } while (rc->coalesce && total < sz); + + if (total > 0) + return total; + return ret; +} + +/* Read one application data record on the client, returning the number of + * recv() callback invocations it took. */ +static int test_wolfSSL_read_ahead_one(int enableReadAhead, int *recvCalls) +{ + EXPECT_DECLS; + WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL; + WOLFSSL *ssl_c = NULL, *ssl_s = NULL; + struct test_memio_ctx test_ctx; + struct test_read_ahead_ctx recv_ctx; + /* Use a multi-KB record so the test exercises a realistically sized record + * (larger than a minimal read-ahead window) rather than a tiny one. */ + byte msg[4000]; + byte reply[4000]; + + XMEMSET(msg, 'A', sizeof(msg)); + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&recv_ctx, 0, sizeof(recv_ctx)); + recv_ctx.memio = &test_ctx; + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_2_client_method, wolfTLSv1_2_server_method), 0); + + if (enableReadAhead) + ExpectIntEQ(wolfSSL_set_read_ahead(ssl_c, 1), WOLFSSL_SUCCESS); + + ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 10, NULL), 0); + + /* Server sends a single application data record. */ + ExpectIntEQ(wolfSSL_write(ssl_s, msg, (int)sizeof(msg)), (int)sizeof(msg)); + + /* Count the recv() callbacks used while the client reads that record. */ + wolfSSL_SSLSetIORecv(ssl_c, test_read_ahead_recv_cb); + wolfSSL_SetIOReadCtx(ssl_c, &recv_ctx); + XMEMSET(reply, 0, sizeof(reply)); + ExpectIntEQ(wolfSSL_read(ssl_c, reply, (int)sizeof(reply)), (int)sizeof(msg)); + ExpectIntEQ(XMEMCMP(reply, msg, sizeof(msg)), 0); + + if (recvCalls != NULL) + *recvCalls = recv_ctx.count; + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + + return EXPECT_RESULT(); +} + +static int test_wolfSSL_read_ahead(void) +{ + EXPECT_DECLS; + int callsOff = 0; + int callsOn = 0; + + ExpectIntEQ(test_wolfSSL_read_ahead_one(0, &callsOff), TEST_SUCCESS); + ExpectIntEQ(test_wolfSSL_read_ahead_one(1, &callsOn), TEST_SUCCESS); + + /* Without read-ahead the record header and body are fetched with separate + * recv() calls; with read-ahead the whole record arrives in a single + * recv(), so the body read issues no syscall. */ + ExpectIntGE(callsOff, 2); + ExpectIntEQ(callsOn, 1); + ExpectIntGT(callsOff, callsOn); + + return EXPECT_RESULT(); +} + +/* When read-ahead coalesces an application data record together with a + * following record (here a close_notify), the app data must still be delivered + * before the connection-close is reported, and the buffered record must be + * visible to wolfSSL_has_pending(). */ +static int test_wolfSSL_read_ahead_coalesced(void) +{ + EXPECT_DECLS; + WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL; + WOLFSSL *ssl_c = NULL, *ssl_s = NULL; + struct test_memio_ctx test_ctx; + struct test_read_ahead_ctx recv_ctx; + char msg[] = "hello wolfssl read ahead"; + char reply[64]; + + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&recv_ctx, 0, sizeof(recv_ctx)); + recv_ctx.memio = &test_ctx; + recv_ctx.coalesce = 1; + + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_2_client_method, wolfTLSv1_2_server_method), 0); + ExpectIntEQ(wolfSSL_set_read_ahead(ssl_c, 1), WOLFSSL_SUCCESS); + ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 10, NULL), 0); + + /* Queue two records back-to-back: app data, then close_notify. */ + ExpectIntEQ(wolfSSL_write(ssl_s, msg, (int)sizeof(msg)), (int)sizeof(msg)); + wolfSSL_shutdown(ssl_s); + + wolfSSL_SSLSetIORecv(ssl_c, test_read_ahead_recv_cb); + wolfSSL_SetIOReadCtx(ssl_c, &recv_ctx); + + /* First read returns the app data, not the connection close. */ + XMEMSET(reply, 0, sizeof(reply)); + ExpectIntEQ(wolfSSL_read(ssl_c, reply, (int)sizeof(reply)), (int)sizeof(msg)); + ExpectIntEQ(XMEMCMP(reply, msg, sizeof(msg)), 0); + + /* The close_notify record was pulled in by read-ahead and is still + * buffered, so has_pending must report it even though no socket read + * happened. */ + ExpectIntEQ(wolfSSL_has_pending(ssl_c), 1); + + /* Next read consumes the buffered close_notify and reports the close. */ + ExpectIntEQ(wolfSSL_read(ssl_c, reply, (int)sizeof(reply)), 0); + ExpectIntEQ(wolfSSL_get_error(ssl_c, 0), WOLFSSL_ERROR_ZERO_RETURN); + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + + return EXPECT_RESULT(); +} + +/* Max per-record payload the helper can drive; actual record size is a + * parameter. Kept so several records stay under the 64 KB memio buffer. */ +#define TEST_READ_AHEAD_MAX_REC 16000 + +/* Drive numRec records of recSz bytes through the client with a configurable + * read-ahead window. Reports how many recv() callbacks it took to drain them + * (recvCalls) and the input buffer's retained size afterwards (bufSize). + * bufLen == 0 leaves the one-record default in place. */ +static int test_wolfSSL_read_ahead_buffer_len_run(word32 bufLen, int recSz, + int numRec, int *recvCalls, word32 *bufSize) +{ + EXPECT_DECLS; + WOLFSSL_CTX *ctx_c = NULL, *ctx_s = NULL; + WOLFSSL *ssl_c = NULL, *ssl_s = NULL; + struct test_memio_ctx test_ctx; + struct test_read_ahead_ctx recv_ctx; + byte msg[TEST_READ_AHEAD_MAX_REC]; + byte reply[TEST_READ_AHEAD_MAX_REC]; + int i; + + XMEMSET(msg, 'A', sizeof(msg)); + XMEMSET(&test_ctx, 0, sizeof(test_ctx)); + XMEMSET(&recv_ctx, 0, sizeof(recv_ctx)); + recv_ctx.memio = &test_ctx; + recv_ctx.coalesce = 1; + + ExpectIntEQ(test_memio_setup(&test_ctx, &ctx_c, &ctx_s, &ssl_c, &ssl_s, + wolfTLSv1_2_client_method, wolfTLSv1_2_server_method), 0); + ExpectIntEQ(wolfSSL_set_read_ahead(ssl_c, 1), WOLFSSL_SUCCESS); + if (bufLen > 0) { + ExpectIntEQ(wolfSSL_set_default_read_buffer_len(ssl_c, bufLen), + WOLFSSL_SUCCESS); + /* Values within range are stored as given (sub-record sizes are + * honoured, not rounded up to a record). Clamping of oversized values + * is covered by test_wolfSSL_read_ahead_ctx_inherit. */ + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(ssl_c), (long)bufLen); + } + ExpectIntEQ(test_memio_do_handshake(ssl_c, ssl_s, 10, NULL), 0); + + /* Server queues several records back-to-back before the client reads. */ + for (i = 0; i < numRec; i++) { + ExpectIntEQ(wolfSSL_write(ssl_s, msg, recSz), recSz); + } + + wolfSSL_SSLSetIORecv(ssl_c, test_read_ahead_recv_cb); + wolfSSL_SetIOReadCtx(ssl_c, &recv_ctx); + + for (i = 0; i < numRec; i++) { + XMEMSET(reply, 0, sizeof(reply)); + ExpectIntEQ(wolfSSL_read(ssl_c, reply, recSz), recSz); + ExpectIntEQ(XMEMCMP(reply, msg, (size_t)recSz), 0); + } + + if (recvCalls != NULL) + *recvCalls = recv_ctx.count; + /* Capture the retained input buffer size while the session is still alive; + * read-ahead keeps the grown buffer rather than shrinking per record. */ + if (bufSize != NULL && ssl_c != NULL) + *bufSize = ssl_c->buffers.inputBuffer.bufferSize; + + wolfSSL_free(ssl_c); + wolfSSL_free(ssl_s); + wolfSSL_CTX_free(ctx_c); + wolfSSL_CTX_free(ctx_s); + + return EXPECT_RESULT(); +} + +static int test_wolfSSL_read_ahead_buffer_len(void) +{ + EXPECT_DECLS; + int callsDefault = 0; + int callsLarge = 0; + word32 bufDefault = 0; + word32 bufSmall = 0; + word32 bufShrunk = 0; + + /* Coalescing: with three near-max records, a window spanning all of them + * pulls every record in fewer recv() callbacks than the one-record default + * (which can only hold one such record at a time). */ + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(0, TEST_READ_AHEAD_MAX_REC, + 3, &callsDefault, NULL), TEST_SUCCESS); + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run( + 3 * TEST_READ_AHEAD_MAX_REC + 4096, TEST_READ_AHEAD_MAX_REC, 3, + &callsLarge, NULL), TEST_SUCCESS); + ExpectIntGT(callsDefault, callsLarge); + + /* Footprint: with small records, a sub-record window is honoured (not + * clamped up to a full record), so the retained input buffer is much + * smaller than the one-record default while data is still read correctly. */ + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(0, 1024, 4, NULL, + &bufDefault), TEST_SUCCESS); + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(2048, 1024, 4, NULL, + &bufSmall), TEST_SUCCESS); + /* The footprint comparison only holds when the input buffer actually goes + * dynamic, i.e. the static buffer is smaller than the configured window. + * With LARGE_STATIC_BUFFERS (or a large user STATIC_BUFFER_LEN) the buffer + * never grows and both runs retain STATIC_BUFFER_LEN, so skip the size + * checks there; the runs above still verify the data is read correctly. + * STATIC_BUFFER_LEN resolves to an enum constant, so this is a runtime (not + * preprocessor) guard that the compiler folds away. */ + if (STATIC_BUFFER_LEN < 2048) { + ExpectIntGT(bufDefault, bufSmall); + /* The 2 KB window retains roughly a 2 KB buffer, below one TLS record. */ + ExpectIntLE(bufSmall, 4096); + } + + /* Shrink-back: an 8 KB record exceeds the 2 KB window, growing the buffer to + * receive it, but read-ahead reallocates back down to the window afterwards, + * so the retained buffer tracks the window (~2 KB) rather than the record. */ + ExpectIntEQ(test_wolfSSL_read_ahead_buffer_len_run(2048, 8000, 2, NULL, + &bufShrunk), TEST_SUCCESS); + if (STATIC_BUFFER_LEN < 2048) { + ExpectIntLE(bufShrunk, 4096); + } + + return EXPECT_RESULT(); +} + +/* Cover the CTX-level read-buffer-len setter/getter, CTX->SSL inheritance via + * SetSSL_CTX(), NULL-argument handling for all four accessors, and the + * oversized-window clamp. */ +static int test_wolfSSL_read_ahead_ctx_inherit(void) +{ + EXPECT_DECLS; + WOLFSSL_CTX* ctx_c = NULL; + WOLFSSL* ssl_c = NULL; + + /* NULL arguments must fail cleanly, not crash. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(NULL, 4096), + WOLFSSL_FAILURE); + ExpectIntEQ(wolfSSL_set_default_read_buffer_len(NULL, 4096), + WOLFSSL_FAILURE); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(NULL), WOLFSSL_FAILURE); + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(NULL), WOLFSSL_FAILURE); + + ExpectNotNull(ctx_c = wolfSSL_CTX_new(wolfTLSv1_2_client_method())); + + /* A fresh context carries the one-record default window, not 0. */ + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), + (long)WOLFSSL_READ_AHEAD_SZ); + + /* CTX-level setter/getter round-trip. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, 8192), + WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), 8192); + + /* 0 resets to the one-record default rather than storing 0. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, 0), + WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), + (long)WOLFSSL_READ_AHEAD_SZ); + + /* A window above the maximum is clamped to WOLFSSL_MAX_READ_AHEAD_SZ, not + * wrapped around by the size_t->word32 store. */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, + (size_t)WOLFSSL_MAX_READ_AHEAD_SZ + 4096), WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_CTX_get_default_read_buffer_len(ctx_c), + (long)WOLFSSL_MAX_READ_AHEAD_SZ); + + /* Set a plain value and confirm a freshly created WOLFSSL inherits it + * through SetSSL_CTX(). */ + ExpectIntEQ(wolfSSL_CTX_set_default_read_buffer_len(ctx_c, 8192), + WOLFSSL_SUCCESS); + ExpectNotNull(ssl_c = wolfSSL_new(ctx_c)); + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(ssl_c), 8192); + + /* The session-level setter overrides the inherited value. */ + ExpectIntEQ(wolfSSL_set_default_read_buffer_len(ssl_c, 2048), + WOLFSSL_SUCCESS); + ExpectIntEQ(wolfSSL_get_default_read_buffer_len(ssl_c), 2048); + + wolfSSL_free(ssl_c); + wolfSSL_CTX_free(ctx_c); + + return EXPECT_RESULT(); +} + +#undef TEST_READ_AHEAD_MAX_REC +#else +static int test_wolfSSL_read_ahead(void) +{ + return TEST_SKIPPED; +} +static int test_wolfSSL_read_ahead_coalesced(void) +{ + return TEST_SKIPPED; +} +static int test_wolfSSL_read_ahead_buffer_len(void) +{ + return TEST_SKIPPED; +} +static int test_wolfSSL_read_ahead_ctx_inherit(void) +{ + return TEST_SKIPPED; +} +#endif + static int test_wolfSSL_inject(void) { EXPECT_DECLS; @@ -37716,6 +38064,10 @@ TEST_CASE testCases[] = { TEST_DECL(test_wolfSSL_SendUserCanceled), TEST_DECL(test_wolfSSL_SSLDisableRead), TEST_DECL(test_wolfSSL_shutdown_pending_data_uaf), + TEST_DECL(test_wolfSSL_read_ahead), + TEST_DECL(test_wolfSSL_read_ahead_coalesced), + TEST_DECL(test_wolfSSL_read_ahead_buffer_len), + TEST_DECL(test_wolfSSL_read_ahead_ctx_inherit), TEST_DECL(test_wolfSSL_inject), TEST_DECL(test_ocsp_status_callback), TEST_DECL(test_ocsp_basic_verify), diff --git a/wolfssl/internal.h b/wolfssl/internal.h index d54496e0f9c..94ec917f6ff 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -2375,6 +2375,29 @@ enum { #define STATIC_BUFFER_LEN RECORD_HEADER_SZ #endif +/* Default read-ahead window: when read-ahead is enabled the record header read + * requests up to a full record's worth of data in a single recv() so the body + * (and possibly following records) can be pulled in without a second syscall. + * Sized to one maximum TLS record (MAX_RECORD_SIZE, not the buffer-sizing + * RECORD_SIZE which may be small) so the whole record is captured. Defined + * unconditionally so the setters and CTX init can reference it as the default + * window even when read-ahead I/O is not built. */ +#ifndef WOLFSSL_READ_AHEAD_SZ +#define WOLFSSL_READ_AHEAD_SZ (RECORD_HEADER_SZ + MAX_RECORD_SIZE + \ + COMP_EXTRA + MTU_EXTRA + MAX_MSG_EXTRA) +#endif + +/* Upper bound for a caller-configured read-ahead window + * (wolfSSL_CTX/SSL_set_default_read_buffer_len()). The window feeds signed int + * arithmetic in GetInputData_ex(); bounding it well below INT_MAX ensures a + * large caller-supplied size can never overflow that arithmetic to a negative + * value (which would skip GrowInputBuffer() and drive an oversized recv()). + * 16 MB is far above any realistic coalescing window. Defined unconditionally + * so the setters can clamp even when read-ahead I/O is not built. */ +#ifndef WOLFSSL_MAX_READ_AHEAD_SZ +#define WOLFSSL_MAX_READ_AHEAD_SZ (16 * 1024 * 1024) +#endif + typedef struct { ALIGN16 byte staticBuffer[STATIC_BUFFER_LEN]; byte* buffer; /* place holder for static or dynamic buffer */ @@ -4225,8 +4248,16 @@ struct WOLFSSL_CTX { WOLFSSL_X509_STORE x509_store; /* points to ctx->cm */ WOLFSSL_X509_STORE* x509_store_pt; /* take ownership of external store */ #endif -#if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) || defined(WOLFSSL_WPAS_SMALL) +#if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) || \ + defined(WOLFSSL_WPAS_SMALL) || defined(WOLFSSL_TLS_READ_AHEAD) byte readAhead; +#endif +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) + /* Read-ahead coalescing buffer size. 0 = use one record (default). See + * wolfSSL_CTX_set_default_read_buffer_len(). */ + word32 readAheadSz; +#endif +#if defined(OPENSSL_EXTRA) || defined(HAVE_WEBSERVER) || defined(WOLFSSL_WPAS_SMALL) void* userPRFArg; /* passed to prf callback */ #endif #ifdef HAVE_EX_DATA @@ -6282,8 +6313,12 @@ struct WOLFSSL { defined(OPENSSL_ALL) unsigned long peerVerifyRet; #endif -#ifdef OPENSSL_EXTRA +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) byte readAhead; + /* Read-ahead coalescing buffer size; 0 = one record (default). */ + word32 readAheadSz; +#endif +#ifdef OPENSSL_EXTRA #ifdef HAVE_PK_CALLBACKS void* loggingCtx; /* logging callback argument */ #endif diff --git a/wolfssl/openssl/ssl.h b/wolfssl/openssl/ssl.h index cdc8ea64adb..70028632cbb 100644 --- a/wolfssl/openssl/ssl.h +++ b/wolfssl/openssl/ssl.h @@ -1438,6 +1438,8 @@ typedef WOLFSSL_SRTP_PROTECTION_PROFILE SRTP_PROTECTION_PROFILE; #define SSL_set_read_ahead wolfSSL_set_read_ahead #define SSL_CTX_get_read_ahead wolfSSL_CTX_get_read_ahead #define SSL_CTX_set_read_ahead wolfSSL_CTX_set_read_ahead +#define SSL_CTX_set_default_read_buffer_len wolfSSL_CTX_set_default_read_buffer_len +#define SSL_set_default_read_buffer_len wolfSSL_set_default_read_buffer_len #define SSL_CTX_set_tlsext_status_arg wolfSSL_CTX_set_tlsext_status_arg #define SSL_CTX_set_tlsext_opaque_prf_input_callback_arg \ wolfSSL_CTX_set_tlsext_opaque_prf_input_callback_arg diff --git a/wolfssl/ssl.h b/wolfssl/ssl.h index b5a0fb4a07a..0b0ab164d52 100644 --- a/wolfssl/ssl.h +++ b/wolfssl/ssl.h @@ -2718,10 +2718,6 @@ WOLFSSL_API long wolfSSL_CTX_sess_set_cache_size(WOLFSSL_CTX* ctx, long sz); WOLFSSL_API long wolfSSL_CTX_sess_get_cache_size(WOLFSSL_CTX* ctx); WOLFSSL_API long wolfSSL_CTX_get_session_cache_mode(WOLFSSL_CTX* ctx); -WOLFSSL_API int wolfSSL_get_read_ahead(const WOLFSSL* ssl); -WOLFSSL_API int wolfSSL_set_read_ahead(WOLFSSL* ssl, int v); -WOLFSSL_API int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); -WOLFSSL_API int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v); WOLFSSL_API long wolfSSL_CTX_set_tlsext_opaque_prf_input_callback_arg( WOLFSSL_CTX* ctx, void* arg); WOLFSSL_API int wolfSSL_CTX_add_client_CA(WOLFSSL_CTX* ctx, WOLFSSL_X509* x509); @@ -2754,6 +2750,23 @@ WOLFSSL_API long wolfSSL_get_verify_result(const WOLFSSL *ssl); WOLFSSL_API void* wolfSSL_get_app_data( const WOLFSSL *ssl); #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */ +/* Read-ahead control is part of the OpenSSL compatibility layer, and is also + * exposed when TLS read-ahead support is built without that layer. Guard must + * match the definitions in ssl.c (which use the readAhead/readAheadSz struct + * members available only under these macros), so it deliberately excludes + * OPENSSL_EXTRA_X509_SMALL. */ +#if defined(OPENSSL_EXTRA) || defined(WOLFSSL_TLS_READ_AHEAD) +WOLFSSL_API int wolfSSL_get_read_ahead(const WOLFSSL* ssl); +WOLFSSL_API int wolfSSL_set_read_ahead(WOLFSSL* ssl, int v); +WOLFSSL_API int wolfSSL_CTX_get_read_ahead(WOLFSSL_CTX* ctx); +WOLFSSL_API int wolfSSL_CTX_set_read_ahead(WOLFSSL_CTX* ctx, int v); +WOLFSSL_API int wolfSSL_CTX_set_default_read_buffer_len(WOLFSSL_CTX* ctx, + size_t len); +WOLFSSL_API int wolfSSL_set_default_read_buffer_len(WOLFSSL* ssl, size_t len); +WOLFSSL_API long wolfSSL_CTX_get_default_read_buffer_len(WOLFSSL_CTX* ctx); +WOLFSSL_API long wolfSSL_get_default_read_buffer_len(const WOLFSSL* ssl); +#endif + #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL) || \ defined(HAVE_WEBSERVER) || defined(HAVE_MEMCACHED)