Reduce H2 header allocations

include/proxy/ProxyTransaction.h

@@ -61,6 +61,12 @@ class ProxyTransaction : public VConnection
   virtual bool expect_receive_trailer() const;
   virtual void set_expect_receive_trailer();
 
+  virtual bool           supports_direct_header_passing() const;
+  virtual bool           is_parsed_receive_header_ready() const;
+  virtual const HTTPHdr *parsed_receive_header() const;
+
+  virtual bool has_pending_send_header() const;
+
   // Implement VConnection interface.
   VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override;
   VIO *do_io_write(Continuation *c = nullptr, int64_t nbytes = INT64_MAX, IOBufferReader *buf = nullptr,
@@ -320,6 +326,30 @@ ProxyTransaction::cancel_active_timeout()
   }
 }
 
+inline bool
+ProxyTransaction::supports_direct_header_passing() const
+{
+  return false;
+}
+
+inline bool
+ProxyTransaction::is_parsed_receive_header_ready() const
+{
+  return false;
+}
+
+inline const HTTPHdr *
+ProxyTransaction::parsed_receive_header() const
+{
+  return nullptr;
+}
+
+inline bool
+ProxyTransaction::has_pending_send_header() const
+{
+  return false;
+}
+
 // See if we need to schedule on the primary thread for the transaction or change the thread that is associated with the VC.
 // If we reschedule, the scheduled action is returned.  Otherwise, NULL is returned
 inline Action *

include/proxy/hdrs/HdrHeap.h

@@ -175,6 +175,7 @@ struct StrHeapDesc {
 class HdrHeap
 {
 public:
+  // Also sizes HTTP/2's on-stack HEADERS encode buffer (2*this).
   static constexpr int DEFAULT_SIZE = 2048;
 
   void init();

include/proxy/hdrs/URL.h

@@ -215,6 +215,9 @@ ParseResult url_parse_http(HdrHeap *heap, URLImpl *url, const char **start, cons
                            bool verify_host_characters);
 ParseResult url_parse_http_regex(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings);
 
+// strict_uri_parsing 0 = no check; 1/2 apply the same test url_parse() does.
+bool url_is_uri_compliant(int strict_uri_parsing, std::string_view value);
+
 char *url_unescapify(Arena *arena, const char *str, int length);
 
 void unescape_str(char *&buf, char *buf_e, const char *&str, const char *str_e, int &state);

include/proxy/http/HttpSM.h

@@ -610,6 +610,12 @@ class HttpSM : public Continuation, public PluginUserArgs<TS_USER_ARGS_TXN>
   IOBufferReader *_netvc_reader      = nullptr;
   MIOBuffer      *_netvc_read_buffer = nullptr;
 
+  bool _client_response_header_is_ready = false;
+  bool _server_request_header_is_ready  = false;
+
+  // Direct-passed headers bypass the tunnel, so client_response_hdr_bytes stays 0.
+  int _direct_response_hdr_bytes = 0;
+
   void kill_this();
   void update_stats();
   void transform_cleanup(TSHttpHookID hook, HttpTransformInfo *info);
@@ -626,6 +632,23 @@ class HttpSM : public Continuation, public PluginUserArgs<TS_USER_ARGS_TXN>
   int     client_transaction_priority_weight() const;
   int     client_transaction_priority_dependence() const;
 
+  HTTPHdr *get_client_response_header();
+  HTTPHdr *get_server_request_header();
+
+  // For logging/SDK: client_response_hdr_bytes counts only what the tunnel wrote.
+  int
+  reported_client_response_hdr_bytes() const
+  {
+    return client_response_hdr_bytes > 0 ? client_response_hdr_bytes : _direct_response_hdr_bytes;
+  }
+
+  void
+  clear_pending_send_header()
+  {
+    _client_response_header_is_ready = false;
+    _server_request_header_is_ready  = false;
+  }
+
   ink_hrtime get_server_inactivity_timeout();
   ink_hrtime get_server_active_timeout();
   ink_hrtime get_server_connect_timeout();
@@ -713,13 +736,30 @@ HttpSM::get_cache_sm()
 inline int
 HttpSM::write_response_header_into_buffer(HTTPHdr *h, MIOBuffer *b)
 {
-  if (t_state.client_info.http_version == HTTPVersion(0, 9)) {
+  if (_ua.get_txn()->supports_direct_header_passing()) {
+    // Nothing lands in the buffer, so 0 keeps the tunnel's byte math honest.
+    _client_response_header_is_ready = true;
+    _direct_response_hdr_bytes       = h->length_get();
+    return 0;
+  } else if (t_state.client_info.http_version == HTTPVersion(0, 9)) {
     return 0;
   } else {
     return write_header_into_buffer(h, b);
   }
 }
 
+inline HTTPHdr *
+HttpSM::get_client_response_header()
+{
+  return _client_response_header_is_ready ? &t_state.hdr_info.client_response : nullptr;
+}
+
+inline HTTPHdr *
+HttpSM::get_server_request_header()
+{
+  return _server_request_header_is_ready ? &t_state.hdr_info.server_request : nullptr;
+}
+
 inline int
 HttpSM::find_server_buffer_size()
 {

include/proxy/http2/Http2Stream.h

@@ -82,7 +82,14 @@ class Http2Stream : public ProxyTransaction
   bool expect_receive_trailer() const override;
   void set_expect_receive_trailer() override;
 
+  bool           supports_direct_header_passing() const override;
+  bool           is_parsed_receive_header_ready() const override;
+  const HTTPHdr *parsed_receive_header() const override;
+  bool           has_pending_send_header() const override;
+
   Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size, uint32_t header_field_max_size);
+  Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size, uint32_t header_field_max_size,
+                                      const uint8_t *block, uint32_t block_len);
   void           send_headers(Http2ConnectionState &cstate);
   void           initiating_close();
   bool           is_outbound_connection() const;
@@ -220,6 +227,7 @@ class Http2Stream : public ProxyTransaction
   int              _sent_request_method{-1};
 
   HTTPHdr _receive_header;
+  bool    _is_parsed_receive_header_ready = false;
 #if TS_USE_MALLOC_ALLOCATOR
   MIOBuffer _receive_buffer{BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(4096)};
 #else

src/api/InkAPI.cc

@@ -5565,7 +5565,7 @@ TSHttpTxnClientRespHdrBytesGet(TSHttpTxn txnp)
   sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS);
 
   HttpSM *sm = reinterpret_cast<HttpSM *>(txnp);
-  return sm->client_response_hdr_bytes;
+  return sm->reported_client_response_hdr_bytes();
 }
 
 int64_t

src/proxy/hdrs/URL.cc

@@ -1212,6 +1212,22 @@ url_is_mostly_compliant(const char *start, const char *end)
 } // namespace UrlImpl
 using namespace UrlImpl;
 
+bool
+url_is_uri_compliant(int strict_uri_parsing, std::string_view value)
+{
+  const char *start = value.data();
+  const char *end   = start + value.length();
+
+  switch (strict_uri_parsing) {
+  case 1:
+    return url_is_strictly_compliant(start, end);
+  case 2:
+    return url_is_mostly_compliant(start, end);
+  default:
+    return true;
+  }
+}
+
 ParseResult
 url_parse(HdrHeap *heap, URLImpl *url, const char **start, const char *end, bool copy_strings_p, int strict_uri_parsing,
           bool verify_host_characters)

src/proxy/hdrs/VersionConverter.cc

@@ -201,7 +201,15 @@ VersionConverter::_convert_req_from_2_to_1(HTTPHdr &header) const
   if (MIMEField *field = header.field_find(PSEUDO_HEADER_AUTHORITY);
       field != nullptr && field->value_is_valid(is_control_BIT | is_ws_BIT)) {
     auto authority{field->value_get()};
-    header.m_http->u.req.m_url_impl->set_host(header.m_heap, authority, true);
+
+    // Match url_parse_http(): require full consumption, else a stray '/', '?' or '#' is dropped.
+    const char *astart = authority.data();
+    const char *aend   = authority.data() + authority.length();
+
+    if (url_parse_internet(header.m_heap, header.m_http->u.req.m_url_impl, &astart, aend, true, true) != ParseResult::DONE ||
+        astart != aend) {
+      return ParseResult::ERROR;
+    }
 
     if (!is_connect_method) {
       MIMEField *host = header.field_find(static_cast<std::string_view>(MIME_FIELD_HOST));
@@ -229,14 +237,26 @@ VersionConverter::_convert_req_from_2_to_1(HTTPHdr &header) const
     // :path
     if (MIMEField *field = header.field_find(PSEUDO_HEADER_PATH);
         field != nullptr && field->value_is_valid(is_control_BIT | is_ws_BIT)) {
-      auto path{field->value_get()};
+      auto  path{field->value_get()};
+      auto *url = header.m_http->u.req.m_url_impl;
+
+      // Split as url_parse_http() would, so cache keys and remap see the same fields.
+      if (auto hpos = path.find('#'); hpos != std::string_view::npos) {
+        url->set_fragment(header.m_heap, path.substr(hpos + 1), true);
+        path = path.substr(0, hpos);
+      }
+
+      if (auto qpos = path.find('?'); qpos != std::string_view::npos) {
+        url->set_query(header.m_heap, path.substr(qpos + 1), true);
+        path = path.substr(0, qpos);
+      }
 
-      // cut first '/' if there, because `url_print()` add '/' before printing path
-      if (path.starts_with("/"sv)) {
+      // url_parse_http() strips every leading '/'; url_print() re-adds exactly one.
+      while (path.starts_with("/"sv)) {
         path.remove_prefix(1);
       }
 
-      header.m_http->u.req.m_url_impl->set_path(header.m_heap, path, true);
+      url->set_path(header.m_heap, path, true);
 
       header.field_delete(field);
     } else {

src/proxy/http/HttpSM.cc

@@ -620,10 +620,25 @@ HttpSM::state_read_client_request_header(int event, void *data)
   // tokenize header //
   /////////////////////
 
-  ParseResult state = t_state.hdr_info.client_request.parse_req(&http_parser, _ua.get_txn()->get_remote_reader(), &bytes_used,
-                                                                _ua.get_entry()->eos, t_state.http_config_param->strict_uri_parsing,
-                                                                t_state.http_config_param->http_request_line_max_size,
-                                                                t_state.http_config_param->http_hdr_field_max_size);
+  ParseResult       state;
+  ProxyTransaction *ua_txn               = _ua.get_txn();
+  bool              direct_header_passed = false;
+
+  if (ua_txn->supports_direct_header_passing() && ua_txn->is_parsed_receive_header_ready()) {
+    // UA_FIRST_READ never fires here: the read buffer stays empty.
+    if (milestones[TS_MILESTONE_UA_FIRST_READ] == 0) {
+      ATS_PROBE1(milestone_ua_first_read, sm_id);
+      milestones[TS_MILESTONE_UA_FIRST_READ] = ink_get_hrtime();
+    }
+    t_state.hdr_info.client_request.copy(ua_txn->parsed_receive_header());
+    bytes_used           = t_state.hdr_info.client_request.length_get();
+    state                = ParseResult::DONE;
+    direct_header_passed = true;
+  } else {
+    state = t_state.hdr_info.client_request.parse_req(
+      &http_parser, ua_txn->get_remote_reader(), &bytes_used, _ua.get_entry()->eos, t_state.http_config_param->strict_uri_parsing,
+      t_state.http_config_param->http_request_line_max_size, t_state.http_config_param->http_hdr_field_max_size);
+  }
 
   client_request_hdr_bytes += bytes_used;
 
@@ -705,7 +720,8 @@ HttpSM::state_read_client_request_header(int event, void *data)
     // Disable further I/O on the client
     _ua.get_entry()->read_vio->nbytes = _ua.get_entry()->read_vio->ndone;
 
-    (bytes_used > t_state.http_config_param->http_request_line_max_size) ?
+    // bytes_used is the whole header set on the direct path, not a request-line length.
+    (!direct_header_passed && bytes_used > t_state.http_config_param->http_request_line_max_size) ?
       t_state.http_return_code = HTTPStatus::REQUEST_URI_TOO_LONG :
       t_state.http_return_code = HTTPStatus::NONE;
 
@@ -4014,7 +4030,9 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p)
   case VC_EVENT_INACTIVITY_TIMEOUT:
   case VC_EVENT_ACTIVE_TIMEOUT:
   case HTTP_TUNNEL_EVENT_PARSE_ERROR:
-    if (client_response_hdr_bytes == 0) {
+    // Not the raw counter: it stays 0 when the header bypassed the tunnel, which would
+    // synthesize an error over a response the client already got.
+    if (reported_client_response_hdr_bytes() == 0) {
       p->handler_state = static_cast<int>(HttpSmPost_t::UA_FAIL);
       set_ua_abort(HttpTransact::ABORTED, event);
 
@@ -7011,7 +7029,13 @@ HttpSM::setup_server_send_request()
   // We need a reader so bytes don't fall off the end of
   //  the buffer
   IOBufferReader *buf_start = server_entry->write_buffer->alloc_reader();
-  server_request_hdr_bytes = hdr_length = write_header_into_buffer(&t_state.hdr_info.server_request, server_entry->write_buffer);
+
+  if (server_txn->supports_direct_header_passing()) {
+    _server_request_header_is_ready = true;
+    server_request_hdr_bytes = hdr_length = 0;
+  } else {
+    server_request_hdr_bytes = hdr_length = write_header_into_buffer(&t_state.hdr_info.server_request, server_entry->write_buffer);
+  }
 
   // the plugin decided to append a message to the request
   if (t_state.api_server_request_body_set) {
@@ -7098,7 +7122,7 @@ HttpSM::setup_cache_read_transfer()
   // Now dump the header into the buffer
   ink_assert(t_state.hdr_info.client_response.status_get() != HTTPStatus::NOT_MODIFIED);
   client_response_hdr_bytes = hdr_size = write_response_header_into_buffer(&t_state.hdr_info.client_response, buf);
-  cache_response_hdr_bytes             = client_response_hdr_bytes;
+  cache_response_hdr_bytes             = reported_client_response_hdr_bytes();
 
   HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler);
 
@@ -8001,7 +8025,7 @@ HttpSM::update_stats()
 
   HttpTransact::update_size_and_time_stats(
     &t_state, total_time, ua_write_time, os_read_time, client_request_hdr_bytes, client_request_body_bytes,
-    client_response_hdr_bytes, client_response_body_bytes, server_request_hdr_bytes, server_request_body_bytes,
+    reported_client_response_hdr_bytes(), client_response_body_bytes, server_request_hdr_bytes, server_request_body_bytes,
     server_response_hdr_bytes, server_response_body_bytes, pushed_response_hdr_bytes, pushed_response_body_bytes, milestones);
   /*
       if (is_action_tag_set("http_handler_times")) {

src/proxy/http/HttpTunnel.cc

@@ -1211,7 +1211,12 @@ HttpTunnel::producer_run(HttpTunnelProducer *p)
       }
     }
 
-    if (c_write == 0) {
+    // A bodyless HTTP/2 response (204/304/HEAD) still owes a HEADERS frame, and the stream
+    // self-signals WRITE_COMPLETE. Match by identity: plugin agents are HTTP_CLIENT too.
+    ProxyTransaction *ua_txn       = sm->get_ua_txn();
+    bool const        flush_header = c_write == 0 && ua_txn != nullptr && c->vc == ua_txn && ua_txn->has_pending_send_header();
+
+    if (c_write == 0 && !flush_header) {
       // Nothing to do, call back the cleanup handlers
       c->write_vio = nullptr;
       consumer_handler(VC_EVENT_WRITE_COMPLETE, c);
@@ -1234,10 +1239,9 @@ HttpTunnel::producer_run(HttpTunnelProducer *p)
       Dbg(dbg_ctl_http_tunnel, "Start write vio %" PRId64 " bytes", c_write);
       // Start the writes now that we know we will consume all the initial data
       c->write_vio = c->vc->do_io_write(this, c_write, c->buffer_reader);
-      ink_assert(c_write > 0);
       if (c->write_vio == nullptr) {
         consumer_handler(VC_EVENT_ERROR, c);
-      } else if (c->write_vio->ntodo() == 0 && c->alive) {
+      } else if (!flush_header && c->write_vio->ntodo() == 0 && c->alive) {
         consumer_handler(VC_EVENT_WRITE_COMPLETE, c);
       }
     }

src/proxy/http2/Http2ConnectionState.cc

@@ -455,11 +455,22 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame)
   if (stream->trailing_header_is_possible()) {
     // Don't leak the header_blocks from the initial, non-trailing headers.
     ats_free(stream->header_blocks);
+    stream->header_blocks = nullptr;
   }
-  stream->header_blocks = static_cast<uint8_t *>(ats_malloc(header_block_fragment_length));
-  frame.reader()->memcpy(stream->header_blocks, header_block_fragment_length, header_block_fragment_offset);
 
-  if (frame.header().flags & HTTP2_FLAGS_HEADERS_END_HEADERS) {
+  // In-place decode avoids the per-request malloc+memcpy+free; needs one contiguous block.
+  bool const     end_headers   = frame.header().flags & HTTP2_FLAGS_HEADERS_END_HEADERS;
+  uint8_t const *inplace_block = nullptr;
+
+  if (end_headers && frame.reader()->block_read_avail() >=
+                       static_cast<int64_t>(header_block_fragment_offset) + static_cast<int64_t>(header_block_fragment_length)) {
+    inplace_block = reinterpret_cast<uint8_t const *>(frame.reader()->start()) + header_block_fragment_offset;
+  } else {
+    stream->header_blocks = static_cast<uint8_t *>(ats_malloc(header_block_fragment_length));
+    frame.reader()->memcpy(stream->header_blocks, header_block_fragment_length, header_block_fragment_offset);
+  }
+
+  if (end_headers) {
     // NOTE: If there are END_HEADERS flag, decode stored Header Blocks.
     if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, frame.header().flags)) {
       return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR,
@@ -478,8 +489,13 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame)
     } else {
       stream->mark_milestone(Http2StreamMilestone::START_DECODE_HEADERS);
     }
-    Http2ErrorCode result = stream->decode_header_blocks(
-      *this->local_hpack_handle, this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE), _header_field_max_size);
+    Http2ErrorCode result = inplace_block ?
+                              stream->decode_header_blocks(*this->local_hpack_handle,
+                                                           this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE),
+                                                           _header_field_max_size, inplace_block, header_block_fragment_length) :
+                              stream->decode_header_blocks(*this->local_hpack_handle,
+                                                           this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE),
+                                                           _header_field_max_size);
 
     // If this was an outbound connection and the state was already closed, just clear the
     // headers after processing.  We just processed the header blocks to keep the dynamic table in
@@ -2514,9 +2530,10 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream)
     http2_convert_header_from_1_1_to_2(send_hdr);
   }
 
-  uint32_t        buf_len = send_hdr->length_get() * 2; // Make it double just in case
-  ts::LocalBuffer local_buffer(buf_len);
-  uint8_t        *buf = local_buffer.data();
+  uint32_t buf_len = send_hdr->length_get() * 2; // Make it double just in case
+  static_assert(HdrHeap::DEFAULT_SIZE * 2 <= 8192, "keep HEADERS encode stack buffer within the event-thread stack budget");
+  ts::LocalBuffer<uint8_t, HdrHeap::DEFAULT_SIZE * 2> local_buffer(buf_len);
+  uint8_t                                            *buf = local_buffer.data();
 
   stream->mark_milestone(Http2StreamMilestone::START_ENCODE_HEADERS);
   Http2ErrorCode result = http2_encode_header_blocks(send_hdr, buf, buf_len, &header_blocks_size, *(this->peer_hpack_handle),