diff --git a/doc/admin-guide/plugins/slice.en.rst b/doc/admin-guide/plugins/slice.en.rst index 40d606129c5..f5e8b411d15 100644 --- a/doc/admin-guide/plugins/slice.en.rst +++ b/doc/admin-guide/plugins/slice.en.rst @@ -186,7 +186,21 @@ The slice plugin supports the following options:: that causes `cache_range_requests` to be bypassed in such requests, and allow ATS to handle those range requests internally. - + --purge-probe-blocks= (optional) + Default is 8 + How many consecutive uncached slice blocks a PURGE walks, before any block + has reported the object's extent, until it concludes that nothing about the + object is cached. May be overridden per request with the header named by + ``--purge-probe-header``. See `Purge Requests`_. + -q for short + + --purge-probe-header= (optional) + Default is X-Slice-Purge-Probe + Name of the request header a PURGE may use to override + ``--purge-probe-blocks`` for that request. A malformed value is ignored + in favour of the configured default. Slice strips this header from the + block requests it issues. + -H for short Examples:: @@ -338,17 +352,70 @@ requests may end up being served from temporally different assets. Purge Requests -------------- -The slice plugin supports PURGE requests, discarding the requested object from cache. -If a range is given in the client request, only the slice blocks from the -requested range will be purged (if in cache). If not, all of the blocks will be discarded -from the cache. - -If a block receives a 404, indicating the requested block to be purged is not in the cache, -slice will not continue to purge the following blocks. - -The functionality works with `--ref-relative` both enabled and disabled. If `--ref-relative` is -disabled (using slice 0 as the reference block), requesting to PURGE a block that does not have -slice 0 in its range will still PURGE the slice 0 block, as the reference block is always processed. +The slice plugin supports PURGE requests, discarding the requested object from +cache. Without a range every block is discarded; with a range, the blocks that +range covers are. Two cases below purge more than the range names: a suffix range, +and block 0 when ``--ref-relative`` is disabled. + +Slice issues one PURGE per block and walks every block it was asked for, whether +or not each one is currently cached. A block that is already absent answers 404 +internally; that is simply noted and the walk continues, so a gap left by +per-block eviction cannot leave the blocks behind it in cache. + +Slice learns where the object ends from the blocks it removes. PURGE is a Traffic +Server extension, so a successful block purge reports the removed object's extent +in a ``X-Purged-Content-Range`` header, and the walk continues to the last block that +extent implies. Blocks of one object can disagree about its length when the origin +object has been replaced in place; slice takes the largest extent any block +reports, so the longer generation's tail is not left behind. + +Until some block has reported an extent, the walk has no end but the miss bound: +it stops after ``--purge-probe-blocks`` consecutive uncached blocks and reports +that nothing was found. That is what bounds a PURGE for a URL which is not cached +at all. + +An operator often knows more about the object than the plugin does, since the +block count is just the object's size divided by the block size. That count can be +supplied per request with the header named by ``--purge-probe-header``, default +``X-Slice-Purge-Probe``:: + + PURGE /obj HTTP/1.1 + X-Slice-Purge-Probe: 64 + +This only changes how long the walk keeps going without having found anything; it +never limits how many blocks are purged once an extent is known. + +The bound has to be able to span a whole object, because in the worst case only +the object's last block is still cached, so the value an operator wants is the +object's size divided by the block size: a 10 GB object in 1 MB blocks needs +10240. There is no ceiling on it beyond that, since reaching the bound costs one +internal cache lookup per block and PURGE is already restricted by +:file:`ip_allow.yaml`. A malformed value is ignored in favour of the configured +default, and slice strips the header from the block requests it issues. + +If the bound is reached, slice logs that it gave up and reports ``404`` even though +later blocks may still be cached. Raise ``--purge-probe-blocks``, or send the +override, for objects whose leading blocks are routinely absent. + +A client range that is already closed, such as ``bytes=0-6399999999``, bounds the +walk directly, and is clamped against the object's extent as soon as some block +reports one. An over-estimate therefore costs no extra block PURGEs beyond the end +of the object, and if no block is cached at all the miss bound stops the walk. + +A suffix range, ``bytes=-``, names its blocks by their distance from an end +slice does not know yet, and purging is the only way it could find out. Rather +than guess at the start, such a purge is widened to the whole object: a superset of +what was asked for, so the named blocks certainly go. Note this is the one place a +PURGE removes more than its range names; a ``GET`` with the same header is +unaffected and still returns exactly the last *n* bytes. + +The response is sent once the walk is complete: ``200`` if at least one block was +removed, ``404`` if none was found. This matches what Traffic Server reports for a +PURGE of an object that is not sliced. + +The functionality works with ``--ref-relative`` both enabled and disabled. With it +disabled, block 0 is always the first block walked, so a PURGE whose range does not +cover block 0 still purges it. Conditional Slicing ------------------- diff --git a/doc/admin-guide/storage/index.en.rst b/doc/admin-guide/storage/index.en.rst index aa3754df5eb..af6d23de065 100644 --- a/doc/admin-guide/storage/index.en.rst +++ b/doc/admin-guide/storage/index.en.rst @@ -304,7 +304,7 @@ from any other IP, we connect to the daemon via localhost: :: > Host: example.com > Accept: */* > - < HTTP/1.1 200 Ok + < HTTP/1.1 200 OK < Date: Thu, 08 Jan 2010 20:32:07 GMT < Connection: keep-alive @@ -312,6 +312,21 @@ The next time Traffic Server receives a request for the removed object, it will contact the origin server to retrieve a new copy, which will replace the previously cached version in Traffic Server. +If the removed object was stored as a partial response, that is if it carried a +``Content-Range``, then the ``200 OK`` also reports that range back in a +``X-Purged-Content-Range`` header:: + + < HTTP/1.1 200 OK + < X-Purged-Content-Range: bytes 0-1048575/9437184 + +This lets a caller that holds one piece of a larger resource learn the whole +resource's extent without a second lookup. It is what allows the +:ref:`admin-plugins-slice` plugin to purge an object block by block and know which +block is the last one. The range is reported under its own header name rather than +as ``Content-Range``, because ``Content-Range`` on a ``200`` response has no +meaning under :rfc:`9110` and is read by other components as a sign that a stored +partial response is being served. + This procedure only removes the index to the object from a specific Traffic Server cache. While the object remains on disk, Traffic Server will no longer able to find the object. The next request for that object will result in a fresh copy of the diff --git a/plugins/slice/Config.cc b/plugins/slice/Config.cc index b95ace3bab0..b5e19f71891 100644 --- a/plugins/slice/Config.cc +++ b/plugins/slice/Config.cc @@ -27,8 +27,9 @@ namespace { -constexpr std::string_view DefaultSliceSkipHeader = {"X-Slicer-Info"}; -constexpr std::string_view DefaultCrrIdentHeader = {"X-Crr-Ident"}; +constexpr std::string_view DefaultSliceSkipHeader = {"X-Slicer-Info"}; +constexpr std::string_view DefaultCrrIdentHeader = {"X-Crr-Ident"}; +constexpr std::string_view DefaultPurgeProbeHeader = {"X-Slice-Purge-Probe"}; } // namespace Config::~Config() @@ -121,13 +122,15 @@ Config::fromArgs(int const argc, char const *const argv[]) {const_cast("minimum-size"), required_argument, nullptr, 'm'}, {const_cast("metadata-cache-size"), required_argument, nullptr, 'z'}, {const_cast("stats-prefix"), required_argument, nullptr, 'x'}, + {const_cast("purge-probe-blocks"), required_argument, nullptr, 'q'}, + {const_cast("purge-probe-header"), required_argument, nullptr, 'H'}, {nullptr, 0, nullptr, 0 }, }; // getopt assumes args start at '1' so this hack is needed char *const *argvp = (const_cast(argv) - 1); for (;;) { - int const opt = getopt_long(argc + 1, argvp, "b:de:g:i:lm:p:r:s:t:x:z:", longopts, nullptr); + int const opt = getopt_long(argc + 1, argvp, "b:de:g:H:i:lm:p:q:r:s:t:x:z:", longopts, nullptr); if (-1 == opt) { break; } @@ -248,6 +251,19 @@ Config::fromArgs(int const argc, char const *const argv[]) stat_prefix = optarg; DEBUG_LOG("Stat prefix: %s", stat_prefix.c_str()); } break; + case 'q': { + int const blocksread = atoi(optarg); + if (0 < blocksread) { + m_purge_probe_blocks = blocksread; + DEBUG_LOG("Using purge probe blocks %d", m_purge_probe_blocks); + } else { + ERROR_LOG("Invalid purge-probe-blocks: %s", optarg); + } + } break; + case 'H': { + m_purge_probe_header.assign(optarg); + DEBUG_LOG("Using purge probe header %s", optarg); + } break; default: break; } @@ -275,6 +291,10 @@ Config::fromArgs(int const argc, char const *const argv[]) m_skip_header = DefaultSliceSkipHeader; DEBUG_LOG("Using default slice skip header %s", m_skip_header.c_str()); } + if (m_purge_probe_header.empty()) { + m_purge_probe_header = DefaultPurgeProbeHeader; + DEBUG_LOG("Using default purge probe header %s", m_purge_probe_header.c_str()); + } if (m_min_size_to_slice > 0) { if (m_oscache.has_value()) { diff --git a/plugins/slice/Config.h b/plugins/slice/Config.h index 4fd15a2506b..f9888e66404 100644 --- a/plugins/slice/Config.h +++ b/plugins/slice/Config.h @@ -36,6 +36,8 @@ struct Config { static constexpr int64_t const blockbytesmax = 1024 * 1024 * 128; // 128MB static constexpr int64_t const blockbytesdefault = 1024 * 1024; // 1MB + static constexpr int const purgeprobeblocksdefault = 8; + int64_t m_blockbytes{blockbytesdefault}; std::string m_remaphost; // remap host to use for loopback slice GET std::string m_regexstr; // regex string for things to slice (default all) @@ -49,8 +51,12 @@ struct Config { bool m_head_strip_range{false}; // strip range header for head requests uint64_t m_min_size_to_slice{0}; // Only strip objects larger than this + // consecutive uncached blocks a purge tolerates before giving up on the object + int m_purge_probe_blocks{purgeprobeblocksdefault}; + std::string m_skip_header; std::string m_crr_ident_header; + std::string m_purge_probe_header; // request header overriding m_purge_probe_blocks // Convert optarg to bytes static int64_t bytesFrom(char const *const valstr); diff --git a/plugins/slice/Data.h b/plugins/slice/Data.h index 0c19dfaafa5..f5ef86c5db7 100644 --- a/plugins/slice/Data.h +++ b/plugins/slice/Data.h @@ -77,6 +77,10 @@ struct Data { int64_t m_blockskip{0}; // number of bytes to skip in this block int64_t m_blockconsumed{0}; // body bytes consumed + int64_t m_purge_hits{0}; // blocks a purge actually removed + int m_purge_misses{0}; // consecutive uncached blocks the walk has seen + int m_purge_miss_bound{0}; // from the config or the request header + BlockState m_blockstate{Pending}; // is there an active slice block int64_t m_bytestosend{0}; // header + content bytes to send @@ -109,11 +113,25 @@ struct Data { memset(&m_client_ip, 0, sizeof(m_client_ip)); } - // Check if response only expects header + // HEAD only; a purge sends just a header too but never reaches the transfer path bool onlyHeader() const { - return (m_method_type == TS_HTTP_METHOD_HEAD || m_method_type == TS_HTTP_METHOD_PURGE); + return m_method_type == TS_HTTP_METHOD_HEAD; + } + + bool + is_purge() const + { + return m_method_type == TS_HTTP_METHOD_PURGE; + } + + // The purge range, closed against the object length once known. m_req_range + // stays as sent so a longer extent can widen the walk; a clamp could only shrink. + Range + purge_range() const + { + return (m_contentlen < 0) ? m_req_range : m_req_range.intersectedWith(Range(0, m_contentlen)); } ~Data() diff --git a/plugins/slice/HttpHeader.cc b/plugins/slice/HttpHeader.cc index d08e76db7a9..e075036a240 100644 --- a/plugins/slice/HttpHeader.cc +++ b/plugins/slice/HttpHeader.cc @@ -326,6 +326,32 @@ HttpHeader::toString() const /////// HdrMgr +bool +HdrMgr::create_response(TSHttpStatus const status) +{ + resetHeader(); + + if (nullptr == m_buffer) { + m_buffer = TSMBufferCreate(); + } + + m_lochdr = TSHttpHdrCreate(m_buffer); + if (nullptr == m_lochdr) { + return false; + } + + TSHttpHdrTypeSet(m_buffer, m_lochdr, TS_HTTP_TYPE_RESPONSE); + TSHttpHdrVersionSet(m_buffer, m_lochdr, TS_HTTP_VERSION(1, 1)); + TSHttpHdrStatusSet(m_buffer, m_lochdr, status); + + char const *const reason = TSHttpHdrReasonLookup(status); + if (nullptr != reason) { + TSHttpHdrReasonSet(m_buffer, m_lochdr, reason, strlen(reason)); + } + + return true; +} + TSParseResult HdrMgr::populateFrom(TSHttpParser const http_parser, TSIOBufferReader const reader, HeaderParseFunc const parsefunc, int64_t *const bytes) diff --git a/plugins/slice/HttpHeader.h b/plugins/slice/HttpHeader.h index c52738e5d18..0fd50c779a9 100644 --- a/plugins/slice/HttpHeader.h +++ b/plugins/slice/HttpHeader.h @@ -39,6 +39,10 @@ constexpr std::string_view SLICE_CRR_HEADER = {"Slice-Crr-Status"}; constexpr std::string_view SLICE_CRR_VAL = "1"; +// extent of the object a PURGE removed, reported by ATS on a successful purge. +// Emitted by HttpTransact::delete_all_document_alternates_and_return. +constexpr std::string_view PURGED_CONTENT_RANGE = {"X-Purged-Content-Range"}; + /** Designed to be a cheap throwaway struct which allows a consumer to make various calls to manipulate headers. @@ -207,6 +211,13 @@ struct HdrMgr { } } + /** Create an owned HTTP/1.1 response header with the given status. + * + * For a response slice forms itself, with no server response to relay. An + * intercept is an HTTP/1.x channel, so the version is not negotiable. + */ + bool create_response(TSHttpStatus const status); + void resetHeader() { diff --git a/plugins/slice/client.cc b/plugins/slice/client.cc index 1494ffa38a5..92ea2738fe2 100644 --- a/plugins/slice/client.cc +++ b/plugins/slice/client.cc @@ -19,9 +19,46 @@ #include "client.h" #include "Config.h" +#include "server.h" #include "util.h" +#include "swoc/TextView.h" + +#include #include +#include + +namespace +{ +// Miss bound for this purge: the request header when usable, else the config value. +int +purge_miss_bound(HttpHeader const &header, Config const *const conf) +{ + char probestr[64]; + int probelen = sizeof(probestr); + + if (!header.valueForKey(conf->m_purge_probe_header.data(), conf->m_purge_probe_header.size(), probestr, &probelen)) { + return conf->m_purge_probe_blocks; + } + + swoc::TextView value{probestr, static_cast(probelen)}; + // isspace is only defined for values representable as unsigned char + value.trim_if([](char c) { return 0 != isspace(static_cast(c)); }); + + swoc::TextView parsed; + intmax_t const blocks = swoc::svtoi(value, &parsed, 10); + + // parsed must cover the whole value: "8abc" is a mistake, not eight blocks + if (parsed.size() != value.size() || blocks <= 0 || std::numeric_limits::max() < blocks) { + ERROR_LOG("Ignoring invalid %.*s value '%.*s'", static_cast(conf->m_purge_probe_header.size()), + conf->m_purge_probe_header.data(), probelen, probestr); + return conf->m_purge_probe_blocks; + } + + return static_cast(blocks); +} + +} // namespace // this is called once per transaction when the client sends a req header bool @@ -94,6 +131,28 @@ handle_client_req(TSCont contp, TSEvent event, Data *const data) data->m_req_range = rangebe; + if (data->is_purge()) { + // The substituted range covers block 0, so walking it would delete the head + if (TS_HTTP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE == data->m_statustype) { + ERROR_LOG("Refusing PURGE with an unparseable range"); + finish_purge(contp, data, TS_HTTP_STATUS_BAD_REQUEST); + return true; + } + + data->m_purge_miss_bound = purge_miss_bound(header, conf); + DEBUG_LOG("%p Purge miss bound %d block(s)", data, data->m_purge_miss_bound); + + // A suffix range cannot know its start block, so purge a superset: everything + if (data->m_req_range.isEndBytes()) { + data->m_req_range = Range(0, Range::maxval); + data->m_blocknum = 0; + DEBUG_LOG("%p Purge suffix range widened to the whole object", data); + } + + // The miss bound is for this proxy to act on, not to propagate + header.removeKey(conf->m_purge_probe_header.data(), conf->m_purge_probe_header.size()); + } + // remove ATS keys to avoid 404 loop header.removeKey(TS_MIME_FIELD_VIA, TS_MIME_LEN_VIA); header.removeKey(TS_MIME_FIELD_X_FORWARDED_FOR, TS_MIME_LEN_X_FORWARDED_FOR); @@ -126,6 +185,11 @@ handle_client_resp(TSCont contp, TSEvent event, Data *const data) { switch (event) { case TS_EVENT_VCONN_WRITE_READY: { + // finish_purge writes the whole response at once; nothing to throttle or pull + if (data->is_purge()) { + break; + } + switch (data->m_blockstate) { case BlockState::Fail: case BlockState::PendingRef: diff --git a/plugins/slice/response.cc b/plugins/slice/response.cc index 94081cf7b20..a34ef3a6cb0 100644 --- a/plugins/slice/response.cc +++ b/plugins/slice/response.cc @@ -86,6 +86,21 @@ string502(int const httpver) return msg; } +// Form the response to a sliced PURGE, once every block has been walked +bool +form_purge_response(HdrMgr &hdrmgr, TSHttpStatus const status) +{ + if (!hdrmgr.create_response(status)) { + return false; + } + + // The core adds Date, Age, Server and Connection to an intercept's response + HttpHeader header(hdrmgr.m_buffer, hdrmgr.m_lochdr); + header.setKeyVal(TS_MIME_FIELD_CONTENT_LENGTH, TS_MIME_LEN_CONTENT_LENGTH, "0", 1); + + return true; +} + void form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string const &bodystr) { diff --git a/plugins/slice/response.h b/plugins/slice/response.h index c116d22e21c..569af1120bc 100644 --- a/plugins/slice/response.h +++ b/plugins/slice/response.h @@ -25,4 +25,10 @@ std::string string502(int const httpver); std::string const &bodyString416(); +/** Fill hdrmgr with the response to a sliced PURGE, for the caller to print. + * + * The header is owned by hdrmgr and destroyed with it. + */ +bool form_purge_response(HdrMgr &hdrmgr, TSHttpStatus const status); + void form416HeaderAndBody(HttpHeader &header, int64_t const contentlen, std::string const &bodystr); diff --git a/plugins/slice/server.cc b/plugins/slice/server.cc index 7105d5157bd..88c5a49afc7 100644 --- a/plugins/slice/server.cc +++ b/plugins/slice/server.cc @@ -32,7 +32,7 @@ namespace { ContentRange -contentRangeFrom(HttpHeader const &header) +content_range_for_key(HttpHeader const &header, char const *const key, int const keylen) { ContentRange bcr; @@ -42,22 +42,25 @@ contentRangeFrom(HttpHeader const &header) char rangestr[1024]; int rangelen = sizeof(rangestr); - // look for expected Content-Range field - bool const hasContentRange(header.valueForKey(TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE, rangestr, &rangelen)); - - if (!hasContentRange) { - DEBUG_LOG("invalid response header, no Content-Range"); + if (!header.valueForKey(key, keylen, rangestr, &rangelen)) { + DEBUG_LOG("invalid response header, no %.*s", keylen, key); } else { // ensure null termination rangestr[rangelen] = '\0'; if (!bcr.fromStringClosed(rangestr)) { - DEBUG_LOG("invalid response header, malformed Content-Range, %s", rangestr); + DEBUG_LOG("invalid response header, malformed %.*s, %s", keylen, key, rangestr); } } return bcr; } +ContentRange +contentRangeFrom(HttpHeader const &header) +{ + return content_range_for_key(header, TS_MIME_FIELD_CONTENT_RANGE, TS_MIME_LEN_CONTENT_RANGE); +} + int64_t contentLengthFrom(HttpHeader const &header) { @@ -136,7 +139,7 @@ handleFirstServerHeader(Data *const data, TSCont const contp) int64_t const hlen = TSHttpHdrLengthGet(header.m_buffer, header.m_lochdr); int64_t const clen = contentLengthFrom(header); if (TS_HTTP_STATUS_OK == header.status() && data->onlyHeader()) { - DEBUG_LOG("HEAD/PURGE request stripped Range header: expects 200"); + DEBUG_LOG("HEAD request stripped Range header: expects 200"); data->m_bytestosend = hlen; data->m_blockexpected = 0; TSVIONBytesSet(output_vio, hlen); @@ -499,12 +502,173 @@ handleNextServerHeader(Data *const data) return true; } +// Take the largest extent any block reports: blocks disagree when the origin +// object was replaced in place, and the shorter one would leave a tail cached. +void +note_purge_extent(Data *const data, int64_t const length) +{ + if (length <= data->m_contentlen) { + return; + } + + data->m_contentlen = length; + DEBUG_LOG("purge extent now %" PRId64 ", walking through block %" PRId64, length, + data->purge_range().lastBlockFor(data->m_config->m_blockbytes)); +} + +// Record what the block response said, without answering the client. +void +note_purge_block_result(Data *const data) +{ + HttpHeader const header(data->m_resp_hdrmgr.m_buffer, data->m_resp_hdrmgr.m_lochdr); + DEBUG_LOG("Purge block header\n%s", header.toString().c_str()); + + TSHttpStatus const status = header.status(); + + if (TS_HTTP_STATUS_OK == status) { + ++data->m_purge_hits; + data->m_purge_misses = 0; + + // Not Content-Range: cache_range_requests reads that on a 200 as a stored 206 + // and rewrites the status + ContentRange const purgedcr = content_range_for_key(header, PURGED_CONTENT_RANGE.data(), PURGED_CONTENT_RANGE.size()); + if (purgedcr.isValid() && 0 < purgedcr.m_length) { + note_purge_extent(data, purgedcr.m_length); + } else { + DEBUG_LOG("Purged block %" PRId64 " reported no usable extent", data->m_blocknum); + } + } else { + // Already absent. The walk used to stop here, leaving every later block cached. + ++data->m_purge_misses; + DEBUG_LOG("Purge block %" PRId64 " was not cached (%d)", data->m_blocknum, status); + } +} + +// Issue the next purge, or answer the client if the walk is over. +void +advance_purge(TSCont const contp, Data *const data) +{ + int64_t const blockbytes = data->m_config->m_blockbytes; + Range const range = data->purge_range(); + + ++data->m_blocknum; + int64_t const firstblock = range.firstBlockFor(blockbytes); + if (data->m_blocknum < firstblock) { + data->m_blocknum = firstblock; + } + + if (data->m_contentlen < 0) { + // With no extent reported yet, the miss bound is the only end condition + if (data->m_purge_miss_bound <= data->m_purge_misses) { + DEBUG_LOG("purge gave up after %d consecutive uncached block(s)", data->m_purge_misses); + finish_purge(contp, data); + return; + } + } else if (!range.blockIsInside(blockbytes, data->m_blocknum)) { + finish_purge(contp, data); + return; + } + + data->m_blockstate = BlockState::Pending; + if (!request_block(contp, data)) { + ERROR_LOG("Failed to issue purge for block %" PRId64, data->m_blocknum); + finish_purge(contp, data); + } +} + } // namespace +// Answer the client once every block has been walked. Nothing is written +// downstream before this, so one uncached block cannot leak a 404 to the client. +// A non-NONE status overrides the outcome of the walk. +void +finish_purge(TSCont const contp, Data *const data, TSHttpStatus const status) +{ + data->m_upstream.close(); + data->m_blockstate = BlockState::Done; + + TSHttpStatus const reply = + (TS_HTTP_STATUS_NONE != status) ? status : (0 < data->m_purge_hits ? TS_HTTP_STATUS_OK : TS_HTTP_STATUS_NOT_FOUND); + + DEBUG_LOG("purge removed %" PRId64 " block(s), answering %d", data->m_purge_hits, reply); + + if (!data->m_dnstream.isOpen()) { + shutdown(contp, data); + return; + } + + HdrMgr synthmgr; + if (!form_purge_response(synthmgr, reply)) { + ERROR_LOG("Failed forming the purge response"); + shutdown(contp, data); + return; + } + + HttpHeader const synth(synthmgr.m_buffer, synthmgr.m_lochdr); + int const hlen = synth.byteSize(); + + data->m_dnstream.setupVioWrite(contp, hlen); + TSHttpHdrPrint(synthmgr.m_buffer, synthmgr.m_lochdr, data->m_dnstream.m_write.m_iobuf); + data->m_bytessent = hlen; + TSVIOReenable(data->m_dnstream.m_write.m_vio); +} + +// A purge walks blocks instead of transferring them, so it runs its own machine +void +handle_purge_resp(TSCont const contp, TSEvent const event, Data *const data) +{ + switch (event) { + case TS_EVENT_VCONN_READ_READY: + case TS_EVENT_VCONN_READ_COMPLETE: { + if (!data->m_server_block_header_parsed) { + int64_t consumed = 0; + TSIOBufferReader const reader = data->m_upstream.m_read.m_reader; + TSVIO const input_vio = data->m_upstream.m_read.m_vio; + TSParseResult const res = data->m_resp_hdrmgr.populateFrom(data->m_http_parser, reader, TSHttpHdrParseResp, &consumed); + + TSVIONDoneSet(input_vio, TSVIONDoneGet(input_vio) + consumed); + + if (TS_PARSE_CONT == res) { + return; + } + + data->m_server_block_header_parsed = true; + note_purge_block_result(data); + } + + // No block PURGE response has a body worth reading, but drop whatever arrives + // so the upstream read cannot stall on a full buffer + data->m_upstream.m_read.drainReader(); + } break; + + case TS_EVENT_VCONN_EOS: { + if (!data->m_server_block_header_parsed) { + // No response at all; count it as absent so the walk moves on + ++data->m_purge_misses; + DEBUG_LOG("Purge block %" PRId64 " ended with no response header", data->m_blocknum); + } + + // The next block cannot be requested while this one holds the upstream + data->m_upstream.close(); + advance_purge(contp, data); + } break; + + default: { + DEBUG_LOG("%p handle_purge_resp unhandled event: %s", data, TSHttpEventNameLookup(event)); + } break; + } +} + // this is called every time the server has data for us void handle_server_resp(TSCont contp, TSEvent event, Data *const data) { + // A purge never transfers content, so it gets its own state machine + if (data->is_purge()) { + handle_purge_resp(contp, event, data); + return; + } + switch (event) { case TS_EVENT_VCONN_READ_READY: { if (data->m_blockstate == BlockState::Passthru) { @@ -691,10 +855,7 @@ handle_server_resp(TSCont contp, TSEvent event, Data *const data) // isn't keeping up bool start_next_block = false; - if (data->m_method_type == TS_HTTP_METHOD_PURGE) { - // for PURGE requests, clients won't request more data (no body content) - start_next_block = true; - } else if (data->m_dnstream.m_write.isOpen()) { + if (data->m_dnstream.m_write.isOpen()) { // check throttle condition TSVIO const output_vio = data->m_dnstream.m_write.m_vio; int64_t const output_done = TSVIONDoneGet(output_vio); diff --git a/plugins/slice/server.h b/plugins/slice/server.h index c0d77e48032..667b854875e 100644 --- a/plugins/slice/server.h +++ b/plugins/slice/server.h @@ -35,3 +35,14 @@ */ void handle_server_resp(TSCont contp, TSEvent event, Data *const data); + +/** Walk the object's slice blocks issuing a PURGE for each. + * + * A purge transfers no content, so it runs a separate state machine: it walks + * every block whether or not each is cached, taking the object's extent from the + * ones it removes, and answers the client only once the walk is done. + */ +void handle_purge_resp(TSCont contp, TSEvent event, Data *const data); + +// Answer a purge: 200 if any block was removed, 404 if none was, or status if given +void finish_purge(TSCont contp, Data *const data, TSHttpStatus const status = TS_HTTP_STATUS_NONE); diff --git a/plugins/slice/util.cc b/plugins/slice/util.cc index 2f3a578629d..96b6060f8b2 100644 --- a/plugins/slice/util.cc +++ b/plugins/slice/util.cc @@ -138,7 +138,7 @@ request_block(TSCont contp, Data *const data) } header.removeKey(SLICE_CRR_HEADER.data(), SLICE_CRR_HEADER.size()); - if (data->m_config->m_prefetchcount > 0 && data->m_req_range.m_beg >= 0 && + if (!data->is_purge() && data->m_config->m_prefetchcount > 0 && data->m_req_range.m_beg >= 0 && data->m_blocknum == data->m_req_range.firstBlockFor(data->m_config->m_blockbytes)) { header.setKeyVal(SLICE_CRR_HEADER.data(), SLICE_CRR_HEADER.size(), SLICE_CRR_VAL.data(), SLICE_CRR_VAL.size()); } @@ -232,6 +232,12 @@ request_block(TSCont contp, Data *const data) bool reader_avail_more_than(TSIOBufferReader const reader, int64_t bytes) { + // A purge refused before opening an upstream has no reader, and TSIOBufferReaderStart + // does not tolerate a null one + if (nullptr == reader) { + return false; + } + TSIOBufferBlock block = TSIOBufferReaderStart(reader); if (nullptr == block) { diff --git a/src/proxy/http/HttpTransact.cc b/src/proxy/http/HttpTransact.cc index b87412d7b29..5dea7018ebe 100644 --- a/src/proxy/http/HttpTransact.cc +++ b/src/proxy/http/HttpTransact.cc @@ -7478,6 +7478,25 @@ HttpTransact::delete_all_document_alternates_and_return(State *s, bool cache_hit build_response(s, &s->hdr_info.client_response, s->client_info.http_version, (cache_hit == true) ? HTTPStatus::OK : HTTPStatus::NOT_FOUND); + // Report what was removed, so a caller holding one piece of a larger resource + // can learn its extent without a second lookup. Not Content-Range itself: on + // a 200 that is meaningless per RFC 9110, and cache_range_requests reads the + // pair as a stored 206 being served as 200 and rewrites the status. + if (cache_hit == true && s->method == HTTP_WKSIDX_PURGE && s->cache_info.object_read != nullptr) { + // read by the slice plugin as PURGED_CONTENT_RANGE in plugins/slice/HttpHeader.h + static constexpr std::string_view PURGED_CONTENT_RANGE{"X-Purged-Content-Range"}; + HTTPHdr *const cached_response = s->cache_info.object_read->response_get(); + + if (cached_response != nullptr) { + auto value{cached_response->value_get(static_cast(MIME_FIELD_CONTENT_RANGE))}; + if (!value.empty()) { + s->hdr_info.client_response.value_set(PURGED_CONTENT_RANGE, value); + TxnDbg(dbg_ctl_http_trans, "PURGE reporting X-Purged-Content-Range: %.*s", static_cast(value.length()), + value.data()); + } + } + } + return true; } else { if (valid_max_forwards) { diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_purge_gaps_client.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_purge_gaps_client.replay.yaml new file mode 100644 index 00000000000..9dd763119d6 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_purge_gaps_client.replay.yaml @@ -0,0 +1,709 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Client side of slice_purge_gaps.test.py. Each transaction is one phase, selected +# by uuid with the verifier-client --keys option; several phases share a run where +# they are independent. The uuid also picks which origin transaction the block +# requests slice derives from will match. +# +# Blocks are 10 bytes: block 0 is "a", 1 is "b", 2 is "c", 4 is "e". Every proxy +# runs --ref-relative, so a ranged GET touches only the blocks its range covers +# and a fill phase can leave a chosen block uncached. +# + +meta: + version: "1.0" + +sessions: +- transactions: + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, hole-fill-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, hole-fill-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [uuid, hole-purge] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, hole-check-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, hole-check-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /nofirst + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, nofirst-fill-1] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 10-19/30, as: equal}] + content: + encoding: plain + data: 'bbbbbbbbbb' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /nofirst + headers: + fields: + - [Host, slice] + - [uuid, nofirst-purge] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /nofirst + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, nofirst-check-1] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 10-19/30, as: equal}] + content: + encoding: plain + data: 'bbbbbbbbbb' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, mixed-fill-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, mixed-fill-1] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 10-19/50, as: equal}] + content: + encoding: plain + data: 'bbbbbbbbbb' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, mixed-fill-4] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 40-49/50, as: equal}] + content: + encoding: plain + data: 'eeeeeeeeee' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [uuid, mixed-purge] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, mixed-check-4] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 40-49/50, as: equal}] + content: + encoding: plain + data: 'eeeeeeeeee' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, mixed-check-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, mixed-check-1] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 10-19/50, as: equal}] + content: + encoding: plain + data: 'bbbbbbbbbb' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, ranged-fill-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, ranged-fill-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=0-29] + - [uuid, ranged-purge] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, ranged-check-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, ranged-check-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, openend-fill-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, openend-fill-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=20-] + - [uuid, openend-purge] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, openend-check-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, openend-check-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, endbytes-fill-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, endbytes-fill-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=-10] + - [uuid, endbytes-purge] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, endbytes-check-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, endbytes-check-2] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 20-29/30, as: equal}] + content: + encoding: plain + data: 'cccccccccc' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, sparse-fill-4] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 40-49/50, as: equal}] + content: + encoding: plain + data: 'eeeeeeeeee' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [X-Slice-Purge-Probe, not-a-number] + - [uuid, sparse-purge-narrow] + proxy-response: + status: 404 + + - client-request: + method: GET + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, sparse-check-alive] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 40-49/50, as: equal}] + content: + encoding: plain + data: 'eeeeeeeeee' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [X-Slice-Purge-Probe, '8'] + - [uuid, sparse-purge-wide] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '0', as: equal}] + + - client-request: + method: GET + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, sparse-check-gone] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 40-49/50, as: equal}] + content: + encoding: plain + data: 'eeeeeeeeee' + verify: {as: equal} + + - client-request: + method: GET + version: "1.1" + url: /badrange + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, badrange-fill-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} + + - client-request: + method: PURGE + version: "1.1" + url: /badrange + headers: + fields: + - [Host, slice] + - [Range, bytes=not-a-range] + - [uuid, badrange-purge] + proxy-response: + status: 400 + + - client-request: + method: GET + version: "1.1" + url: /badrange + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, badrange-check-0] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 0-9/30, as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaa' + verify: {as: equal} diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_purge_gaps_server.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_purge_gaps_server.replay.yaml new file mode 100644 index 00000000000..201c280d30a --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_purge_gaps_server.replay.yaml @@ -0,0 +1,491 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Origin side of slice_purge_gaps.test.py. +# +# Keyed on "{url}{field.range}{field.uuid}", so one transaction answers one block +# of one phase. Blocks are 10 bytes: block 0 is "a", 1 is "b", 2 is "c", 4 is "e". +# +# PURGE is answered by ATS and the plugin issues no other request kind, so neither +# appears here. +# +# A check phase for a block expected to SURVIVE its purge is deliberately absent: +# if such a block were wrongly purged, ATS would ask for an unregistered key and +# the client's own expectation would fail too. +# + +meta: + version: "1.0" + +sessions: +- transactions: + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, hole-fill-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, hole-fill-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, hole-check-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /hole + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, hole-check-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /nofirst + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, nofirst-fill-1] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 10-19/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'bbbbbbbbbb', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /nofirst + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, nofirst-check-1] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 10-19/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'bbbbbbbbbb', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, mixed-fill-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=10-19] + - [uuid, mixed-fill-1] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 10-19/50] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'bbbbbbbbbb', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, mixed-fill-4] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 40-49/50] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'eeeeeeeeee', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, mixed-check-4] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 40-49/50] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'eeeeeeeeee', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, ranged-fill-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, ranged-fill-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, ranged-check-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /ranged + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, ranged-check-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, openend-fill-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, openend-fill-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /openend + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, openend-check-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, endbytes-fill-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, endbytes-fill-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, endbytes-check-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /endbytes + headers: + fields: + - [Host, slice] + - [Range, bytes=20-29] + - [uuid, endbytes-check-2] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 20-29/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'cccccccccc', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, sparse-fill-4] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 40-49/50] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'eeeeeeeeee', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /sparse + headers: + fields: + - [Host, slice] + - [Range, bytes=40-49] + - [uuid, sparse-check-gone] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 40-49/50] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'eeeeeeeeee', size: 10} + + - client-request: + method: GET + version: "1.1" + url: /badrange + headers: + fields: + - [Host, slice] + - [Range, bytes=0-9] + - [uuid, badrange-fill-0] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-9/30] + - [Content-Length, '10'] + - [Cache-Control, 'public, max-age=86400'] + content: {encoding: plain, data: 'aaaaaaaaaa', size: 10} diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_stale_generation_client.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_stale_generation_client.replay.yaml new file mode 100644 index 00000000000..b000ceda3b7 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_stale_generation_client.replay.yaml @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Client side of slice_stale_generation.test.py. Each transaction is one phase, +# selected with the verifier-client --keys option, and its uuid also selects +# which generation the origin serves for the block requests that slice derives +# from it. +# +# Generation 1 is 32 bytes of "a" with ETag "v1". Generation 2, which replaces +# it under the same URL once the cache is filled, is 64 bytes of "b" with ETag +# "v2". +# + +meta: + version: "1.0" + +sessions: +- transactions: + + # + # Fill the cache while the origin holds generation 1. + # + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [uuid, fill] + proxy-response: + status: 200 + headers: + fields: + - [Content-Length, {value: '32', as: equal}] + - [ETag, {value: '"v1"', as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + verify: {as: equal} + + # + # The bug. Bytes 16-47 are entirely present in the current 64 byte object, but + # the cached reference block still reports the old 32 byte length, so the range + # is clipped to the stale object's end and served with the stale ETag as a + # fresh hit. Correct would be: 206, bytes 16-47/64, ETag "v2", 32 bytes of "b". + # + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [Range, bytes=16-47] + - [x-debug, x-cache] + - [uuid, clipped] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 16-31/32, as: equal}] + - [Content-Length, {value: '16', as: equal}] + - [ETag, {value: '"v1"', as: equal}] + - [X-Cache, {value: hit-fresh, as: prefix}] + content: + encoding: plain + data: 'aaaaaaaaaaaaaaaa' + verify: {as: equal} + + # + # Worse: a range that starts past the stale length is refused outright, even + # though those bytes exist in the current object. Correct would be: 206, + # bytes 32-63/64. + # + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [Range, bytes=32-63] + - [uuid, unsatisfiable] + proxy-response: + status: 416 + headers: + fields: + - [Content-Range, {value: '*/32', as: equal}] + - [ETag, {as: absent}] + + # + # Control: a path first fetched after the object was replaced is served + # correctly, so the two phases above are measuring the stale cached generation + # and not a broken plugin or harness. + # + - client-request: + method: GET + version: "1.1" + url: /fresh + headers: + fields: + - [Host, slice] + - [Range, bytes=16-47] + - [x-debug, x-cache] + - [uuid, control] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 16-47/64, as: equal}] + - [Content-Length, {value: '32', as: equal}] + - [ETag, {value: '"v2"', as: equal}] + content: + encoding: plain + data: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' + verify: {as: equal} + + # + # Cache an interior block of /mixed at generation 1, for + # SliceMixedGenerationTest. The reference block is cached with a one second + # lifetime and the interior block with a day, so only the reference block + # revalidates later. + # + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=16-31] + - [uuid, fill-interior] + proxy-response: + status: 206 + headers: + fields: + - [Content-Range, {value: bytes 16-31/32, as: equal}] + - [Content-Length, {value: '16', as: equal}] + - [ETag, {value: '"v1"', as: equal}] + content: + encoding: plain + data: 'aaaaaaaaaaaaaaaa' + verify: {as: equal} + + # + # The mixed generation failure. The reference block revalidates to generation + # 2 while the cached interior block is still generation 1, so slice advertises + # the current object correctly and then cannot deliver it: the interior block's + # Content-Range disagrees, the self heal refetches the interior and then the + # reference and gets the same blocks back, and the transaction is aborted with + # the response header already on the wire. The client is left holding a well + # formed 206 that promises 16 bytes and delivers none. + # + # This is the production symptom behind the block walk's "curl exit 18": + # a correct looking Content-Range of .../7031250004 followed by a body that + # stops early. + # + # The client is left with nothing parsable: slice aborts the transaction, and + # the response header it had already formed from the reference block never + # reaches the wire. There is deliberately no proxy-response node here, because + # no response arrives to verify. The test asserts the failure on the + # verifier-client's own output and in diags.log instead. + # + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=16-31] + - [x-debug, x-cache] + - [uuid, mixed] + + # A second child with a completely cold cache, pointed at the same parent. It + # never saw the previous generation, yet it fails identically, because the two + # blocks it fetches come from the parent and the parent is holding one of each. + # This is the incident's shape: the mixed set existed in exactly one place, the + # parent, and every child node inherited it. It also shows slice cannot evict: + # the earlier abort left the mixed pair on the parent untouched. + # + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=16-31] + - [x-debug, x-cache] + - [uuid, cold-child] diff --git a/tests/gold_tests/pluginTest/slice/replay/slice_stale_generation_server.replay.yaml b/tests/gold_tests/pluginTest/slice/replay/slice_stale_generation_server.replay.yaml new file mode 100644 index 00000000000..21d778a1f95 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/replay/slice_stale_generation_server.replay.yaml @@ -0,0 +1,286 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Origin side of slice_stale_generation.test.py. +# +# The server is keyed on "{url}{field.range}{field.uuid}", so each slice block +# request selects a transaction by the block's byte range and by the phase uuid +# that the client propagated through the plugin. The phase uuid is what replaces +# the origin object: the fill phase is answered with generation 1 (32 bytes, +# ETag "v1"), every later phase with generation 2 (64 bytes, ETag "v2"). +# +# For the two phases that read the already cached object there is deliberately +# only one transaction each, the reference block. Those exist so the origin +# really does hold the new object, but the test asserts the server never +# receives them: the cached blocks are fresh for a day, so nothing revalidates. +# If slice did go upstream it would either pick up generation 2, failing the +# client side assertions, or ask for an unregistered block and get a 404, +# failing the diags.log assertion. +# + +meta: + version: "1.0" + +sessions: +- transactions: + + # + # Phase fill: the origin holds generation 1, cached in 16 byte blocks with a + # day of freshness. + # + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [Range, bytes=0-15] + - [uuid, fill] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-15/32] + - [Content-Length, '16'] + - [ETag, '"v1"'] + - [Last-Modified, 'Mon, 01 Jun 2026 00:00:00 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'aaaaaaaaaaaaaaaa', size: 16} + + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [Range, bytes=16-31] + - [uuid, fill] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 16-31/32] + - [Content-Length, '16'] + - [ETag, '"v1"'] + - [Last-Modified, 'Mon, 01 Jun 2026 00:00:00 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'aaaaaaaaaaaaaaaa', size: 16} + + # + # The object is replaced here. Generation 2 is longer, with a new ETag and + # Last-Modified, under the same URL. The reference block is registered for + # each phase that reads the cached object, so the origin genuinely holds the + # new object, but the test asserts these keys are never requested. + # + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [Range, bytes=0-15] + - [uuid, clipped] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-15/64] + - [Content-Length, '16'] + - [ETag, '"v2"'] + - [Last-Modified, 'Wed, 08 Jul 2026 23:54:41 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'bbbbbbbbbbbbbbbb', size: 16} + + - client-request: + method: GET + version: "1.1" + url: /obj + headers: + fields: + - [Host, slice] + - [Range, bytes=0-15] + - [uuid, unsatisfiable] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-15/64] + - [Content-Length, '16'] + - [ETag, '"v2"'] + - [Last-Modified, 'Wed, 08 Jul 2026 23:54:41 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'bbbbbbbbbbbbbbbb', size: 16} + + # + # Phase control: a path first requested after the object was replaced, so + # nothing about it is cached and the client must see generation 2. + # + - client-request: + method: GET + version: "1.1" + url: /fresh + headers: + fields: + - [Host, slice] + - [Range, bytes=0-15] + - [uuid, control] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-15/64] + - [Content-Length, '16'] + - [ETag, '"v2"'] + - [Last-Modified, 'Wed, 08 Jul 2026 23:54:41 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'bbbbbbbbbbbbbbbb', size: 16} + + - client-request: + method: GET + version: "1.1" + url: /fresh + headers: + fields: + - [Host, slice] + - [Range, bytes=16-31] + - [uuid, control] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 16-31/64] + - [Content-Length, '16'] + - [ETag, '"v2"'] + - [Last-Modified, 'Wed, 08 Jul 2026 23:54:41 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'bbbbbbbbbbbbbbbb', size: 16} + + - client-request: + method: GET + version: "1.1" + url: /fresh + headers: + fields: + - [Host, slice] + - [Range, bytes=32-47] + - [uuid, control] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 32-47/64] + - [Content-Length, '16'] + - [ETag, '"v2"'] + - [Last-Modified, 'Wed, 08 Jul 2026 23:54:41 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'bbbbbbbbbbbbbbbb', size: 16} + + # + # Phase fill-interior, used by SliceMixedGenerationTest: /mixed at generation + # 1. The reference block is given a one second freshness lifetime so that + # later only it revalidates. In production the reference block was evicted and + # refetched after the replacement while the interior blocks survived; a short + # lifetime reaches the same end state without depending on eviction. + # + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=0-15] + - [uuid, fill-interior] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-15/32] + - [Content-Length, '16'] + - [ETag, '"v1"'] + - [Last-Modified, 'Mon, 01 Jun 2026 00:00:00 GMT'] + - [Cache-Control, 'public, max-age=1'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'aaaaaaaaaaaaaaaa', size: 16} + + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=16-31] + - [uuid, fill-interior] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 16-31/32] + - [Content-Length, '16'] + - [ETag, '"v1"'] + - [Last-Modified, 'Mon, 01 Jun 2026 00:00:00 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'aaaaaaaaaaaaaaaa', size: 16} + + # + # Phase mixed: the object has been replaced. Only the reference block is + # stale, so only it revalidates, and it comes back as generation 2 while the + # cached interior block is still generation 1. That is the disagreement slice + # cannot heal. + # + - client-request: + method: GET + version: "1.1" + url: /mixed + headers: + fields: + - [Host, slice] + - [Range, bytes=0-15] + - [uuid, mixed] + server-response: + status: 206 + reason: Partial Content + headers: + fields: + - [Content-Range, bytes 0-15/64] + - [Content-Length, '16'] + - [ETag, '"v2"'] + - [Last-Modified, 'Wed, 08 Jul 2026 23:54:41 GMT'] + - [Cache-Control, 'public, max-age=86400'] + - [Accept-Ranges, bytes] + content: {encoding: plain, data: 'bbbbbbbbbbbbbbbb', size: 16} diff --git a/tests/gold_tests/pluginTest/slice/slice_purge_gaps.test.py b/tests/gold_tests/pluginTest/slice/slice_purge_gaps.test.py new file mode 100644 index 00000000000..2f3ee9cc210 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_purge_gaps.test.py @@ -0,0 +1,281 @@ +"""Verify a PURGE traverses every slice block, not just the ones before a gap.""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = __doc__ + +Test.SkipUnless( + Condition.PluginExists('slice.so'), + Condition.PluginExists('cache_range_requests.so'), +) +Test.ContinueOnFail = True + + +class SlicePurgeGapsTest: + """Verify a PURGE removes every block it was asked for. + + Slice turns one client PURGE into one PURGE per block. The walk used to stop + at the first block that was not cached, because it had no other end + condition: m_contentlen was only ever set from a 206's Content-Range, a PURGE + response carried none, and so m_req_range.blockIsInside() was true for every + block number. The 404 stood in for a length the plugin never learned. + + ATS now reports the removed object's extent as X-Purged-Content-Range, so the + walk learns where the object ends from the blocks it is already deleting, and + a 404 is merely noted and stepped over. Until some block reports an extent the + only end condition is a bound on consecutive misses. + + Each object below exercises one thing, and each is purged exactly once, + because a purge consumes the state it is measured against. + + /hole blocks 0 and 2 cached, block 1 absent. A 404 mid-walk must not end + it, so block 2 goes too. + /nofirst only block 1 cached. The walk steps over block 0's 404, picks the + extent up from block 1, and the client must not be handed that 404. + /mixed blocks reporting a 30 byte and a 50 byte object, as happens when + the origin object is replaced in place. The walk must follow the + largest extent reported, not the first. + /ranged same gap as /hole, purged with a closed range. Already bounded, so + this isolates the 404 handling from the extent discovery. + /openend purged with "bytes=20-". The start is stated, so only block 2 may + go and block 0 must survive. + /endbytes purged with "bytes=-10". A suffix range cannot know which block it + starts at, so it is widened to the whole object and every block + goes. + /sparse only block 4 cached, on a proxy whose miss bound is 2, so the + default walk cannot reach it. Covers the bound, its per-request + override, and the fallback when the override is malformed. + /badrange purged with an unparseable range, which must be refused rather + than silently applied to block 0. + + Whether a block was purged is measured on the origin, not on the response + body: the origin serves each check phase exactly what the matching fill phase + served, so a purged block and a surviving block are indistinguishable to the + client, and the only difference is whether the origin was asked again. + """ + + _client_replay: str = 'replay/slice_purge_gaps_client.replay.yaml' + _server_replay: str = 'replay/slice_purge_gaps_server.replay.yaml' + + _block_bytes: int = 10 + + # Low enough that the default walk cannot reach /sparse's block 4, which is + # what makes the per-request override observable. + _low_miss_bound: int = 2 + + # Keyed on the block range as well as the phase uuid, so one origin + # transaction answers one block of one phase. + _origin_key_format: str = '--format "{url}{field.range}{field.uuid}"' + + def __init__(self) -> None: + """Declare the origin and the two proxies.""" + self._started = False + self._configure_origin() + self._ts = self._make_ts('ts') + self._ts_bound = self._make_ts('ts-bound', miss_bound=self._low_miss_bound) + + self._ts.Disk.traffic_out.Content = Testers.ContainsExpression( + 'Purge suffix range widened to the whole object', + 'A suffix range purge should be widened rather than guessing at its start.') + + self._ts_bound.Disk.traffic_out.Content = Testers.ContainsExpression( + f'gave up after {self._low_miss_bound} consecutive uncached block', + 'The walk should stop at its configured miss bound rather than scanning the whole range.') + self._ts_bound.Disk.diags_log.Content = Testers.ContainsExpression( + 'Ignoring invalid X-Slice-Purge-Probe', 'A malformed override should be rejected, not acted on.') + + # A purge issues nothing but block PURGEs. request_block logs every request + # header it builds at debug, so an only-if-cached here would mean a + # read-only length probe had been reintroduced. + for ts in (self._ts, self._ts_bound): + ts.Disk.traffic_out.Content += Testers.ExcludesExpression( + 'only-if-cached', 'A purge should not issue a read-only length probe.') + + def _configure_origin(self) -> None: + """Configure the origin.""" + self._origin = Test.MakeVerifierServerProcess('origin', self._server_replay, other_args=self._origin_key_format) + + # ATS answers PURGE itself and the plugin issues no other request kind, so + # neither may ever be seen upstream. + self._origin.Streams.stdout += Testers.ExcludesExpression( + 'PURGE', 'A PURGE should be answered by ATS and never forwarded to the origin.') + self._origin.Streams.stdout += Testers.ExcludesExpression('HEAD /', 'A purge should never issue a HEAD upstream.') + + def _make_ts(self, label: str, miss_bound: int = None) -> 'Process': + """Create a proxy that slices in front of cache_range_requests. + + --ref-relative keeps a ranged GET from dragging block 0 in as a reference + block, which is what lets a fill phase leave a chosen block uncached. + + :param label: process name suffix. + :param miss_bound: --purge-probe-blocks value, or None for the default. + """ + ts = Test.MakeATSProcess(label, enable_cache=True) + + bound = '' if miss_bound is None else f' @pparam=--purge-probe-blocks={miss_bound}' + ts.Disk.remap_config.AddLine( + f'map http://slice/ http://127.0.0.1:{self._origin.Variables.http_port}/' + f' @plugin=slice.so @pparam=--blockbytes-test={self._block_bytes} @pparam=--ref-relative{bound}' + ' @plugin=cache_range_requests.so') + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'slice|cache_range_requests', + }) + return ts + + def _run(self, summary: str, phases: str, ts: 'Process' = None) -> None: + """Add a TestRun replaying one or more phases, in replay file order. + + :param summary: TestRun description. + :param phases: space separated verifier-client keys. + :param ts: proxy to replay against, defaulting to the ordinary one. + """ + ts = self._ts if ts is None else ts + tr = Test.AddTestRun(summary) + if not self._started: + tr.Processes.Default.StartBefore(self._origin) + tr.Processes.Default.StartBefore(self._ts) + tr.Processes.Default.StartBefore(self._ts_bound) + self._started = True + tr.AddVerifierClientProcess( + f"client-{phases.replace(' ', '-')}", self._client_replay, http_ports=[ts.Variables.port], keys=phases) + tr.StillRunningAfter = ts + + def _origin_saw(self, phase: str, block: str, why: str) -> None: + """Assert the origin was asked for a block under a given phase.""" + self._origin.Streams.stdout += Testers.ContainsExpression(f'request with key /{block}{phase}', why) + + def _fill(self, summary: str, url: str, blocks: list, ts: 'Process' = None) -> None: + """Cache the given blocks of an object, and prove each fill happened. + + The fill assertions carry weight: a check phase only observes that the + origin was asked, which is equally true of a block that was purged and one + that was never cached at all. Asserting the fill reached the origin is what + makes the later check mean "removed" rather than merely "absent". + """ + self._run(summary, ' '.join(f'{url}-fill-{block}' for block in blocks), ts) + for block in blocks: + self._origin_saw( + f'{url}-fill-{block}', f'{url}bytes={block * 10}-{block * 10 + 9}', + f'Block {block} of /{url} should have been fetched and cached.') + + def _purged(self, phase: str, block: str, why: str) -> None: + """Assert a phase's block request reached the origin, so it was purged.""" + self._origin_saw(phase, block, why) + + def _survived(self, phase: str, block: str, why: str) -> None: + """Assert a phase's block request never reached the origin, so it survived. + + The origin has no transaction registered for such a phase either, so a + wrongly purged block fails the client's own expectation as well. + """ + self._origin.Streams.stdout += Testers.ExcludesExpression(f'request with key /{block}{phase}', why) + + def _gap_mid_walk(self) -> None: + """A 404 in the middle of the walk must not end it.""" + self._fill('Cache blocks 0 and 2 of /hole, leaving block 1 uncached', 'hole', [0, 2]) + self._run('PURGE the whole /hole object', 'hole-purge') + self._run('Both cached blocks of /hole were purged', 'hole-check-0 hole-check-2') + self._purged('hole-check-0', 'holebytes=0-9', 'Block 0 is in front of the gap, so it should be purged.') + self._purged('hole-check-2', 'holebytes=20-29', 'A PURGE should traverse blocks behind an uncached one.') + + def _uncached_first_block(self) -> None: + """A miss on the first block must not stop the purge before it starts.""" + self._fill('Cache only block 1 of /nofirst', 'nofirst', [1]) + self._run('PURGE /nofirst, whose block 0 is not cached', 'nofirst-purge') + self._run('Block 1 of /nofirst was purged', 'nofirst-check-1') + self._purged('nofirst-check-1', 'nofirstbytes=10-19', 'An uncached first block should not stop the purge.') + + def _largest_extent_wins(self) -> None: + """A block reporting a longer object widens the walk.""" + self._fill('Cache blocks 0, 1 and 4 of /mixed, disagreeing about its length', 'mixed', [0, 1, 4]) + self._run('PURGE /mixed', 'mixed-purge') + self._run('Block 4 of /mixed was purged, so the walk took the longer extent', 'mixed-check-4') + self._purged('mixed-check-4', 'mixedbytes=40-49', 'The walk should follow the largest extent any block reports.') + + def _closed_range(self) -> None: + """A closed range bounds the walk itself, isolating the 404 step-over.""" + self._fill('Cache blocks 0 and 2 of /ranged, leaving block 1 uncached', 'ranged', [0, 2]) + self._run('PURGE /ranged with a closed range spanning the whole object', 'ranged-purge') + self._run('Both cached blocks of /ranged were purged', 'ranged-check-0 ranged-check-2') + self._purged('ranged-check-0', 'rangedbytes=0-9', 'A closed range purge should remove the blocks it covers.') + self._purged('ranged-check-2', 'rangedbytes=20-29', 'A 404 should not end a closed range purge either.') + + def _open_ended_range(self) -> None: + """A "bytes=N-" purge states its start, so it purges only what it names.""" + self._fill('Cache blocks 0 and 2 of /openend', 'openend', [0, 2]) + self._run('PURGE /openend from byte 20 on', 'openend-purge') + self._run('Block 2 of /openend went and block 0 stayed', 'openend-check-2 openend-check-0') + self._purged('openend-check-2', 'openendbytes=20-29', 'The block covering the range should be purged.') + self._survived('openend-check-0', 'openendbytes=0-9', 'A purge should not remove blocks before its stated start.') + + def _suffix_range(self) -> None: + """A "bytes=-N" purge is widened to the whole object.""" + self._fill('Cache blocks 0 and 2 of /endbytes', 'endbytes', [0, 2]) + self._run('PURGE the last 10 bytes of /endbytes', 'endbytes-purge') + self._run('Every cached block of /endbytes went, not just the named tail', 'endbytes-check-0 endbytes-check-2') + self._purged('endbytes-check-2', 'endbytesbytes=20-29', 'The block covering the suffix range must be purged.') + self._purged( + 'endbytes-check-0', 'endbytesbytes=0-9', + 'A widened suffix purge removes the whole object, which is a superset of what was named.') + + def _miss_bound_and_override(self) -> None: + """The miss bound stops a walk that has found nothing, and is overridable. + + /sparse has only block 4 of five cached, out of reach of this proxy's + configured bound of two. Nothing about the remap changes between the two + purges below; only the request header does. + """ + ts = self._ts_bound + self._fill('Cache only block 4 of /sparse, out of reach of the configured bound', 'sparse', [4], ts) + + self._run('A purge with a malformed override falls back to the configured bound', 'sparse-purge-narrow', ts) + self._run('Block 4 of /sparse survived the too-narrow purge', 'sparse-check-alive', ts) + self._survived('sparse-check-alive', 'sparsebytes=40-49', 'A walk that gave up before block 4 should not have purged it.') + + self._run('PURGE /sparse with an override wide enough to reach block 4', 'sparse-purge-wide', ts) + self._run('Block 4 of /sparse was purged once the bound reached it', 'sparse-check-gone', ts) + self._purged('sparse-check-gone', 'sparsebytes=40-49', 'A request supplied bound should let the walk reach block 4.') + + def _unparseable_range(self) -> None: + """A purge whose range cannot be parsed is refused, not guessed at. + + An unparseable range leaves the plugin's range covering block 0 only, so + walking it would delete the head of the object and report success. A purge + is destructive, so it is rejected instead. + """ + self._fill('Cache block 0 of /badrange', 'badrange', [0]) + self._run('A PURGE with an unparseable range is refused', 'badrange-purge') + self._run('Block 0 of /badrange survived the refused purge', 'badrange-check-0') + self._survived('badrange-check-0', 'badrangebytes=0-9', 'A refused purge must not have removed anything.') + self._ts.Disk.diags_log.Content = Testers.ContainsExpression( + 'Refusing PURGE with an unparseable range', 'The refusal should be visible in the error log.') + + def run(self) -> None: + """Configure the test runs.""" + self._gap_mid_walk() + self._uncached_first_block() + self._largest_extent_wins() + self._closed_range() + self._open_ended_range() + self._suffix_range() + self._unparseable_range() + self._miss_bound_and_override() + + +SlicePurgeGapsTest().run() diff --git a/tests/gold_tests/pluginTest/slice/slice_stale_generation.test.py b/tests/gold_tests/pluginTest/slice/slice_stale_generation.test.py new file mode 100644 index 00000000000..05381279391 --- /dev/null +++ b/tests/gold_tests/pluginTest/slice/slice_stale_generation.test.py @@ -0,0 +1,327 @@ +"""Verify slice serves a stale object identity after the origin object changes.""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = __doc__ + +Test.SkipUnless( + Condition.PluginExists('slice.so'), + Condition.PluginExists('cache_range_requests.so'), + Condition.PluginExists('xdebug.so'), +) +Test.ContinueOnFail = False + + +class SliceHierarchyTest: + """Build the child/parent hierarchy the incident ran on. + + Both tiers load slice with the same block size, and both put + cache_range_requests behind it, as the affected property does. + + The parent only behaves as a slicing proxy for requests that arrive without + slice's skip header. The child stamps that header onto every block request it + issues (client.cc:66), so slice returns immediately on the parent + (slice.cc:48) and a child block request is handled there by + cache_range_requests alone: look up this exact Range, forward it on a miss, + store whatever 206 comes back. The parent therefore holds N independent + per-Range objects with no shared identity, and cannot notice, refuse or + reconcile a version mix. That is where the mixed set lived in the incident. + + A client hitting the parent directly is a different path: the parent does + slice that request, forms its own reference block and clamps against it. + """ + + _server_replay: str = 'replay/slice_stale_generation_server.replay.yaml' + _client_replay: str = 'replay/slice_stale_generation_client.replay.yaml' + + _block_bytes: int = 16 + + _origin_key_format: str = '--format "{url}{field.range}{field.uuid}"' + + def __init__(self, name: str) -> None: + """Declare the origin, the parent and the child. + + :param name: suffix distinguishing this hierarchy's processes. + """ + self._name = name + self._configure_dns() + self._configure_origin() + self._configure_parent() + self._configure_child() + + def _configure_dns(self) -> None: + """Configure a DNS server so neither tier consults resolv.conf.""" + self._dns = Test.MakeDNServer(f'dns-{self._name}', default='127.0.0.1') + + def _configure_origin(self) -> None: + """Configure the origin. + + The server is keyed on the block's byte range and on the phase uuid that + slice propagates from the client request, so one replay file answers + every block request of every phase and can replace the object between + phases without holding any state. + """ + self._origin = Test.MakeVerifierServerProcess( + f'origin-{self._name}', self._server_replay, other_args=self._origin_key_format) + + def _slice_remap(self, source: str, upstream: str) -> str: + """Build a remap rule carrying slice in front of cache_range_requests.""" + return ( + f'map {source} {upstream}' + f' @plugin=slice.so @pparam=--blockbytes-test={self._block_bytes}' + ' @plugin=cache_range_requests.so') + + def _records(self, ts: 'Process', debug: int) -> None: + """Apply the records.yaml settings common to both tiers.""" + ts.Disk.records_config.update( + { + 'proxy.config.diags.debug.enabled': debug, + 'proxy.config.diags.debug.tags': 'slice|cache_range_requests', + 'proxy.config.dns.nameservers': f'127.0.0.1:{self._dns.Variables.Port}', + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.http.parent_proxy.self_detect': 0, + }) + + def _configure_parent(self) -> None: + """Configure the parent, which the child's block requests reach.""" + self._parent = Test.MakeATSProcess(f'ts-parent-{self._name}') + self._parent.Disk.remap_config.AddLine( + self._slice_remap('http://origin.test/', f'http://127.0.0.1:{self._origin.Variables.http_port}/')) + self._parent.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache') + self._records(self._parent, debug=1) + + # slice is loaded on the parent, but every block request the child sends + # carries the skip header, so slice returns immediately and the request + # is handled by cache_range_requests alone. Plugin debug output lands in + # traffic.out, not diags.log. + self._parent.Disk.traffic_out.Content = Testers.ContainsExpression( + 'slice passing GET or HEAD request through to next plugin', + "The child's block requests should bypass the parent's slice.") + self._parent.Disk.traffic_out.Content += Testers.ExcludesExpression( + 'slice accepting and slicing', 'The parent should never slice a child block request.') + + def _make_child(self, label: str) -> 'Process': + """Create a child tier that slices and forwards to the parent.""" + ts = Test.MakeATSProcess(f'ts-{label}-{self._name}') + ts.Disk.remap_config.AddLine(self._slice_remap('http://slice/', 'http://origin.test/')) + ts.Disk.parent_config.AddLine( + f'dest_domain=. parent=127.0.0.1:{self._parent.Variables.port}' + ' round_robin=consistent_hash go_direct=false') + ts.Disk.plugin_config.AddLine('xdebug.so --enable=x-cache') + self._records(ts, debug=1) + return ts + + def _configure_child(self) -> None: + """Configure the child, which slices and forwards to the parent.""" + self._child = self._make_child('child') + + def _start_hierarchy(self, tr: 'TestRun') -> None: + """Bring up origin, parent and child for the first TestRun.""" + tr.Processes.Default.StartBefore(self._dns) + tr.Processes.Default.StartBefore(self._origin) + tr.Processes.Default.StartBefore(self._parent) + tr.Processes.Default.StartBefore(self._child) + + def _replay_phase(self, tr: 'TestRun', phase: str, ts: 'Process' = None) -> 'Process': + """Replay the client transaction for one phase against a child.""" + ts = self._child if ts is None else ts + return tr.AddVerifierClientProcess( + f'client-{phase}-{self._name}', self._client_replay, http_ports=[ts.Variables.port], keys=phase) + + def _still_running(self, tr: 'TestRun') -> None: + """Assert both tiers survive the TestRun.""" + tr.StillRunningAfter = self._child + tr.StillRunningAfter = self._parent + + +class SliceStaleGenerationTest(SliceHierarchyTest): + """Verify a cached reference block pins a stale object identity. + + The plugin takes the whole object length from the reference block's + Content-Range and clips the client range to it, so the cached reference + block, not the origin, defines the object's identity for every request:: + + server.cc handleFirstServerHeader: + data->m_contentlen = blockcr.m_length; + data->m_req_range.m_end = std::min(data->m_contentlen, data->m_req_range.m_end); + + Replacing the object under the same URL therefore forks every cache into one + that filled before the replacement and one that filled after, for as long as + the reference block stays fresh. On the stale side, ranges that exist in the + current object are answered against the stale length with the stale ETag: + clipped short, or refused with a 416. Neither path logs a block stitch error, + because handleNextServerHeader only complains when blocks disagree with each + other and here they are uniformly stale. + + Modelled on an incident where a versioned, year-cacheable object was replaced + in place. Two edges seven hours apart on either side of the replacement served + object lengths 4043309056 and 7031250004 for the same URL, the stale one + clipping a 64 MiB range request down to 16 MiB. + """ + + # The reference block the origin holds at the new generation for each phase + # that reads the cached object. The test asserts the origin never gets these. + _unreachable_keys = ('/objbytes=0-15clipped', '/objbytes=0-15unsatisfiable') + + def __init__(self) -> None: + """Declare the hierarchy and its assertions.""" + super().__init__('stale') + + for key in self._unreachable_keys: + self._origin.Streams.stdout += Testers.ExcludesExpression( + f'request with key {key}', 'The stale object should never be refetched after the origin object changed.') + + # Debug is enabled on the child, so Config::canLogError cannot suppress a + # block stitch error by pacing. The stale response is served with none. + self._child.Disk.diags_log.Content = Testers.ExcludesExpression( + 'logSliceError', 'The stale response should be served with no block stitch error.') + self._child.Disk.diags_log.Content += Testers.ExcludesExpression( + 'Mismatch/Bad block Content-Range', 'The stale blocks agree with each other, so nothing should mismatch.') + + def _fill_cache(self) -> None: + """Cache the whole object while the origin holds the first generation.""" + tr = Test.AddTestRun('Cache the object while the origin holds the first generation') + self._start_hierarchy(tr) + self._replay_phase(tr, 'fill') + self._still_running(tr) + + def _verify_clipped_range(self) -> None: + """A range inside the current object is clipped to the stale length.""" + tr = Test.AddTestRun('A range inside the current object is clipped to the stale length') + self._replay_phase(tr, 'clipped') + self._still_running(tr) + + def _verify_unsatisfiable_range(self) -> None: + """A range past the stale length is refused with a 416.""" + tr = Test.AddTestRun('A range past the stale length is refused with a 416') + self._replay_phase(tr, 'unsatisfiable') + self._still_running(tr) + + def _verify_uncached_object(self) -> None: + """An object first fetched after the replacement is served correctly.""" + tr = Test.AddTestRun('An object first fetched after the replacement is served correctly') + self._replay_phase(tr, 'control') + self._still_running(tr) + + def run(self) -> None: + """Configure the test runs.""" + self._fill_cache() + self._verify_clipped_range() + self._verify_unsatisfiable_range() + self._verify_uncached_object() + + +class SliceMixedGenerationTest(SliceHierarchyTest): + """Verify the parent stores a version mix and the child cannot recover. + + The other failure mode from the same origin object replacement, and the one + the parent's per-Range cache makes possible. Only the reference block is + refetched after the replacement, so the parent ends up holding two blocks of + one object at two different generations, both fresh, with nothing to relate + them. It serves each on request without complaint. + + The child is the only tier that compares blocks, and only against its own + reference block. It forms the client response header from the reference block, + which is correct for the current object, and only then discovers that the + interior block belongs to the previous one:: + + server.cc handleNextServerHeader: + if (!blockcr.isValid() || blockcr.m_length != data->m_contentlen) { + logSliceError("Mismatch/Bad block Content-Range", data, header); + + The self heal refetches the reference block, which is already the newest one + the parent holds, so the same block comes back and the interior block still + disagrees with it. The second mismatch is where slice gives up. + + The upstream is aborted. Slice can abort but cannot evict, so the mixed pair + on the parent survives. The final TestRun proves where the damage actually + lives: a second child with a completely cold cache, pointed at the same + parent, fails identically. It never saw the previous generation; it simply + inherits the mix from the one place that holds it. That is the incident's + shape, where all 105 blocks hashed to a single parent and every one of the 32 + child nodes served the same broken object. + """ + + # The reference block is cached with a one second lifetime, so let it expire. + _expiry_wait: int = 2 + + def __init__(self) -> None: + """Declare the hierarchy and its assertions.""" + super().__init__('mixed') + self._cold_child = self._make_child('cold-child') + + # Unlike the stale case, the child does report this one: first the interior + # block against the reference block, then the refetch against the interior. + self._child.Disk.diags_log.Content = Testers.ContainsExpression( + 'Mismatch/Bad block Content-Range.*blk_range="16-31".*etag_got="%22v1%22"', + 'The interior block should disagree with the reference block.') + self._child.Disk.diags_log.Content += Testers.ContainsExpression( + 'Mismatch/Bad block Content-Range.*blk_range="0-15".*etag_got="%22v2%22"', + 'The refetched reference block should disagree in turn, leaving no way out.') + + # The parent never compares blocks, so it never complains about the mix + # it is storing and serving to the child. + self._parent.Disk.diags_log.Content += Testers.ExcludesExpression( + 'logSliceError', 'The parent should not notice the version mix it holds.') + self._parent.Disk.diags_log.Content += Testers.ExcludesExpression( + 'Mismatch/Bad block Content-Range', 'The parent should not compare blocks at all.') + + def _fill_interior_block(self) -> None: + """Cache an interior block at the first generation, on both tiers.""" + tr = Test.AddTestRun('Cache an interior block at the first generation') + self._start_hierarchy(tr) + self._replay_phase(tr, 'fill-interior') + self._still_running(tr) + + def _verify_aborted_response(self) -> None: + """The reference block moves on and the transaction cannot be completed.""" + tr = Test.AddTestRun('A mixed generation object cannot be delivered and the request fails') + client = self._replay_phase(tr, 'mixed') + # Let the reference block go stale so that only it is revalidated. + tr.Processes.Default.Command = f'sleep {self._expiry_wait}; ' + tr.Processes.Default.Command + # Slice aborts the transaction, so the client never reads a response at + # all: not a short body, no response header. verifier-client exits 1. + client.ReturnCode = 1 + client.Streams.stdout += Testers.ContainsExpression( + 'Failed to find a well-formed, completed HTTP response: PARSE_INCOMPLETE', + 'The client should not receive a parsable response.') + client.Streams.stdout += Testers.ContainsExpression( + 'Failed HTTP/1 transaction with key: mixed', 'The transaction should fail.') + self._still_running(tr) + + def _verify_mix_is_on_the_parent(self) -> None: + """A cold child fails identically, because the mix lives on the parent.""" + tr = Test.AddTestRun('A second child with a cold cache inherits the mix from the parent') + tr.Processes.Default.StartBefore(self._cold_child) + client = self._replay_phase(tr, 'cold-child', ts=self._cold_child) + client.ReturnCode = 1 + client.Streams.stdout += Testers.ContainsExpression( + 'Failed HTTP/1 transaction with key: cold-child', 'A node that never saw the old generation should fail the same way.') + self._cold_child.Disk.diags_log.Content = Testers.ContainsExpression( + 'Mismatch/Bad block Content-Range', 'The cold child should hit the same mismatch.') + tr.StillRunningAfter = self._cold_child + self._still_running(tr) + + def run(self) -> None: + """Configure the test runs.""" + self._fill_interior_block() + self._verify_aborted_response() + self._verify_mix_is_on_the_parent() + + +SliceStaleGenerationTest().run() +SliceMixedGenerationTest().run()