diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
index 3b4c9a90..d1b0f13e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java
@@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) {
this.idToSymbol = new ObjList<>(initialCapacity);
}
+ /**
+ * Appends {@code symbol} at the next sequential id, matching a recovered /
+ * persisted dictionary's dense id order, WITHOUT de-duplicating.
+ *
+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted})
+ * replays the persisted entries in id order to rebuild this dictionary. It must
+ * NOT collapse two source strings that decode to the same characters, because
+ * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread
+ * catch-up mirror all key on the entry POSITION (id), not on the string. The
+ * only strings that collide this way are malformed lone UTF-16 surrogates,
+ * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would
+ * de-dup them and leave this dictionary SHORTER than the persisted entry count,
+ * desyncing the producer's delta baseline from the catch-up mirror (which uses
+ * {@code pd.size()}) and silently misattributing later symbols. Appending
+ * unconditionally keeps {@link #size()} equal to that count. The reverse lookup
+ * keeps the highest id for a colliding string, which is harmless: both ids
+ * encode to the same bytes, so resolving either is equivalent.
+ *
+ * @param symbol the recovered symbol string (must not be null)
+ * @return the id assigned (the previous {@link #size()})
+ */
+ public int addRecoveredSymbol(String symbol) {
+ if (symbol == null) {
+ throw new IllegalArgumentException("symbol cannot be null");
+ }
+ int newId = idToSymbol.size();
+ symbolToId.put(symbol, newId);
+ idToSymbol.add(symbol);
+ return newId;
+ }
+
/**
* Clears all symbols from the dictionary.
*
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
index aad95f3c..86343772 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java
@@ -76,6 +76,27 @@ public static int varintSize(long value) {
return (64 - Long.numberOfLeadingZeros(value) + 6) / 7;
}
+ /**
+ * Writes {@code value} as an unsigned LEB128 varint directly at native address
+ * {@code addr} and returns the address just past the last byte. The canonical
+ * raw-address varint writer shared by the SF cursor's persisted dictionary and
+ * catch-up frame builder.
+ *
+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits
+ * a SINGLE truncated byte for a negative long, whereas {@link #varintSize}
+ * returns 10 for it -- a size/write mismatch that would corrupt the stream. All
+ * callers pass ids/lengths/counts (non-negative); the assert pins that contract.
+ */
+ public static long writeVarint(long addr, long value) {
+ assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value;
+ while (value > 0x7F) {
+ Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80));
+ value >>>= 7;
+ }
+ Unsafe.getUnsafe().putByte(addr++, (byte) value);
+ return addr;
+ }
+
@Override
public void close() {
if (bufferPtr != 0) {
@@ -305,6 +326,7 @@ public void putUtf8(CharSequence value) {
*/
@Override
public void putVarint(long value) {
+ assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value;
ensureCapacity(10); // max varint bytes
long addr = bufferPtr + position;
while (value > 0x7F) {
@@ -336,11 +358,7 @@ public void skip(int bytes) {
}
private static void writeVarintDirect(long addr, long value) {
- while (value > 0x7F) {
- Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80));
- value >>>= 7;
- }
- Unsafe.getUnsafe().putByte(addr, (byte) value);
+ writeVarint(addr, value);
}
private void encodeUtf8(CharSequence value, int utf8Len) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
index ced1a1b5..8013330a 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java
@@ -39,6 +39,14 @@ public class QwpWebSocketEncoder implements QuietCloseable {
private final QwpColumnWriter columnWriter = new QwpColumnWriter();
private NativeBufferWriter buffer;
+ // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region
+ // ([len][utf8]... only, without the two section varints) that beginMessage
+ // last wrote. Let the producer persist those bytes straight to the slot's
+ // .symbol-dict instead of re-encoding the same symbols (see
+ // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next
+ // beginMessage; stored as offsets so they survive a buffer realloc.
+ private int deltaEntriesEnd;
+ private int deltaEntriesStart;
// QWP ingress always advertises Gorilla timestamp encoding. The column
// writer still emits a per-column encoding byte and falls back to raw
// values when delta-of-delta overflows int32.
@@ -75,10 +83,12 @@ public void beginMessage(
payloadStart = buffer.getPosition();
buffer.putVarint(deltaStart);
buffer.putVarint(deltaCount);
+ deltaEntriesStart = buffer.getPosition();
for (int id = deltaStart; id < deltaStart + deltaCount; id++) {
String symbol = globalDict.getSymbol(id);
buffer.putString(symbol);
}
+ deltaEntriesEnd = buffer.getPosition();
columnWriter.setBuffer(buffer);
}
@@ -122,6 +132,22 @@ public QwpBufferWriter getBuffer() {
return buffer;
}
+ /**
+ * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...},
+ * excluding the two section varints) that {@link #beginMessage} last wrote.
+ */
+ public int getDeltaEntriesLen() {
+ return deltaEntriesEnd - deltaEntriesStart;
+ }
+
+ /**
+ * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY
+ * region {@link #beginMessage} last wrote.
+ */
+ public int getDeltaEntriesStart() {
+ return deltaEntriesStart;
+ }
+
public void setDeferCommit(boolean defer) {
if (defer) {
flags |= FLAG_DEFER_COMMIT;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index d1744065..70a7e877 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -46,6 +46,7 @@
import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener;
import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler;
import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher;
import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher;
import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher;
@@ -56,6 +57,7 @@
import io.questdb.client.std.Decimal128;
import io.questdb.client.std.Decimal256;
import io.questdb.client.std.Decimal64;
+import io.questdb.client.std.IntList;
import io.questdb.client.std.Misc;
import io.questdb.client.std.Numbers;
import io.questdb.client.std.NumericException;
@@ -168,6 +170,11 @@ public class QwpWebSocketSender implements Sender {
// behind a drainer's endpoint walk.
private final ReentrantLock connectWalkLock = new ReentrantLock();
private final QwpHostHealthTracker hostTracker;
+ // Per-table encoded body byte counts captured during flushPendingRows' combined
+ // encode, reused by flushPendingRowsSplit to size each split frame arithmetically
+ // instead of re-encoding the batch a second time. Cleared and repopulated on every
+ // flush; only consumed on the (exceptional) split path. Reused to stay zero-GC.
+ private final IntList splitFrameBodyBytes = new IntList();
private final CharSequenceObjHashMap tableBuffers;
// null means plain text (no TLS)
private final ClientTlsConfiguration tlsConfig;
@@ -221,6 +228,17 @@ public class QwpWebSocketSender implements Sender {
private CursorSendEngine cursorEngine;
private CursorWebSocketSendLoop cursorSendLoop;
private boolean deferCommit;
+ // True when the sender emits incremental (delta) symbol dictionaries: each
+ // message carries only symbol ids not yet sent on the wire, rather than the
+ // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from
+ // the in-process ring) and in file-mode store-and-forward when the per-slot
+ // persisted dictionary opened. In both, the I/O thread re-registers the whole
+ // dictionary via a catch-up frame before replaying, so a non-self-sufficient
+ // delta frame never dangles an id on a fresh server. Falls back to full
+ // self-sufficient frames only when the persisted dictionary is unavailable in
+ // file-mode (recovery/orphan-drain would then have nothing to rebuild the
+ // deltas from). Set in setCursorEngine.
+ private boolean deltaDictEnabled;
// User-supplied observer for background orphan-slot drainer events.
// Volatile: written by setDrainerListener (any thread, before or after
// startOrphanDrainers) and read at pool-creation time. Null -> drainers
@@ -313,6 +331,12 @@ public class QwpWebSocketSender implements Sender {
// beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the
// first round indicates the initial connect.
private long roundSeq;
+ // Highest global symbol id the producer has baked into a frame so far, or -1.
+ // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because
+ // the I/O thread re-registers the full dictionary via a catch-up frame before
+ // replaying, so the producer's delta baseline stays valid across the wire
+ // boundary. Used only when deltaDictEnabled; ignored in full-dict mode.
+ private int sentMaxSymbolId = -1;
// When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only
// explicit flush() triggers the server-side commit. Enables accumulating
// arbitrarily large datasets that exceed the server's recv buffer.
@@ -2215,6 +2239,18 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) {
}
this.cursorEngine = engine;
this.ownsCursorEngine = takeOwnership && engine != null;
+ // Delta encoding is available in memory-mode (in-process catch-up) and in
+ // file-mode when the persisted dictionary opened (recovery / orphan-drain
+ // rebuild the dictionary from it). Otherwise fall back to full self-
+ // sufficient frames. See CursorSendEngine.isDeltaDictEnabled.
+ this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled();
+ // Recovery: repopulate the producer's global dictionary from the slot's
+ // persisted dictionary so newly ingested symbols continue from the
+ // recovered ids (rather than colliding with them at 0), and the delta
+ // baseline resumes where the crashed session left off.
+ if (deltaDictEnabled && engine.wasRecoveredFromDisk()) {
+ seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict());
+ }
}
/**
@@ -3332,6 +3368,17 @@ private void ensureConnected() {
cursorSendLoop.setConnectionDispatcher(connectionDispatcher);
cursorSendLoop.start();
} catch (Throwable t) {
+ // start() (or dispatcher construction) failed after cursorSendLoop was
+ // assigned. Close it so a caller that retries -- re-entering
+ // ensureConnected and reassigning cursorSendLoop above -- cannot orphan
+ // a recovered slot's ctor-seeded native mirror (freed only by close()
+ // or the I/O loop, neither of which has run). close() is idempotent and
+ // frees the mirror via its loopNeverRan path; it also closes the shared
+ // client, so the client.close() below is a safe idempotent no-op.
+ if (cursorSendLoop != null) {
+ cursorSendLoop.close();
+ cursorSendLoop = null;
+ }
if (client != null) {
client.close();
client = null;
@@ -3359,18 +3406,18 @@ private void ensureConnected() {
host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes);
} else {
// Async mode: I/O thread will drive the connect. Encoder uses
- // its default version (V1). The symbol-dict watermark still gets
- // reset for consistency with the sync path; the post-connect replay
- // path does not need a producer-side reset signal because every
- // cursor frame is self-sufficient.
+ // its default version (V1). The per-batch symbol-dict watermark still
+ // gets reset for consistency with the sync path; the post-connect
+ // replay path needs no producer-side reset signal (see below).
Endpoint ep = endpoints.get(0);
LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]",
ep.host, ep.port, endpoints.size());
}
- // Server starts fresh on each connection, so reset the symbol-dict
- // watermark. Cursor frames are self-sufficient (every frame carries its
- // full inline schema + a symbol-dict delta from id 0), so post-reconnect
- // replay needs no producer-side reset signal.
+ // Server starts fresh on each connection, so reset the per-batch
+ // symbol-dict watermark. Every frame still carries its full inline schema,
+ // and the fresh server's dictionary is re-established either by a full-dict
+ // frame (full-dict mode) or by an I/O-thread catch-up frame before replay
+ // (delta mode), so post-reconnect replay needs no producer-side reset signal.
resetSymbolDictStateForNewConnection();
connectionError.set(null);
@@ -3423,14 +3470,25 @@ private void flushPendingRows(boolean deferCommit) {
}
ensureActiveBufferReady();
- // Cursor SF requires every on-disk frame to be self-sufficient:
- // recorded frames replay to fresh server connections (orphan-slot
- // drainers and post-reconnect replay), so always emit the full
- // symbol-dict delta from id=0 and the full column schema inline,
- // never a back-reference the target server may not have seen.
+ // In full-dict mode every frame is self-sufficient: it carries the whole
+ // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh
+ // server never dangles a symbol id. In delta mode (memory-mode, and
+ // file-mode store-and-forward once the persisted dictionary opened) each
+ // frame carries only ids above sentMaxSymbolId; a reconnect re-registers
+ // the dictionary via an I/O-thread catch-up frame before replay, so the
+ // producer's monotonic baseline stays valid across the wire boundary.
encoder.setDeferCommit(deferCommit);
encoder.beginMessage(tableCount, globalSymbolDictionary,
- /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId);
+ symbolDeltaBaseline(), currentBatchMaxSymbolId);
+ // Record each table's encoded body size (position delta across addTable) so
+ // the split path can size its per-table frames arithmetically rather than
+ // re-encoding the whole batch. Body encoding is context-free (a table's bytes
+ // don't depend on its siblings or the delta section), so the size a table
+ // takes here equals the size it takes in its own split frame. Only consumed
+ // when the batch overflows the cap; the capture is a couple of int ops per
+ // table on the common path.
+ splitFrameBodyBytes.clear();
+ int bodyStart = encoder.getBuffer().getPosition();
for (int i = 0, n = keys.size(); i < n; i++) {
CharSequence tableName = keys.getQuick(i);
if (tableName == null) {
@@ -3447,19 +3505,41 @@ private void flushPendingRows(boolean deferCommit) {
}
encoder.addTable(tableBuffer);
+ int bodyEnd = encoder.getBuffer().getPosition();
+ splitFrameBodyBytes.add(bodyEnd - bodyStart);
+ bodyStart = bodyEnd;
}
int messageSize = encoder.finishMessage();
QwpBufferWriter buffer = encoder.getBuffer();
- if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) {
- flushPendingRowsSplit(keys, deferCommit);
+ // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers
+ // serverMaxBatchSize on a mid-stream failover to a smaller-cap node
+ // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop
+ // re-read the field independently, a failover between them would size frames
+ // against two different caps -- breaking the all-or-nothing guarantee and
+ // firing the publish-loop assert on a legitimate race. Both use this snapshot;
+ // the next flush picks up the new cap.
+ int cap = serverMaxBatchSize;
+ if (cap > 0 && messageSize > cap) {
+ // The combined frame's delta-entry bytes are byte-identical to the first
+ // split frame's (same baseline + batch max), so capture the length now
+ // for the arithmetic frame-sizing in flushPendingRowsSplit.
+ flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen(), cap);
return;
}
+ // Write-ahead: durably persist this frame's new symbols BEFORE it is
+ // published, so a recovered/orphan-drained slot can always rebuild the
+ // dictionary the (non-self-sufficient) delta frame references. No-op in
+ // memory mode and when the frame introduces no new symbols.
+ persistNewSymbolsBeforePublish();
activeBuffer.ensureCapacity(messageSize);
activeBuffer.write(buffer.getBufferPtr(), messageSize);
activeBuffer.incrementRowCount();
sealAndSwapBuffer();
+ // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring;
+ // advance the delta baseline so the next frame ships only newer ids.
+ advanceSentMaxSymbolId();
hasDeferredMessages = deferCommit;
if (!deferCommit) {
@@ -3475,26 +3555,83 @@ private void flushPendingRows(boolean deferCommit) {
* own message. All messages except the last carry FLAG_DEFER_COMMIT
* so the server appends rows without committing until the final
* message arrives.
+ *
+ * Not atomic across frames. The frames publish one at a time, so a
+ * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on
+ * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout --
+ * leaves frames 1..k-1 already on the ring as deferred (appended, not yet
+ * committed). The throw propagates past the {@code resetTableBuffersAfterFlush}
+ * at the end of the loop, so the source rows survive in their table buffers
+ * and the NEXT flush re-emits the whole batch; the eventual commit then
+ * commits the already-published prefix alongside the re-sent copies,
+ * delivering those rows at-least-once (duplicated), not exactly-once. This is
+ * within store-and-forward's at-least-once contract -- a DEDUP table or a
+ * durable-ack await absorbs the duplicate, and the symbol-dict state stays
+ * consistent on the retry (the re-sent frames carry empty deltas and the
+ * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic
+ * (rolling back the published prefix, or skipping it on retry) would be a
+ * larger change.
*
* @param deferCommit when true, ALL messages (including the last)
* carry FLAG_DEFER_COMMIT. When false, only the
* last message omits the flag.
*/
- private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) {
+ private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen, int cap) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit);
- }
-
- // Collect non-empty table indices so we know which is last.
+ LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit);
+ }
+
+ // Collect non-empty table indices so we know which is last, AND pre-flight
+ // every split frame's size BEFORE publishing any of them. The split hands
+ // frames to the ring one at a time (all but the last deferred -- appended but
+ // uncommitted); if a later table's frame were only found oversized
+ // mid-publish, the already-published prefix would strand on the ring, a
+ // subsequent commit would deliver it as a partial batch, and
+ // resetTableBuffersAfterFlush would discard every source row -- a partial
+ // commit the caller was told (by the throw) had failed. Checking all sizes up
+ // front makes the split all-or-nothing: either every frame fits and all
+ // publish, or none publish and we throw with nothing stranded.
+ //
+ // Each split frame's size is derived ARITHMETICALLY from the combined encode
+ // flushPendingRows already performed -- header + the two delta-section varints
+ // + the delta entries (byte-identical to the combined frame's when this frame
+ // carries them) + the table's own body bytes (captured in splitFrameBodyBytes,
+ // context-free so identical here and in its solo frame) -- rather than
+ // re-encoding every table a second time. simBaseline mirrors the publish loop's
+ // baseline advance (advanceSentMaxSymbolId), so each size equals the frame the
+ // publish loop will build; this pass mutates no delta/persist state.
int nonEmptyCount = 0;
+ int simBaseline = symbolDeltaBaseline();
+ int bodyIdx = 0;
for (int i = 0, n = keys.size(); i < n; i++) {
CharSequence tableName = keys.getQuick(i);
if (tableName == null) {
continue;
}
QwpTableBuffer tableBuffer = tableBuffers.get(tableName);
- if (tableBuffer != null && tableBuffer.getRowCount() > 0) {
- nonEmptyCount++;
+ if (tableBuffer == null || tableBuffer.getRowCount() == 0) {
+ continue;
+ }
+ nonEmptyCount++;
+ int deltaStart = simBaseline + 1;
+ int deltaCount = Math.max(0, currentBatchMaxSymbolId - simBaseline);
+ int messageSize = QwpConstants.HEADER_SIZE
+ + NativeBufferWriter.varintSize(deltaStart)
+ + NativeBufferWriter.varintSize(deltaCount)
+ + (deltaCount > 0 ? combinedDeltaEntriesLen : 0)
+ + splitFrameBodyBytes.getQuick(bodyIdx);
+ bodyIdx++;
+ if (messageSize > cap) {
+ resetTableBuffersAfterFlush(keys);
+ throw new LineSenderException("single table batch too large for server batch cap")
+ .put(" [table=").put(tableName)
+ .put(", messageSize=").put(messageSize)
+ .put(", serverMaxBatchSize=").put(cap).put(']');
+ }
+ // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's
+ // new ids, the remaining frames carry an empty delta above the baseline.
+ if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) {
+ simBaseline = currentBatchMaxSymbolId;
}
}
@@ -3514,25 +3651,39 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm
boolean deferThis = deferCommit || !isLast;
encoder.setDeferCommit(deferThis);
+ // Each split frame emits the delta above sentMaxSymbolId; the first
+ // frame ships the whole batch's new ids and advances the baseline, so
+ // the remaining frames carry an empty delta and just reference ids the
+ // first frame already registered.
encoder.beginMessage(1, globalSymbolDictionary,
- /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId);
+ symbolDeltaBaseline(), currentBatchMaxSymbolId);
encoder.addTable(tableBuffer);
int messageSize = encoder.finishMessage();
QwpBufferWriter buffer = encoder.getBuffer();
-
- if (messageSize > serverMaxBatchSize) {
- resetTableBuffersAfterFlush(keys);
- throw new LineSenderException("single table batch too large for server batch cap")
- .put(" [table=").put(tableName)
- .put(", messageSize=").put(messageSize)
- .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']');
- }
-
+ // The pre-flight pass above already verified every split frame fits the
+ // cap, so none can be found oversized here -- which is what keeps this
+ // loop from publishing (and stranding) a deferred prefix before an
+ // oversized table. Both passes size against the SAME snapshot cap, so a
+ // mid-flush failover cannot make them disagree; the assert therefore only
+ // catches a genuine divergence between the pre-flight arithmetic and the
+ // real encode (a future bug), not a cap race. It deliberately does NOT
+ // reset+throw here, because by this point a prefix may already be on the ring.
+ assert messageSize <= cap
+ : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName
+ + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']';
+
+ // Write-ahead persist before publish (see flushPendingRows). The
+ // first split frame carries the batch's new symbols; the rest are
+ // no-ops once the baseline has advanced past them.
+ persistNewSymbolsBeforePublish();
ensureActiveBufferReady();
activeBuffer.ensureCapacity(messageSize);
activeBuffer.write(buffer.getBufferPtr(), messageSize);
activeBuffer.incrementRowCount();
sealAndSwapBuffer();
+ // Frame queued: advance so the next split frame's delta starts above
+ // the ids this one just registered.
+ advanceSentMaxSymbolId();
}
encoder.setDeferCommit(false);
@@ -3573,8 +3724,23 @@ private void sendCommitMessage() {
LOG.debug("Sending commit message for deferred batch");
}
encoder.setDeferCommit(false);
+ // A commit carries no rows, and it must also carry NO new symbols. Unlike
+ // the flush paths, sendCommitMessage does NOT write-ahead-persist the
+ // dictionary, so shipping a symbol here would put an id on the wire that a
+ // recovered slot cannot rebuild from the persisted .symbol-dict, diverging
+ // the producer dictionary from the surviving frames and silently
+ // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit
+ // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back
+ // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at
+ // what has already been sent -- and therefore already persisted. In delta
+ // mode pass sentMaxSymbolId, yielding an empty delta
+ // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep
+ // currentBatchMaxSymbolId so the frame stays self-sufficient. Any symbol a
+ // cancelled row leaked is picked up (and persisted) by the next real flush,
+ // whose persistNewSymbolsBeforePublish resumes from pd.size().
+ int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId;
encoder.beginMessage(0, globalSymbolDictionary,
- /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId);
+ symbolDeltaBaseline(), commitBatchMaxId);
int messageSize = encoder.finishMessage();
QwpBufferWriter buffer = encoder.getBuffer();
ensureActiveBufferReady();
@@ -3586,15 +3752,190 @@ private void sendCommitMessage() {
lastCommitBoundaryFsn = cursorEngine.publishedFsn();
}
+ /**
+ * Advances the delta baseline once a frame carrying the current batch's
+ * symbols has been queued onto the ring. No-op in full-dict mode. Only ever
+ * moves the baseline forward, so a batch that used no new symbols leaves it
+ * unchanged.
+ */
+ private void advanceSentMaxSymbolId() {
+ if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) {
+ sentMaxSymbolId = currentBatchMaxSymbolId;
+ }
+ }
+
+ /**
+ * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 ..
+ * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the
+ * frame is published to the ring. This write-ahead ordering keeps the
+ * persisted dictionary a superset of every process-crash-recoverable frame's
+ * references, so recovery and orphan-drain can re-register it on a fresh
+ * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it
+ * is caught by the send loop's replay guard. No-op in memory mode (no
+ * persisted dictionary) and when the frame introduces no new symbols.
+ */
+ private void persistNewSymbolsBeforePublish() {
+ if (!deltaDictEnabled || cursorEngine == null) {
+ return;
+ }
+ PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict();
+ if (pd == null) {
+ return;
+ }
+ // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the
+ // frame is published.
+ //
+ // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1:
+ // the persist advances pd.size() only after a full write, whereas
+ // sentMaxSymbolId only advances after the WHOLE frame is published (via
+ // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist
+ // threw (short write -- disk full/quota) or the publish threw, the frame
+ // was not published and sentMaxSymbolId stayed put, while the symbols
+ // before the failure are already on disk. Keying the resume point off
+ // sentMaxSymbolId+1 would re-append that persisted prefix on the retry,
+ // duplicating entries and corrupting the dense id->symbol mapping recovery
+ // relies on (position i must be symbol id i). pd.size() resumes exactly
+ // past what is already durable, so the write-ahead is idempotent.
+ int from = pd.size();
+ if (currentBatchMaxSymbolId < from) {
+ return; // nothing new to persist (warm batch, or an idempotent retry)
+ }
+ // Fast path: the frame the encoder just built already holds these symbols
+ // in its delta section as [len][utf8]... -- byte-identical to what
+ // PersistedSymbolDict stores. In the common case pd.size() equals the
+ // frame's delta start id (sentMaxSymbolId+1), so persist those bytes
+ // straight from the frame instead of re-encoding the symbols. After a
+ // failed publish the durable size has run ahead of the wire baseline, so
+ // the frame's delta covers MORE than remains to persist; then re-encode
+ // just the [from .. currentBatchMaxSymbolId] suffix.
+ try {
+ if (from == sentMaxSymbolId + 1) {
+ QwpBufferWriter buffer = encoder.getBuffer();
+ pd.appendRawEntries(
+ buffer.getBufferPtr() + encoder.getDeltaEntriesStart(),
+ encoder.getDeltaEntriesLen(),
+ currentBatchMaxSymbolId - from + 1);
+ } else {
+ pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId);
+ }
+ } catch (Throwable t) {
+ // A short write (disk full / quota) to the persisted dictionary throws
+ // a low-level IllegalStateException. Surface it as a LineSenderException
+ // -- like every other flush-path failure, e.g. the cursor append in
+ // sealAndSwapBuffer -- so a caller catching LineSenderException around
+ // flush() also catches a disk-full during the write-ahead persist. The
+ // persist ran before publish and pd.size() did not advance on the short
+ // write, so the still-buffered rows re-persist the same range
+ // idempotently on retry. A JVM Error is never a persist failure; let it
+ // propagate.
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ throw new LineSenderException("failed to persist symbol dictionary before publish", t);
+ }
+ }
+
private void resetSymbolDictStateForNewConnection() {
- // The new server has an empty symbol dictionary, so the next batch
- // must ship a delta starting at id 0. beginMessage() always passes
- // confirmedMaxId = -1; resetting the batch watermark here keeps a
- // stale value from suppressing re-emission of symbol ids the new
- // server has never seen.
+ // Runs on the foreground (initial) connect only -- NOT on the I/O thread's
+ // reconnect/failover path. The per-batch watermark is drained state, so
+ // clearing it here is harmless. sentMaxSymbolId is deliberately left
+ // untouched: in delta mode the I/O thread re-registers the whole
+ // dictionary with a catch-up frame on reconnect, so the producer's
+ // monotonic baseline must survive the wire boundary; resetting it would
+ // desync the producer from the I/O thread's sent-dictionary count.
currentBatchMaxSymbolId = -1;
}
+ /**
+ * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} from
+ * the slot's persisted dictionary (ids assigned in the same ascending order,
+ * so they match the recovered frames) and resumes the delta baseline at the
+ * recovered tip, so newly ingested symbols continue above the recovered ids.
+ *
+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup):
+ * the persisted dictionary, the on-wire delta and the send-loop catch-up mirror
+ * all key on the entry POSITION (id), so the producer id space must match the
+ * persisted entry count exactly. {@code getOrAddSymbol} would collapse two
+ * source strings that decode to the same characters -- only malformed lone
+ * UTF-16 surrogates, which UTF-8-encode to {@code '?'} -- leaving this
+ * dictionary shorter than {@code pd.size()} and desyncing
+ * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()},
+ * which silently misattributes later symbols after a reconnect.
+ *
+ * Host-crash tear guard. The persisted dictionary is NOT fsync'd (see
+ * {@code PersistedSymbolDict}), so a host/power crash can lose its
+ * most-recently-written (highest-id) entries while the segment frames that
+ * introduced those ids survive -- and those newest frames, being the least
+ * likely to be acked, replay on recovery. The send loop's catch-up mirror then
+ * rebuilds the missing ids from those frames' own delta bytes, but THIS producer
+ * -- seeded only from the shorter dictionary -- would assign its next new symbol
+ * an id the surviving frames already define, putting two symbols on one id and
+ * silently misattributing values. The send loop's replay guard only catches a
+ * GAP ({@code deltaStart > sentDictCount}); a frame that introduces exactly the
+ * torn-off id ({@code deltaStart == pd.size()}) slips through and self-heals the
+ * mirror, leaving only this producer diverged. Detect it here -- the surviving
+ * frames reference an id at or beyond the recovered dictionary size -- and fail
+ * clean: the affected data must be resent, matching the design's torn-dict
+ * "resend required" contract.
+ *
+ * The background drainer self-heals the mirror ONLY when a surviving frame
+ * STRADDLES the tear ({@code deltaStart <= pd.size() < deltaStart + deltaCount}):
+ * such a frame carries the torn-off ids in its own delta and
+ * {@code accumulateSentDict} re-registers them, so the drainer drains the slot.
+ * But when the symbol-introducing frames were already acked and trimmed and only
+ * a HIGHER-baseline frame survives ({@code deltaStart > pd.size()} -- e.g. a
+ * commit or a symbol-reusing frame, since {@code beginMessage} always sets the
+ * delta flag), the drainer's own replay guard ({@code deltaStart > sentDictCount})
+ * fires too and quarantines the slot: the recorded bytes are not silently lost,
+ * but the slot is NOT auto-drained -- it must be resent. That is a deliberate
+ * CONSERVATIVE over-strand -- the guard keys on {@code deltaStart}, not on the
+ * frame's actual highest referenced id, to avoid parsing row data at recovery,
+ * so it may reject a frame whose rows reference only ids the truncated
+ * dictionary still holds. It fails clean rather than risk a silent id shift.
+ * Only a host crash reaches this -- a process crash keeps the page cache, so the
+ * write-ahead ordering keeps the dictionary a superset of the frames.
+ */
+ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
+ if (pd == null) {
+ return;
+ }
+ // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit
+ // below: a TOTAL tear (pd.size() == 0) with surviving symbol-bearing frames
+ // must fail clean too, not slip through. Such a frame starts at deltaStart=0
+ // and self-heals the I/O-thread catch-up mirror, so the send loop's replay
+ // guard (deltaStart > sentDictCount) never fires -- this seed-time guard is
+ // then the only defense against the producer resuming unseeded and silently
+ // reusing ids the frames already define. A genuinely empty slot (no
+ // symbol-bearing frames) has recoveredMaxSymbolId() == -1, so -1 >= 0 is false
+ // and the pd.size() == 0 return below still fires.
+ if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) {
+ throw new LineSenderException(
+ "recovered store-and-forward symbol dictionary is a subset of the surviving frames "
+ + "(likely a host crash tore its unsynced tail): frames reference symbol id "
+ + cursorEngine.recoveredMaxSymbolId() + " but the recovered dictionary holds only "
+ + pd.size() + " id(s); resuming would reuse ids the frames already define -- "
+ + "resend the affected data");
+ }
+ if (pd.size() == 0) {
+ return;
+ }
+ ObjList symbols = pd.readLoadedSymbols();
+ for (int i = 0, n = symbols.size(); i < n; i++) {
+ globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i));
+ }
+ sentMaxSymbolId = globalSymbolDictionary.size() - 1;
+ }
+
+ /**
+ * The symbol id below which the server already holds every dictionary entry,
+ * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is
+ * the producer's monotonic sent watermark; in full-dict mode it is -1 so every
+ * frame re-ships the dictionary from id 0.
+ */
+ private int symbolDeltaBaseline() {
+ return deltaDictEnabled ? sentMaxSymbolId : -1;
+ }
+
private void rollbackRow() {
if (currentTableBuffer != null) {
currentTableBuffer.cancelCurrentRow();
@@ -3656,7 +3997,7 @@ private void sealAndSwapBuffer() {
// back to it; flushPendingRows aborts its post-enqueue state
// updates after this throw, so the source rows stay intact and the
// next batch re-emits the same rows along with the full inline
- // schema and symbol-dict delta from id 0.
+ // schema and the symbol-dict delta the batch requires.
if (toSend.isSending()) {
toSend.markRecycled();
} else if (toSend.isSealed()) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 64ca75d0..dde24142 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -84,9 +84,9 @@ public final class CursorSendEngine implements QuietCloseable {
private final SlotLock slotLock;
// True when the constructor recovered an existing on-disk slot rather
// than starting fresh. Diagnostic accessor for tests and observability;
- // cursor frames are self-sufficient (every frame carries full schema +
- // full symbol-dict delta), so producer-side schema reset on recovery
- // is not required.
+ // every frame carries its full inline schema, so producer-side schema reset
+ // on recovery is not required (the symbol dictionary, which delta frames do
+ // NOT carry in full, is re-registered by an I/O-thread catch-up instead).
private final boolean wasRecoveredFromDisk;
// FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a
// ring recovered from disk, or -1 for fresh/memory rings and recovered
@@ -96,6 +96,15 @@ public final class CursorSendEngine implements QuietCloseable {
// covers them. Read by the sender's close-time drain to avoid waiting on
// acks that cannot arrive.
private long recoveredCommitBoundaryFsn = -1L;
+ // Highest symbol id any recovered delta frame references, or -1 for
+ // fresh/memory rings (and recovered rings with no symbol-bearing frame). A
+ // resuming producer seeds its dictionary baseline from the persisted
+ // .symbol-dict; if that dictionary was torn below this id by a host crash
+ // (the side-file is not fsync'd), the producer would re-use ids the surviving
+ // frames already define. seedGlobalDictionaryFromPersisted compares this
+ // against the recovered dictionary size to fail clean instead. Computed once
+ // in the constructor's recovery branch; -1 elsewhere.
+ private long recoveredMaxSymbolId = -1L;
// FSN of the last frame of a recovered orphaned deferred tail, or -1 when
// the recovered ring has no such tail. When >= 0, frames
// [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry
@@ -110,6 +119,13 @@ public final class CursorSendEngine implements QuietCloseable {
// in the constructor, closed by {@link #close()}. The segment manager
// writes through this on every tick where ackedFsn has advanced.
private final AckWatermark watermark;
+ // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null}
+ // in memory mode and if open() failed). Enables delta-encoded SF frames:
+ // recovery / orphan-drain load it to re-register the dictionary on the fresh
+ // server before replaying non-self-sufficient frames. Opened in the
+ // constructor, closed by {@link #close()}. When null in disk mode the engine
+ // reports delta encoding as unavailable and the sender keeps full-dict frames.
+ private final PersistedSymbolDict persistedSymbolDict;
// close() is publicly callable from any thread (Sender.close from a user
// thread, JVM shutdown hooks, test cleanup). volatile + synchronized
// close() makes the check-and-set atomic and gives readers a fence.
@@ -199,6 +215,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
// reference instead of orphaning the mmap'd segments + fds.
SegmentRing ringInProgress = null;
AckWatermark watermarkInProgress = null;
+ PersistedSymbolDict persistedDictInProgress = null;
try {
// Disk mode: try to recover any *.sfa files left behind by a prior
// session before deciding to start fresh. Without this the engine
@@ -257,6 +274,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
// mmap doesn't take down the engine -- we just fall
// back to the bare lowestBase - 1 seed.
watermarkInProgress = AckWatermark.open(sfDir);
+ // Load the persisted symbol dictionary so delta-encoded frames
+ // in this recovered slot can be re-registered on the fresh
+ // server before replay. Null on open failure -> delta disabled.
+ persistedDictInProgress = PersistedSymbolDict.open(sfDir);
long baseSeed = lowestBase - 1;
long watermarkFsn = watermarkInProgress != null
? watermarkInProgress.read()
@@ -300,6 +321,22 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L),
recoveredCommitBoundaryFsn, publishedFsn);
}
+ // Highest symbol id the surviving COMMITTED frames reference. A
+ // resuming producer compares this against its recovered dictionary
+ // size (seedGlobalDictionaryFromPersisted) to detect a host-crash
+ // tear: if a committed frame references an id the (unsynced, torn)
+ // .symbol-dict no longer holds, resuming would re-use it. The walk is
+ // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred
+ // tail -- retired without ever being transmitted -- does not inflate
+ // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd
+ // returns 0 when no such frame carries a symbol, yielding -1 here.
+ // Computed before the I/O loop or producer append; single-threaded.
+ this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd(
+ io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE,
+ io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS,
+ io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT,
+ io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE,
+ recoveredCommitBoundaryFsn) - 1L;
} else {
// Fresh start with no recovered segments. Any stale
// watermark from a prior fully-drained session refers
@@ -309,6 +346,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
if (!memoryMode) {
AckWatermark.removeOrphan(sfDir);
watermarkInProgress = AckWatermark.open(sfDir);
+ // A fresh slot MUST start with an EMPTY symbol dictionary.
+ // Unlike the ack watermark above -- a discardable optimization a
+ // max() clamp protects -- the dictionary is load-bearing: a
+ // delta frame referencing an id missing from it is unrecoverable,
+ // and a STALE dictionary inherited here (the segments are gone, so
+ // the producer is NOT seeded from it) shifts the dense id->symbol
+ // mapping and silently misattributes symbols on the next
+ // reconnect. openClean() truncates any survivor to empty rather
+ // than trusting a best-effort delete that may have failed (e.g. a
+ // Windows share lock); if the clean open itself fails,
+ // persistedSymbolDict stays null and the sender falls back to full
+ // self-sufficient frames, which is also safe.
+ persistedDictInProgress = PersistedSymbolDict.openClean(sfDir);
}
MmapSegment initial;
String initialPath = null;
@@ -333,10 +383,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
manager.start();
}
manager.register(ringInProgress, sfDir, watermarkInProgress);
- // All construction succeeded โ commit the ring and
- // watermark references.
+ // All construction succeeded โ commit the ring, watermark and
+ // symbol-dictionary references.
this.ring = ringInProgress;
this.watermark = watermarkInProgress;
+ this.persistedSymbolDict = persistedDictInProgress;
} catch (Throwable t) {
// Stop an owned manager before freeing the ring and watermark it may
// touch, then release the slot lock. Each cleanup is in its own
@@ -362,6 +413,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man
} catch (Throwable ignored) {
}
}
+ if (persistedDictInProgress != null) {
+ try {
+ persistedDictInProgress.close();
+ } catch (Throwable ignored) {
+ }
+ }
if (acquiredLock != null) {
try {
acquiredLock.close();
@@ -541,6 +598,12 @@ public synchronized void close() {
} catch (Throwable ignored) {
}
}
+ if (persistedSymbolDict != null) {
+ try {
+ persistedSymbolDict.close();
+ } catch (Throwable ignored) {
+ }
+ }
if (fullyDrained) {
try {
unlinkAllSegmentFiles(sfDir);
@@ -550,6 +613,11 @@ public synchronized void close() {
AckWatermark.removeOrphan(sfDir);
} catch (Throwable ignored) {
}
+ try {
+ // Slot fully drained: the dictionary has no frames behind it.
+ PersistedSymbolDict.removeOrphan(sfDir);
+ } catch (Throwable ignored) {
+ }
}
} finally {
if (slotLock != null) {
@@ -576,6 +644,15 @@ public MmapSegment firstSealed() {
return ring.firstSealed();
}
+ /**
+ * The engine's persisted symbol dictionary, or {@code null} in memory mode
+ * (and in disk mode if it failed to open). The producer appends new symbols
+ * to it; recovery / orphan-drain read its loaded entries to seed catch-up.
+ */
+ public PersistedSymbolDict getPersistedSymbolDict() {
+ return persistedSymbolDict;
+ }
+
/**
* Number of times {@link #appendBlocking} hit
* {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and
@@ -586,6 +663,18 @@ public long getTotalBackpressureStalls() {
return backpressureStallCount.get();
}
+ /**
+ * Whether the sender may delta-encode symbol dictionaries on this engine.
+ * Always true in memory mode (the send loop keeps an in-process catch-up
+ * mirror). In disk mode it requires the persisted dictionary to have opened,
+ * since delta frames are not self-sufficient and recovery / orphan-drain must
+ * be able to rebuild the dictionary from disk. When false in disk mode the
+ * sender falls back to full self-sufficient frames.
+ */
+ public boolean isDeltaDictEnabled() {
+ return sfDir == null || persistedSymbolDict != null;
+ }
+
/**
* Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}.
*/
@@ -650,6 +739,16 @@ public long recoveredCommitBoundaryFsn() {
return recoveredCommitBoundaryFsn;
}
+ /**
+ * Highest symbol id any recovered delta frame references, or {@code -1} for
+ * fresh/memory rings (and recovered rings with no symbol-bearing frame). A
+ * resuming producer compares this against its recovered dictionary size to
+ * detect a host-crash tear of the persisted {@code .symbol-dict}.
+ */
+ public long recoveredMaxSymbolId() {
+ return recoveredMaxSymbolId;
+ }
+
/**
* FSN of the last frame of a recovered orphaned deferred tail, or
* {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 13a69f77..1ea87523 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -31,14 +31,17 @@
import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
import io.questdb.client.cutlass.http.client.WebSocketUpgradeException;
import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException;
import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException;
import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException;
import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException;
import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException;
import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode;
import io.questdb.client.std.CharSequenceLongHashMap;
+import io.questdb.client.std.MemoryTag;
import io.questdb.client.std.QuietCloseable;
import io.questdb.client.std.Unsafe;
import org.jetbrains.annotations.TestOnly;
@@ -130,6 +133,37 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
*/
public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L;
private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class);
+ // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts
+ // -- catch-ups that reached a fresh server and found a single dictionary entry
+ // too large for its advertised batch cap -- may occur, with no intervening
+ // SUCCESSFUL catch-up, before the sender latches a terminal. This is a SANCTIONED
+ // terminal (a genuine cluster batch-size capability gap), the connect-time analog
+ // of the orphan drainer's durable-ack capability gap
+ // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips
+ // it -- an entry that fit its data frame under a cap always fits its bare catch-up
+ // frame under that same cap -- so it only affects a heterogeneous / rolling-cap
+ // cluster, where a failover to a smaller-cap node can hit it for an entry an
+ // earlier node accepted. Retrying rides out the transient window until a
+ // larger-cap node returns; only a persistent gap (this many cap gaps with no
+ // successful catch-up in between) latches.
+ //
+ // Budget accounting (satisfies "a transient must never burn the terminal
+ // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a
+ // node is reached and an entry is oversized, and resets ONLY when a catch-up fully
+ // succeeds. A TRANSIENT reconnect (connect refuse, upgrade/role failure) never
+ // reaches the catch-up, so it NEITHER increments nor resets the counter -- it only
+ // lengthens the wall-clock settle window. The terminal therefore always requires
+ // this many GENUINE cap gaps; a transient can never inflate it. Deliberately NOT
+ // reset on a mere successful RECONNECT: a reconnect to the small-cap node itself
+ // produces the cap gap, so resetting there would stop a persistent gap from ever
+ // latching -- the reset must gate on a successful CATCH-UP, not on connecting.
+ private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16;
+ // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror
+ // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even
+ // this needs ~200M+ distinct symbols on a single connection, far past any real
+ // workload. The guard exists so that pathological growth fails loudly instead
+ // of overflowing the int capacity math into a heap-corrupting copyMemory.
+ private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8;
/**
* Throttle "reconnect attempt N failed" WARN logs to one per 5 s.
*/
@@ -177,6 +211,51 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
private final long reconnectMaxDurationMillis;
private final WebSocketResponse response = new WebSocketResponse();
private final ResponseHandler responseHandler = new ResponseHandler();
+ // Delta symbol dictionary catch-up state (see swapClient). Active in memory
+ // mode, and in disk mode whenever the per-slot persisted dictionary opened --
+ // fresh slots included, not just recovered / orphan-drained ones. On a
+ // recovered / orphan-drained slot the constructor additionally SEEDS sentDict*
+ // from that persisted dictionary; a fresh slot starts with an empty mirror and
+ // grows it as frames are sent.
+ // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every
+ // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in
+ // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that
+ // on reconnect it can re-register the whole dictionary on the fresh server
+ // (which discards its dictionary on every disconnect) before replaying frames
+ // whose deltas start above id 0. All of this is touched only by the I/O thread.
+ // Footprint note: this mirror is a SECOND copy of the dictionary -- the same
+ // symbols the producer's GlobalSymbolDictionary already holds as Java Strings --
+ // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a
+ // memory-mode connection's steady-state dictionary footprint is ~2x the symbol
+ // set. It is bounded by distinct-symbol count (not per-row) and never trimmed
+ // for the connection's lifetime (a reconnect may need the whole dictionary at
+ // any moment), so it cannot be dropped; it is an intentional cost of the feature.
+ private final boolean deltaDictEnabled;
+ // Cap-gap attempts -- catch-ups that reached a node and found an entry too large
+ // for its batch cap -- since the last SUCCESSFUL catch-up (see
+ // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful
+ // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments
+ // nor resets it. NOT reset per connection -- it measures the cap-gap episode
+ // across reconnects so a persistent gap eventually latches. I/O-thread-only.
+ private int catchUpCapGapAttempts;
+ // True once a real ring frame (data or commit) has been sent on the CURRENT
+ // connection, as opposed to only the dictionary catch-up. The catch-up
+ // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies
+ // "the head frame was sent": onClose's poison-strike gate and
+ // handleServerRejection's pre-send gate key off THIS instead. Without it, a
+ // transient outage AFTER the catch-up but BEFORE the first data frame (a
+ // flapping LB/middlebox that accepts the upgrade + catch-up then closes) would
+ // be mistaken for a deterministic head-frame rejection and escalate to a
+ // PROTOCOL_VIOLATION terminal -- breaking the store-and-forward "retry a
+ // transient outage forever" contract. Reset per connection in
+ // setWireBaselineWithCatchUp; set in trySendOne after a successful send.
+ private boolean dataFrameSentThisConnection;
+ private long sentDictBytesAddr;
+ private int sentDictBytesCapacity;
+ private int sentDictBytesLen;
+ private int sentDictCount;
+ // End position (native address) written by the last readVarintAt() call.
+ private long varintEnd;
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
private final AtomicLong totalAcks = new AtomicLong();
// Counters for observability of the durable-ack path. Both are zero
@@ -489,6 +568,41 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
}
this.client = client;
this.engine = engine;
+ this.deltaDictEnabled = engine.isDeltaDictEnabled();
+ // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror,
+ // so seed it from the slot's persisted dictionary. That way the very first
+ // connection re-registers the whole dictionary (via a catch-up frame)
+ // before replaying the recovered delta frames.
+ if (deltaDictEnabled) {
+ PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+ if (pd != null && pd.size() > 0) {
+ int len = pd.loadedEntriesLen();
+ if (len > 0) {
+ // COPY the persisted dictionary's loaded-entries buffer into this
+ // loop's own mirror rather than taking ownership of it. The engine
+ // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan
+ // drainer path: BackgroundDrainer builds a fresh send loop per wire
+ // session against the same engine on a durable-ack capability-gap
+ // recycle. A one-shot ownership transfer would leave every loop
+ // after the first with an EMPTY mirror -- it would then send no
+ // reconnect catch-up, and the first replayed delta frame
+ // (deltaStart > 0) would trip the torn-dict guard, falsely
+ // quarantining a healthy slot. Copying keeps the dictionary's
+ // loaded entries intact for the engine's lifetime so every
+ // recycled loop re-seeds; pd.close() (at engine close) frees the
+ // dictionary's copy, this loop frees its own copy on exit.
+ sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+ Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len);
+ sentDictBytesCapacity = len;
+ sentDictBytesLen = len;
+ // Set the count only alongside the bytes so sentDictCount can
+ // never claim symbols the mirror does not hold. A recovered slot
+ // always has loadedEntriesLen > 0 when size > 0, so this is the
+ // same result -- it just makes the coupling explicit.
+ sentDictCount = pd.size();
+ }
+ }
+ }
this.fsnAtZero = fsnAtZero;
this.parkNanos = parkNanos;
this.reconnectFactory = reconnectFactory;
@@ -754,6 +868,12 @@ public synchronized void close() {
// after โ the latch await is only skipped when the loop never ran.
running = false;
Thread t = ioThread;
+ // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets
+ // freed on ioLoop's exit path. When t == null the loop never ran (start()
+ // was never called, or t.start() failed before committing ioThread), so
+ // that free never happens and a seeded (recovery / orphan-drain) mirror
+ // would leak. Capture that here; free it below, after client teardown.
+ boolean loopNeverRan = t == null;
if (t != null) {
LockSupport.unpark(t);
// Only await the shutdown latch if the I/O thread actually ran.
@@ -812,6 +932,22 @@ public synchronized void close() {
}
client = null;
}
+ // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never
+ // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and
+ // on the failed-stop path (interrupted latch await, ioThread left set) the
+ // thread may still be mid-send, so touching the mirror here would race.
+ // A duplicate close observes sentDictBytesAddr == 0 and skips.
+ if (loopNeverRan && sentDictBytesAddr != 0) {
+ Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
+ sentDictBytesAddr = 0;
+ sentDictBytesCapacity = 0;
+ sentDictBytesLen = 0;
+ // Reset the count alongside the buffer so the mirror stays all-or-
+ // nothing: a hypothetical close()-then-start() (start() has no closed
+ // guard) must not observe a non-zero sentDictCount against a freed
+ // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up.
+ sentDictCount = 0;
+ }
}
/**
@@ -996,7 +1132,28 @@ public synchronized void start() {
// walks back to the lowest unacked frame so sealed-segment data
// actually reaches the wire โ without it, start() would skip
// straight to the active and orphan everything in sealed.
- positionCursorForStart();
+ try {
+ positionCursorForStart();
+ } catch (CatchUpSendException e) {
+ // A recovered sender re-registers its dictionary with a catch-up on
+ // the very first connect. Here that runs on the CALLER thread (sync
+ // start), so we must NOT let it drive connectLoop -- that would block
+ // Sender construction forever on a transient outage. Drop the dead
+ // client instead: the I/O thread then reconnects via
+ // attemptInitialConnect -> swapClient and re-sends the catch-up off
+ // this thread. If the failure was already terminal (recordFatal set
+ // running=false, e.g. an entry too large for the batch cap), the I/O
+ // thread simply winds down and checkError() surfaces it.
+ WebSocketClient dead = client;
+ client = null;
+ if (dead != null) {
+ try {
+ dead.close();
+ } catch (Throwable ignored) {
+ // best-effort
+ }
+ }
+ }
Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io");
t.setDaemon(true);
try {
@@ -1669,6 +1826,15 @@ private void ioLoop() {
// best-effort
}
}
+ // The symbol-dict mirror is I/O-thread-owned; free it here, on the
+ // owning thread's exit path, after the last send that could touch it.
+ if (sentDictBytesAddr != 0) {
+ Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
+ sentDictBytesAddr = 0;
+ sentDictBytesCapacity = 0;
+ sentDictBytesLen = 0;
+ sentDictCount = 0; // keep the mirror all-or-nothing (see close())
+ }
shutdownLatch.countDown();
// Failed-stop hand-off (see delegateEngineClose): the owner could
// not free the engine safely while this thread was alive, so the
@@ -1691,8 +1857,11 @@ private void ioLoop() {
/**
* Walk the engine's segments to find the one containing {@code targetFsn},
* and set {@code sendOffset} to the byte offset of that frame within it.
- * This is called at startup and after every reconnect, after fsnAtZero has
- * already been reset to {@code targetFsn} and nextWireSeq to 0.
+ * This is called at startup and after every reconnect, once
+ * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline
+ * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq}
+ * past the catch-up frames it emitted. This method only positions the byte
+ * cursor at {@code targetFsn}; it does not touch the wire mapping.
*
* If {@code targetFsn} is already published, the method positions the byte
* cursor exactly at that frame. If {@code targetFsn} is not published yet,
@@ -1849,8 +2018,6 @@ private void swapClient(WebSocketClient newClient) {
// past the tail instead of replaying into it.
tryRetireOrphanTail();
long replayStart = engine.ackedFsn() + 1L;
- this.fsnAtZero = replayStart;
- this.nextWireSeq = 0L;
// Snapshot publishedFsn at swap time โ frames at FSN โค this value
// were already on the wire before the drop and will be replayed.
// trySendOne resets replayTargetFsn to -1 once we cross the boundary.
@@ -1862,9 +2029,374 @@ private void swapClient(WebSocketClient newClient) {
// carrying stale state across the wire boundary would either
// double-trim or starve the queue.
clearDurableAckTracking();
+ setWireBaselineWithCatchUp(replayStart);
positionCursorAt(replayStart);
}
+ /**
+ * Sets the wire-sequence baseline for a fresh connection and, when the symbol
+ * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first
+ * so the fresh server (whose dictionary starts empty) can resolve the
+ * non-self-sufficient delta frames that replay next.
+ *
+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just
+ * below {@code replayStart} (a harmless re-ack); real replay frames then follow
+ * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode),
+ * or before a client exists (async initial connect), keep the plain 1:1
+ * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the
+ * first real connection via swapClient.
+ */
+ private void setWireBaselineWithCatchUp(long replayStart) {
+ // Fresh connection: no data frame has been sent on it yet. Reset before the
+ // catch-up (which sends only dictionary frames) so onClose /
+ // handleServerRejection can tell "only the catch-up went out" from "the
+ // head data frame went out".
+ dataFrameSentThisConnection = false;
+ if (client != null && deltaDictEnabled && sentDictCount > 0) {
+ this.nextWireSeq = 0L;
+ // The catch-up may span several frames when the dictionary exceeds the
+ // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps
+ // to an already-acked FSN, so the first real frame still lands on
+ // replayStart.
+ int catchUpFrames = sendDictCatchUp();
+ this.fsnAtZero = replayStart - catchUpFrames;
+ } else {
+ this.fsnAtZero = replayStart;
+ this.nextWireSeq = 0L;
+ }
+ }
+
+ /**
+ * Returns the symbol-dictionary delta start id of a frame, or -1 when the
+ * frame carries no delta section. Used by the pre-send torn-dictionary guard.
+ */
+ private int frameDeltaStart(long payloadAddr, int payloadLen) {
+ if (!isDeltaFrame(payloadAddr, payloadLen)) {
+ return -1;
+ }
+ return (int) readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen);
+ }
+
+ // True only for a well-formed QWP frame this encoder produced that carries a
+ // delta symbol-dict section. The magic check keeps the dict logic from
+ // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose
+ // bytes happen to set the delta flag.
+ private static boolean isDeltaFrame(long payloadAddr, int payloadLen) {
+ if (payloadLen < QwpConstants.HEADER_SIZE
+ || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) {
+ return false;
+ }
+ byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS);
+ return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0;
+ }
+
+ /**
+ * Copies the symbol-dictionary delta a just-sent frame carries into the
+ * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can
+ * re-register it. Frames are sent in FSN order carrying monotonically
+ * extending deltas, so a frame whose delta starts exactly at
+ * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame
+ * (nothing new) is skipped. Only ever called in delta mode, for a frame the
+ * pre-send guard already classified as a delta frame.
+ *
+ * @param payloadAddr address of the QWP message (12-byte header first)
+ * @param payloadLen message length in bytes
+ * @param deltaStart the frame's delta start id, already decoded by the
+ * pre-send guard ({@link #frameDeltaStart}) -- passed in so
+ * the magic/flags and start-id varint are not re-parsed
+ */
+ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) {
+ long limit = payloadAddr + payloadLen;
+ // deltaStart is known (the guard decoded it); locate deltaCount just past
+ // its canonical LEB128 encoding rather than re-reading the header and the
+ // start-id varint.
+ long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart);
+ long deltaCount = readVarintAt(p, limit);
+ p = varintEnd;
+ // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this
+ // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the
+ // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases:
+ // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard
+ // rejects it before send, so it never reaches here; bail defensively
+ // rather than accumulate past a hole.
+ // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold --
+ // nothing new.
+ // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail.
+ // deltaStart == sentDictCount is the steady-state case (skip == 0).
+ // Handling the partial overlap explicitly -- rather than dropping the whole
+ // frame whenever deltaStart != sentDictCount -- keeps the mirror complete
+ // even if a future producer ever emits a delta that overlaps the tip;
+ // silently dropping the new tail would leave the reconnect catch-up
+ // incomplete and shift server-side ids.
+ long deltaEnd = deltaStart + deltaCount;
+ if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) {
+ return;
+ }
+ // Walk past the already-held prefix [deltaStart, sentDictCount), then copy
+ // the new tail [sentDictCount, deltaEnd).
+ int skip = sentDictCount - deltaStart;
+ for (int i = 0; i < skip; i++) {
+ long len = readVarintAt(p, limit);
+ p = varintEnd + len;
+ if (p > limit) {
+ return; // malformed -- bail rather than corrupt the mirror
+ }
+ }
+ long regionStart = p;
+ long newCount = deltaEnd - sentDictCount;
+ for (long i = 0; i < newCount; i++) {
+ long len = readVarintAt(p, limit);
+ p = varintEnd + len;
+ if (p > limit) {
+ // Malformed -- never happens for frames we encoded; bail rather
+ // than corrupt the mirror.
+ return;
+ }
+ }
+ int regionBytes = (int) (p - regionStart);
+ // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on
+ // a very-high-cardinality lifetime connection; ensureSentDictCapacity then
+ // fails loudly rather than overflowing to a negative int (which would make
+ // the capacity check pass and copyMemory scribble past the buffer).
+ ensureSentDictCapacity((long) sentDictBytesLen + regionBytes);
+ Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes);
+ sentDictBytesLen += regionBytes;
+ sentDictCount += (int) newCount;
+ }
+
+ private void ensureSentDictCapacity(long required) {
+ if (sentDictBytesCapacity >= required) {
+ return;
+ }
+ if (required > MAX_SENT_DICT_BYTES) {
+ // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER
+ // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail()
+ // -> connectLoop, which (running still true) reconnects and replays the
+ // same frame, which re-overflows the never-shrinking mirror -- an
+ // unbounded reconnect livelock rather than the "fails loudly" the
+ // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false
+ // so connectLoop's !running guard winds the loop down and checkError()
+ // surfaces the terminal, matching sendCatchUpChunk's guard for the same
+ // ceiling. The throw still unwinds past the pending copyMemory.
+ LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size ["
+ + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']');
+ recordFatal(err);
+ throw err;
+ }
+ // Grow in long to avoid the capacity*2 int overflow (negative) that would
+ // otherwise degrade the doubling near 1 GB; clamp to the int ceiling.
+ long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required));
+ if (newCap > MAX_SENT_DICT_BYTES) {
+ newCap = MAX_SENT_DICT_BYTES;
+ }
+ sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+ sentDictBytesCapacity = (int) newCap;
+ }
+
+ private long readVarintAt(long p, long limit) {
+ long value = 0;
+ int shift = 0;
+ long cur = p;
+ while (cur < limit) {
+ byte b = Unsafe.getUnsafe().getByte(cur++);
+ value |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ if (shift > 35) {
+ // Defensive bound, matching PersistedSymbolDict.decodeVarint: a
+ // canonical entry-length / delta varint is <= 5 bytes. Every caller
+ // reads freshly-encoded, CRC- or openExisting-validated bytes, so
+ // this is unreachable, but it stops a corrupt continuation run from
+ // over-shifting into a garbage length.
+ break;
+ }
+ }
+ varintEnd = cur;
+ return value;
+ }
+
+ /**
+ * Re-registers the whole symbol dictionary on a fresh connection, split into
+ * as many table-less frames as the server's advertised batch cap requires so
+ * no single frame exceeds it (a large dictionary would otherwise be rejected).
+ * Each chunk carries a contiguous id range {@code [start .. start+count)}, in
+ * order, so the server accumulates them exactly as it would the original
+ * per-frame deltas. Returns the number of frames sent (each consumed a wire
+ * sequence), so the caller can align {@code fsnAtZero}. Throws {@link
+ * CatchUpSendException} on a send error (retriable -- the caller reconnects);
+ * a single entry too large for the cap is non-retriable, so it latches a
+ * terminal before throwing.
+ */
+ private int sendDictCatchUp() {
+ int cap = client.getServerMaxBatchSize();
+ // The frame ceiling a catch-up chunk must not exceed: the server's
+ // advertised cap, or -- when the server advertises none (cap <= 0) --
+ // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE +
+ // varints + symbolsLen) cannot overflow on a pathological multi-GB
+ // dictionary (unreachable at real cardinality; defensive). Used by the
+ // single-entry terminal below, which measures the real solo frame.
+ int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES;
+ // Symbol-bytes budget for PACKING several entries into one chunk, leaving
+ // room for the 12-byte header and the two delta-section varints. Kept
+ // deliberately conservative (reserving 16 for the varints): it only makes a
+ // multi-entry chunk split marginally earlier, never over the cap. It must
+ // NOT gate the single-entry terminal -- that reserve is larger than the
+ // minimal data-frame overhead, so an entry the producer already shipped
+ // under this cap could exceed the reserve yet still fit its own catch-up
+ // frame; the terminal tests the real solo frame against frameLimit instead.
+ int budget = cap > 0
+ ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16)
+ : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16;
+ int framesSent = 0;
+ int chunkStartId = 0;
+ long chunkStartAddr = sentDictBytesAddr;
+ int chunkSymbols = 0;
+ long chunkBytes = 0;
+ long p = sentDictBytesAddr;
+ long limit = sentDictBytesAddr + sentDictBytesLen;
+ while (p < limit) {
+ long entryStart = p;
+ long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix
+ long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes]
+ long entryBytes = entryEnd - entryStart;
+ // The exact table-less frame sendCatchUpChunk would build for THIS entry
+ // alone: header + deltaStart varint (the entry's own global id) +
+ // deltaCount varint (1) + the entry bytes. Terminal only when even that
+ // solo frame exceeds the cap -- i.e. the entry genuinely cannot be
+ // re-registered. Testing the real solo frame (not the conservative
+ // packing budget above) is what keeps a HOMOGENEOUS cluster
+ // livelock-free: an entry the producer already shipped in a data frame
+ // under this cap (header + delta varints + entry + schema + >=1 row) is
+ // strictly larger than its bare catch-up frame, so it always fits here.
+ long soloFrameLen = QwpConstants.HEADER_SIZE
+ + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols)
+ + NativeBufferWriter.varintSize(1)
+ + entryBytes;
+ if (soloFrameLen > frameLimit) {
+ // Cap gap: this entry cannot be re-registered under the fresh
+ // server's advertised cap. A HOMOGENEOUS cluster never reaches here
+ // (an entry that fit its data frame under a cap always fits its bare
+ // catch-up frame under that same cap), so the only way in is a
+ // heterogeneous / rolling-cap failover to a smaller-cap node.
+ //
+ // Give the cluster a settle budget instead of latching on first
+ // sight: a larger-cap node may return, so retry across reconnects
+ // and only latch a terminal after MAX_CATCHUP_CAP_GAP_ATTEMPTS
+ // consecutive attempts still find it too large. Under budget the
+ // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with
+ // backoff and re-runs the catch-up, which resets the counter on a
+ // node that accepts it. A transient reconnect (connect/upgrade
+ // failure, role reject) never reaches the catch-up, so it neither
+ // increments nor burns this budget. On exhaustion latch via
+ // recordFatal, NOT fail() -- failing from inside the catch-up would
+ // re-enter connectLoop (see CatchUpSendException); the data must be
+ // resent after the cap is raised.
+ catchUpCapGapAttempts++;
+ boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS;
+ LineSenderException err = new LineSenderException(
+ "symbol dictionary entry too large for the server batch cap during catch-up ["
+ + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt="
+ + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + ']'
+ + (exhausted
+ ? "; the data must be resent after the cap is raised"
+ : "; retrying -- a larger-cap node may return"));
+ if (exhausted) {
+ recordFatal(err);
+ }
+ throw new CatchUpSendException(err);
+ }
+ if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) {
+ sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes);
+ framesSent++;
+ chunkStartId += chunkSymbols;
+ chunkStartAddr = entryStart;
+ chunkSymbols = 0;
+ chunkBytes = 0;
+ }
+ chunkSymbols++;
+ chunkBytes += entryBytes;
+ p = entryEnd;
+ }
+ if (chunkSymbols > 0) {
+ sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes);
+ framesSent++;
+ }
+ // The whole dictionary re-registered without a cap gap: this node accepts
+ // every entry, so the cap-gap episode (if any) is over -- reset the budget.
+ catchUpCapGapAttempts = 0;
+ return framesSent;
+ }
+
+ /**
+ * Sends one table-less catch-up frame carrying dictionary ids
+ * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link
+ * CatchUpSendException} on a send error instead of calling {@link #fail}
+ * (see that type for why the catch-up must not re-enter the reconnect loop);
+ * the caller turns it into a single, non-re-entrant reconnect.
+ */
+ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) {
+ // Compute the frame size in long and fail loud if it would overflow the int
+ // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each
+ // chunk's symbol bytes under the budget, so this is unreachable at real
+ // cardinality -- but the mirror-side ensureSentDictCapacity guards the same
+ // math, and a future caller must not be able to overflow this one silently.
+ long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart)
+ + NativeBufferWriter.varintSize(deltaCount)
+ + symbolsLen;
+ long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL;
+ if (frameLenL > MAX_SENT_DICT_BYTES) {
+ LineSenderException err = new LineSenderException(
+ "symbol dictionary catch-up frame exceeds the maximum size ["
+ + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']');
+ recordFatal(err);
+ throw new CatchUpSendException(err);
+ }
+ int payloadLen = (int) payloadLenL;
+ int frameLen = (int) frameLenL;
+ long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT);
+ try {
+ Unsafe.getUnsafe().putByte(frame, (byte) 'Q');
+ Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W');
+ Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P');
+ Unsafe.getUnsafe().putByte(frame + 3, (byte) '1');
+ Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion());
+ // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO
+ // rows, so it must never trigger a server-side commit. Today it is
+ // always the first frame on a fresh (empty-transaction) connection, so
+ // committing nothing is a no-op -- but that invariant is load-bearing
+ // and unasserted. Deferring the (empty) commit removes the dependency:
+ // a future mid-stream catch-up cannot prematurely commit an in-flight
+ // deferred transaction. The dictionary delta still registers (as any
+ // deferred data frame's does); only the row commit is deferred, and the
+ // next real frame commits it.
+ Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS,
+ (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT
+ | QwpConstants.FLAG_DEFER_COMMIT));
+ Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount
+ Unsafe.getUnsafe().putInt(frame + 8, payloadLen);
+ long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart);
+ q = NativeBufferWriter.writeVarint(q, deltaCount);
+ Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen);
+ client.sendBinary(frame, frameLen);
+ } catch (Throwable t) {
+ // Do NOT fail() here -- see CatchUpSendException. Signal the failure
+ // up so exactly one non-re-entrant reconnect follows. A JVM Error is
+ // never a transient reconnect case; let it propagate as-is so the
+ // I/O loop latches it terminal rather than looping on it.
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ throw new CatchUpSendException(t);
+ } finally {
+ Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT);
+ }
+ nextWireSeq++; // this catch-up chunk consumed a wire sequence
+ lastFrameOrPingNanos = System.nanoTime();
+ totalFramesSent.incrementAndGet();
+ }
+
private boolean tryReceiveAcks() {
boolean any = false;
try {
@@ -1898,7 +2430,15 @@ private boolean trySendOne() {
// Nothing sent on this connection yet: re-anchor in place past
// the retired tail. The wireSeq<->FSN mapping is untouched
// because no wire sequence has been consumed.
- positionCursorForStart();
+ try {
+ positionCursorForStart();
+ } catch (CatchUpSendException e) {
+ // Re-anchor's catch-up send failed. fail() here is a fresh,
+ // non-re-entrant connectLoop entry from the I/O loop body --
+ // the same recovery a normal trySendOne send failure takes.
+ fail(e.getCause());
+ return false;
+ }
return true;
}
// Frames were already sent on this connection: the linear
@@ -1949,12 +2489,50 @@ private boolean trySendOne() {
if (frameEnd > pub) {
return false; // payload not fully published yet
}
+ long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE;
+ // Torn-dictionary guard. Decode the delta start unconditionally (-1 for a
+ // non-delta frame); the guard MUST run even when deltaDictEnabled is false.
+ // A disk slot recovered with its persisted dictionary unavailable
+ // (PersistedSymbolDict.open() returned null -- fd exhaustion, a read-only
+ // remount, ENOSPC) reports deltaDictEnabled=false, yet its recorded frames
+ // are still DELTA frames (deltaStart > 0). Replaying those against a fresh
+ // empty-dictionary server would null-pad the missing ids and SILENTLY
+ // corrupt the table -- precisely what this guard exists to prevent -- so it
+ // cannot be gated on the very flag that goes false in that failure mode. In
+ // normal operation a delta frame's start id never exceeds the dictionary
+ // coverage established so far (replayed frames overlap the catch-up dict;
+ // fresh frames extend it contiguously), so a gap here means the recovered
+ // dictionary is incomplete (a host/power crash that lost recently-written
+ // entries, SF being process-crash but not host-crash durable). Fail
+ // terminally; the unreplayable data must be resent. Full-dict / fallback
+ // frames carry deltaStart=0 with sentDictCount=0, so 0 > 0 never
+ // false-positives; only the sent-dictionary mirror below stays gated on
+ // deltaDictEnabled.
+ int deltaStart = frameDeltaStart(frameAddr, payloadLen);
+ if (deltaStart > sentDictCount) {
+ recordFatal(new LineSenderException(
+ "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): "
+ + "frame delta start " + deltaStart + " exceeds recovered dictionary size "
+ + sentDictCount + "; cannot replay without corrupting data -- resend required"));
+ return false;
+ }
try {
- client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen);
+ client.sendBinary(frameAddr, payloadLen);
} catch (Throwable t) {
fail(t);
return false;
}
+ // A real ring frame (data or commit) has now gone out on this connection,
+ // as opposed to only the dictionary catch-up. onClose / handleServerRejection
+ // key their poison-strike vs pre-send decision off this, not off nextWireSeq
+ // (which the catch-up advances).
+ dataFrameSentThisConnection = true;
+ if (deltaDictEnabled && deltaStart >= 0) {
+ // Mirror the symbols this frame introduced so a later reconnect can
+ // rebuild the whole dictionary. Idempotent on replay: a frame whose
+ // delta we already hold advances nothing.
+ accumulateSentDict(frameAddr, payloadLen, deltaStart);
+ }
lastFrameOrPingNanos = System.nanoTime();
sendOffset = frameEnd;
long fsnSent = fsnAtZero + nextWireSeq;
@@ -1984,8 +2562,11 @@ void positionCursorForStart() {
// starts past it. Zero wire cost, no recycle.
tryRetireOrphanTail();
long replayStart = engine.ackedFsn() + 1L;
- this.fsnAtZero = replayStart;
- this.nextWireSeq = 0L;
+ // Recovery / orphan-drain seed the dictionary mirror, so the initial
+ // connection may also need a catch-up (client is non-null in the
+ // sync-start and drainer paths; null in async-initial, where swapClient
+ // handles it on the first connect).
+ setWireBaselineWithCatchUp(replayStart);
positionCursorAt(replayStart);
}
@@ -2036,6 +2617,30 @@ public interface ReconnectFactory {
WebSocketClient reconnect() throws Exception;
}
+ /**
+ * Signals that a symbol-dictionary catch-up frame could not be sent on the
+ * current connection. Thrown by {@link #sendDictCatchUp}/{@link
+ * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs
+ * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial
+ * connect, inside {@link #start()} / {@link #trySendOne} on the caller
+ * thread. Calling {@code fail()} from there would re-enter {@code
+ * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire
+ * mapping (a subsequent ACK then trims un-acked frames) and growing the
+ * stack until it overflows into a terminal, breaking the "retry a transient
+ * outage forever" invariant -- or run {@code connectLoop} on the caller
+ * thread and block {@code Sender} construction indefinitely. Each catch
+ * site instead turns it into ONE non-re-entrant reconnect: {@code
+ * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()}
+ * from the I/O loop body (trySendOne path), or dropping the dead client so
+ * the I/O thread reconnects (start path). A JVM {@code Error} is never
+ * wrapped -- it must stay terminal.
+ */
+ private static final class CatchUpSendException extends RuntimeException {
+ CatchUpSendException(Throwable cause) {
+ super(cause);
+ }
+ }
+
/**
* One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table
* (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0)
@@ -2173,7 +2778,7 @@ public void onClose(int code, String reason) {
|| code == WebSocketCloseCode.GOING_AWAY;
LineSenderException cause = new LineSenderException(
"WebSocket closed by server: code=" + code + " reason=" + reason);
- if (!orderly && nextWireSeq > 0) {
+ if (!orderly && dataFrameSentThisConnection) {
if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) {
haltOnPoisonedFrame("ws-close[" + code + ' '
+ WebSocketCloseCode.describe(code) + "]: " + reason,
@@ -2280,17 +2885,21 @@ private void handleServerRejection(long wireSeq) {
// value is only used to attribute an FSN to the error report --
// a rejection never advances the watermark.
long highestSent = nextWireSeq - 1L;
- if (highestSent < 0L) {
- // Pre-send rejection: server emitted an error frame before
- // we sent anything on this connection (typical after a
- // fresh swapClient โ auth failure, server-initiated halt,
- // etc.). The server-named wireSeq does not correspond to
- // any frame we sent, so clamping it to 0 and acknowledging
- // fsnAtZero would silently advance ackedFsn past a real
- // unsent batch (fsnAtZero == ackedFsn + 1 right after a
- // swap). Skip the watermark advance entirely; still surface
- // the error so the user's handler sees it and HALT errors
- // remain producer-observable.
+ if (!dataFrameSentThisConnection) {
+ // Pre-send rejection: the server emitted an error frame before we
+ // sent any DATA frame on this connection (typical after a fresh
+ // swapClient -- auth failure, server-initiated halt, or a rejection
+ // of the dictionary catch-up itself). nextWireSeq may be > 0 here
+ // because the catch-up consumed wire sequences, so this keys off
+ // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a
+ // transient NACK of a catch-up frame would take the post-send
+ // poison-strike path and could escalate a transient outage to a
+ // terminal. The server-named wireSeq does not correspond to any
+ // data frame we sent, so clamping it to 0 and acknowledging
+ // fsnAtZero would silently advance ackedFsn past a real unsent
+ // batch. Skip the watermark advance entirely; still surface the
+ // error so the user's handler sees it and HALT errors remain
+ // producer-observable.
handlePreSendRejection(wireSeq, status, category, policy);
return;
}
@@ -2300,6 +2909,32 @@ private void handleServerRejection(long wireSeq) {
wireSeq, highestSent);
}
long fsn = fsnAtZero + cappedSeq;
+ if (fsn <= engine.ackedFsn()) {
+ // The clamped wire seq maps at or below the replay head, so this
+ // NACK is for a dictionary catch-up frame -- which occupies the
+ // already-acked wire sequences below replayStart -- not a data frame
+ // this connection sent. (dataFrameSentThisConnection can be true here
+ // because trySendOne ships the head data frame before tryReceiveAcks
+ // reads the catch-up's NACK in the same loop iteration.) Attributing
+ // it a data FSN would key recordHeadRejectionStrike() off a
+ // below-baseline FSN -- negative when replayStart < catchUpFrames --
+ // colliding with the poisonFsn == -1 "no suspect" sentinel and
+ // laundering a genuine poison run, and would report a bogus FSN.
+ // Treat it exactly like a pre-send rejection: surface + recycle, no
+ // poison strike, no watermark advance. Symmetric with the success
+ // path, where engine.acknowledge() no-ops at or below ackedFsn. A
+ // real replayed data frame is at fsn > ackedFsn, so it is never
+ // caught here.
+ //
+ // Catch-up frames therefore sit OUTSIDE the poison detector: a
+ // deterministically-NACKed catch-up recycles forever (paced, so no
+ // busy-loop). That is acceptable -- a catch-up only re-registers
+ // symbols the cluster already accepted, so a persistent NACK of one
+ // is a server bug, not a poison-frame the client can quarantine, and
+ // Invariant B's "retry a transient outage forever" applies.
+ handlePreSendRejection(wireSeq, status, category, policy);
+ return;
+ }
// Best-effort table attribution: the parser populates
// response.tableNames on error frames the same way it does on
// STATUS_OK. If exactly one table was named, surface it; if
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
index 78e5db9f..50fc18ff 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
@@ -522,6 +522,90 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in
return best;
}
+ /**
+ * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any
+ * delta-flagged frame in this segment carries, or {@code 0} only when NO
+ * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit
+ * or a symbol-reusing frame -- the encoder always sets the delta flag) still
+ * contributes its {@code deltaStart}, i.e. the producer's baseline at encode
+ * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that
+ * baseline so a dictionary torn below it is detected -- a conservative
+ * over-strand (it may over-reject a frame whose rows reference only surviving
+ * ids), never an under-strand that could silently shift the dense id map.
+ * Read-only walk over the recovered frames, used
+ * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash,
+ * out-of-order page loss) below the ids the surviving frames still reference: if
+ * the max here reaches at or beyond the recovered dictionary size, a resuming
+ * producer -- seeded from the shorter dictionary -- would re-use ids the frames
+ * already define. Only frames at or below {@code maxFsnInclusive} count: frames
+ * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires
+ * without ever transmitting, so their ids must not inflate the result (a resuming
+ * producer never reuses them on the wire). The frame layout mirrors
+ * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header
+ * ({@code qwpHeaderSize} bytes) is followed by the delta section
+ * {@code [deltaStart varint][deltaCount varint]...}.
+ *
+ * @param headerMagic the QWP message magic identifying a well-formed frame
+ * @param flagsOffset byte offset of the flags field within the QWP header
+ * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit
+ * @param qwpHeaderSize the QWP message header size (delta section starts past it)
+ * @param maxFsnInclusive highest frame FSN to consider; frames above it are the
+ * retired orphan-deferred tail -- pass
+ * {@code recoveredCommitBoundaryFsn}
+ */
+ public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) {
+ long maxEnd = 0L;
+ long off = HEADER_SIZE;
+ long frames = frameCount;
+ for (long i = 0; i < frames; i++) {
+ long fsn = baseSeq + i;
+ int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
+ long payload = mmapAddress + off + FRAME_HEADER_SIZE;
+ // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never
+ // sent, so its ids must not count. off still advances below.
+ if (fsn <= maxFsnInclusive
+ && payloadLen >= qwpHeaderSize
+ && payloadLen > flagsOffset
+ && Unsafe.getUnsafe().getInt(payload) == headerMagic
+ && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) {
+ long p = payload + qwpHeaderSize;
+ long limit = payload + payloadLen;
+ long deltaStart = 0L;
+ int shift = 0;
+ while (p < limit) {
+ byte b = Unsafe.getUnsafe().getByte(p++);
+ deltaStart |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ if (shift > 35) {
+ break; // corrupt run; recovered frames are CRC-valid, so defensive only
+ }
+ }
+ long deltaCount = 0L;
+ shift = 0;
+ while (p < limit) {
+ byte b = Unsafe.getUnsafe().getByte(p++);
+ deltaCount |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ if (shift > 35) {
+ break;
+ }
+ }
+ long end = deltaStart + deltaCount;
+ if (end > maxEnd) {
+ maxEnd = end;
+ }
+ }
+ off += FRAME_HEADER_SIZE + payloadLen;
+ }
+ return maxEnd;
+ }
+
/**
* Number of frames written since {@link #create} (or recovered by
* {@link #openExisting}). Used by {@code SegmentRing} to compute
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
new file mode 100644
index 00000000..51c1b19b
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -0,0 +1,699 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Crc32c;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+import org.jetbrains.annotations.TestOnly;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Append-only, per-slot persistence of the global symbol dictionary that a
+ * store-and-forward sender ships to the server with delta encoding. Lives at
+ * {@code /.symbol-dict} alongside the segment files, the slot lock and
+ * {@code .ack-watermark}.
+ *
+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the
+ * symbols it introduces, so recovering (process restart) or draining (orphan
+ * adoption) a slot requires re-registering the whole dictionary on the fresh
+ * server before those frames replay. This file is that dictionary. Unlike
+ * {@link AckWatermark} -- a discardable optimization protected by a
+ * {@code max()} clamp -- this file is load-bearing: a surviving frame
+ * that references an id missing from it is unrecoverable. It is therefore held
+ * to a stronger durability contract.
+ *
+ * Layout (little-endian):
+ *
+ * offset 0: u32 magic = 'SYD1'
+ * offset 4: u8 version = 2
+ * offset 5: 3 bytes reserved (zero)
+ * offset 8: entries, each [len: varint][utf8 bytes][crc32c: u32], in ascending global-id order
+ *
+ * Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned
+ * sequentially from 0), so no id needs to be stored. Each entry carries a
+ * CRC-32C over its {@code [len][utf8]} bytes (the same checksum the SF segment
+ * frames use), so a torn or stale entry is detected on recovery instead of
+ * being silently mis-parsed.
+ *
+ * Durability / write-ahead ordering: the producer appends the symbols a
+ * frame introduces BEFORE that frame is published to the ring, but does NOT
+ * fsync -- matching the rest of store-and-forward, which is page-cache (not
+ * disk) durable. This ordering is sufficient for a process/JVM crash: the
+ * page cache survives, so both the dictionary and the frames survive and the
+ * dictionary is a superset of every recoverable frame's references. It is NOT
+ * sufficient for a host/power crash, where unflushed pages can be lost out
+ * of order and the dictionary may end up torn relative to the frames it serves --
+ * exactly as the segment frames themselves may be lost on a host crash. Two
+ * layers keep a host-crash tear from silently corrupting data:
+ *
+ * - The per-entry CRC-32C: {@link #open} verifies every entry and stops at
+ * the first one whose checksum fails, so an interior page lost out of
+ * order (reading back as zeroes) or a stale entry left past the end by a
+ * failed truncate is DETECTED and the trusted region ends before it --
+ * recovery never mis-parses a corrupt entry as a real symbol nor shifts
+ * the dense id->symbol map. The truncate that drops a torn/stale tail is
+ * now failure-checked (see {@link #open}): a file that cannot be trimmed
+ * is untrusted and recreated empty rather than left exposing stale bytes.
+ * - The send loop's replay guard: once recovery trusts only the intact
+ * prefix, a surviving frame whose delta start id exceeds that prefix
+ * fails loudly (the unreplayable data must be resent) rather than sending
+ * a gapped frame.
+ *
+ * Together these turn every detectable host-crash tear into a fail-clean
+ * "resend required" instead of a silent symbol misattribution -- the same
+ * CRC-32C protection the segment frames carry. A tear that happened to leave a
+ * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32
+ * collision per corrupted entry, no weaker than the frames' own checksum.
+ *
+ * A torn trailing entry from a crash mid-append is self-healing: {@link #open}
+ * stops parsing at the first incomplete entry and the next append overwrites it.
+ *
+ * Lifecycle: single-writer (the producer / user thread) for appends. Read
+ * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The
+ * owner (the engine) closes it, and {@code close()} is callable from any thread
+ * (a shutdown hook, test cleanup). {@code close()} and the append methods are
+ * therefore {@code synchronized}: without that, a close racing an in-flight append
+ * could free the scratch buffer or close the fd mid-write and let the write land
+ * on a descriptor the OS has reused for another file (silent cross-file
+ * corruption). Not thread-safe for concurrent writers.
+ */
+public final class PersistedSymbolDict implements QuietCloseable {
+
+ /**
+ * Filename within the slot directory. Dot-prefixed so directory
+ * enumerators that filter by the {@code .sfa} suffix (segment recovery,
+ * OrphanScanner, trim) skip it automatically.
+ */
+ public static final String FILE_NAME = ".symbol-dict";
+ static final int CRC_SIZE = 4; // u32 CRC-32C trailing every entry
+ static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian
+ static final int HEADER_SIZE = 8;
+ static final byte VERSION = 2; // v2 appended the per-entry CRC-32C
+ private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
+ private final int fd;
+ // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
+ // tests inject a fault facade to exercise recovery I/O failures (a truncate
+ // that cannot drop a torn tail, a short write) without a real broken disk.
+ private final FilesFacade ff;
+ private long appendOffset;
+ private boolean closed;
+ // In-memory copy of the entry region [len][utf8]... exactly as on disk,
+ // populated only when open() recovered existing entries (recovery /
+ // orphan-drain). Zero/empty for a freshly created file. READ (not consumed) to
+ // seed the producer's id map (readLoadedSymbols) and to seed the send loop's
+ // catch-up mirror, which COPIES it. This file retains ownership for the engine's
+ // lifetime -- the orphan drainer builds a fresh send loop per wire session
+ // against the same engine, and each must re-seed its mirror -- and frees this
+ // buffer in close().
+ private long loadedEntriesAddr;
+ private int loadedEntriesLen;
+ private long scratchAddr;
+ private int scratchCap;
+ private int size;
+
+ private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) {
+ this.ff = ff;
+ this.fd = fd;
+ this.appendOffset = appendOffset;
+ this.size = size;
+ this.loadedEntriesAddr = loadedEntriesAddr;
+ this.loadedEntriesLen = loadedEntriesLen;
+ }
+
+ /**
+ * Opens (creating if absent) the dictionary file in {@code slotDir}. An
+ * existing file is parsed and its complete entries are loaded into memory
+ * (see {@link #loadedEntriesAddr()}); a missing or invalid file is (re)created
+ * with a fresh header. Returns {@code null} on any I/O failure -- the caller
+ * then falls back to full-dictionary (self-sufficient) frames for this slot,
+ * so a broken side-file degrades gracefully rather than aborting the sender.
+ */
+ public static PersistedSymbolDict open(String slotDir) {
+ return open(FilesFacade.INSTANCE, slotDir);
+ }
+
+ /**
+ * Facade-aware variant of {@link #open(String)}. Production passes
+ * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery
+ * I/O failures (e.g. a truncate that cannot drop a torn tail).
+ */
+ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
+ String filePath = slotDir + "/" + FILE_NAME;
+ long existing = ff.exists(filePath) ? ff.length(filePath) : -1L;
+ // A dictionary that grew to or past Integer.MAX_VALUE cannot be reopened:
+ // openExisting reads it into ONE int-sized native buffer. PAST 2GiB the
+ // (int) cast is either negative (malloc rejects it), exactly zero (getInt
+ // then reads 4 bytes past a zero-size allocation), or a small positive prefix
+ // (whose validLen < len branch would then DESTRUCTIVELY truncate the multi-GB
+ // file); AT exactly Integer.MAX_VALUE the cast is exact but the ~2GB malloc is
+ // doomed to OutOfMemoryError. The >= guard short-circuits every case to a
+ // clean re-create instead of the doomed allocation -- fail-clean, exactly like
+ // every other unreadable-file case here, so the sender falls back to full
+ // self-sufficient frames. Reaching this needs ~100M+ distinct symbols on one
+ // slot (far past realistic symbol cardinality); the guard keeps the read/write
+ // size boundary safe anyway.
+ if (existing >= Integer.MAX_VALUE) {
+ LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing);
+ return openFresh(ff, filePath);
+ }
+ if (existing >= HEADER_SIZE) {
+ PersistedSymbolDict d = openExisting(ff, filePath, existing);
+ if (d != null) {
+ return d;
+ }
+ // Fall through to a clean re-create: a header/parse failure on an
+ // existing file means it cannot be trusted for delta replay.
+ }
+ return openFresh(ff, filePath);
+ }
+
+ /**
+ * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding
+ * any surviving content. This is the fresh-start counterpart to {@link #open}:
+ * a slot with no recovered segments must start with an empty dictionary, so a
+ * dictionary left by a prior lifecycle -- a fully-drained slot whose
+ * best-effort delete failed, or a crash in the close window -- must NOT be
+ * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for
+ * recovery/orphan-drain replay, this truncates it: the fresh-start producer is
+ * not seeded from the dictionary, so trusting a survivor would leave the
+ * producer's ids diverged from the dictionary the send loop replays and
+ * silently misattribute symbols on the next reconnect. Truncating (rather than
+ * relying on an unlink succeeding first) closes the gap even when the delete is
+ * refused -- e.g. a Windows share lock. Returns {@code null} on I/O failure, so
+ * the caller falls back to full self-sufficient frames exactly as {@link #open}
+ * does.
+ */
+ public static PersistedSymbolDict openClean(String slotDir) {
+ return openClean(FilesFacade.INSTANCE, slotDir);
+ }
+
+ /**
+ * Facade-aware variant of {@link #openClean(String)}.
+ */
+ public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) {
+ return openFresh(ff, slotDir + "/" + FILE_NAME);
+ }
+
+ /**
+ * Best-effort removal of a stale dictionary file. Used at fully-drained close
+ * (the slot is empty, nothing references the dictionary any more), mirroring
+ * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT
+ * use this -- it opens a clean dictionary via {@link #openClean} instead, so a
+ * failed delete cannot leave a stale dictionary a new session would trust.
+ */
+ public static void removeOrphan(String slotDir) {
+ removeOrphan(FilesFacade.INSTANCE, slotDir);
+ }
+
+ /**
+ * Facade-aware variant of {@link #removeOrphan(String)}.
+ */
+ public static void removeOrphan(FilesFacade ff, String slotDir) {
+ ff.remove(slotDir + "/" + FILE_NAME);
+ }
+
+ /**
+ * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the
+ * symbol-dict delta section the frame encoder just wrote -- to the on-disk
+ * dictionary, computing and appending a per-entry CRC-32C as it copies so the
+ * producer does not re-encode the symbols. The on-disk layout is
+ * {@code [len varint][utf8][crc32c]} per entry (see the class layout note); the
+ * {@code addr}/{@code len} bytes carry no CRC, so this walks the {@code count}
+ * entries to insert one. Advances {@code size} by {@code count}. Same
+ * durability/idempotency contract as {@link #appendSymbols}: no fsync, and a
+ * short write throws WITHOUT advancing {@code size}/{@code appendOffset}, so a
+ * retry keyed off {@link #size()} re-persists the same range at the same
+ * offset. No-op when the range is empty or the dictionary is closed.
+ */
+ public synchronized void appendRawEntries(long addr, int len, int count) {
+ if (closed || count <= 0 || len <= 0) {
+ return;
+ }
+ int outLen = len + count * CRC_SIZE;
+ ensureScratch(outLen);
+ long src = addr;
+ long srcLimit = addr + len;
+ long dst = scratchAddr;
+ for (int i = 0; i < count; i++) {
+ long entryStart = src;
+ long symLen = 0;
+ int shift = 0;
+ while (src < srcLimit) {
+ byte b = Unsafe.getUnsafe().getByte(src++);
+ symLen |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ if (shift > 35) {
+ // A canonical entry-length varint is <= 5 bytes; a longer
+ // continuation run is corrupt. The downstream entryEnd > srcLimit
+ // check then rejects it. Matches decodeVarint / readVarintAt.
+ break;
+ }
+ }
+ long entryEnd = src + symLen; // src is just past the len varint
+ if (entryEnd > srcLimit) {
+ throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME
+ + " [entry=" + i + ", count=" + count + ']');
+ }
+ int wireSpan = (int) (entryEnd - entryStart); // [len][utf8]
+ Unsafe.getUnsafe().copyMemory(entryStart, dst, wireSpan);
+ Unsafe.getUnsafe().putInt(dst + wireSpan, Crc32c.update(Crc32c.INIT, entryStart, wireSpan));
+ dst += wireSpan + CRC_SIZE;
+ src = entryEnd;
+ }
+ if (src != srcLimit) {
+ // The count entries did not consume exactly len bytes -- a caller passed an
+ // inconsistent (addr, len, count) triple. Writing outLen would flush an
+ // uninitialised scratch tail and mis-advance size, so fail loudly. The sole
+ // caller derives count and len from one beginMessage, so this cannot fire
+ // today.
+ throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to "
+ + FILE_NAME + " [count=" + count + ", len=" + len
+ + ", consumed=" + (int) (src - addr) + ']');
+ }
+ long written = ff.write(fd, scratchAddr, outLen, appendOffset);
+ if (written != outLen) {
+ throw new IllegalStateException("short write to " + FILE_NAME
+ + " [expected=" + outLen + ", actual=" + written + ']');
+ }
+ appendOffset += outLen;
+ size += count;
+ }
+
+ /**
+ * Appends one symbol, extending the on-disk dictionary. The caller appends a
+ * frame's new symbols BEFORE publishing that frame, so the write ordering
+ * (dictionary entry before referencing frame) holds; no fsync is performed
+ * (see the class-level durability note). Assigns the next dense id implicitly
+ * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC
+ * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on
+ * recovery.
+ *
+ * Test-only: production persists a frame's whole new-symbol range in one write
+ * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to
+ * build a dictionary one entry at a time.
+ */
+ @TestOnly
+ public synchronized void appendSymbol(CharSequence symbol) {
+ if (closed) {
+ return;
+ }
+ int utf8Len = Utf8s.utf8Bytes(symbol);
+ int varLen = NativeBufferWriter.varintSize(utf8Len);
+ int wireLen = varLen + utf8Len; // [len][utf8]
+ int recLen = wireLen + CRC_SIZE; // + trailing crc
+ ensureScratch(recLen);
+ long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len);
+ if (utf8Len > 0) {
+ Utf8s.strCpyUtf8(symbol, p, utf8Len);
+ }
+ Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen));
+ long written = ff.write(fd, scratchAddr, recLen, appendOffset);
+ if (written != recLen) {
+ throw new IllegalStateException("short write to " + FILE_NAME
+ + " [expected=" + recLen + ", actual=" + written + ']');
+ }
+ appendOffset += recLen;
+ size++;
+ }
+
+ /**
+ * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes
+ * the whole {@code [len varint][utf8][crc32c]...} region into scratch first,
+ * then issues one positioned write -- versus one {@code pwrite(2)} per symbol
+ * via {@link #appendSymbol}. That per-symbol syscall count is the hot-path
+ * cost on a high-cardinality batch (one new symbol per row), which is exactly
+ * the store-and-forward workload delta encoding targets. Each entry carries a
+ * trailing CRC-32C over its {@code [len][utf8]} bytes. Callers pass the
+ * dictionary and the range so the ids resolve to their symbol strings.
+ *
+ * Same durability and idempotency contract as {@link #appendSymbol}: no
+ * fsync, and a short write throws WITHOUT advancing {@code size}/{@code
+ * appendOffset}, so a retry keyed off {@link #size()} re-encodes the same
+ * range and overwrites at the same offset. No-op when the range is empty or
+ * the dictionary is closed.
+ */
+ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) {
+ if (closed || to < from) {
+ return;
+ }
+ int len = 0;
+ for (int id = from; id <= to; id++) {
+ CharSequence symbol = dict.getSymbol(id);
+ int utf8Len = Utf8s.utf8Bytes(symbol);
+ int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+ ensureScratch(len + wireLen + CRC_SIZE);
+ long entryStart = scratchAddr + len;
+ long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+ if (utf8Len > 0) {
+ Utf8s.strCpyUtf8(symbol, p, utf8Len);
+ }
+ Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen));
+ len += wireLen + CRC_SIZE;
+ }
+ long written = ff.write(fd, scratchAddr, len, appendOffset);
+ if (written != len) {
+ throw new IllegalStateException("short write to " + FILE_NAME
+ + " [expected=" + len + ", actual=" + written + ']');
+ }
+ appendOffset += len;
+ size += to - from + 1;
+ }
+
+ @Override
+ public synchronized void close() {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ if (loadedEntriesAddr != 0L) {
+ Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT);
+ // Null after freeing (like scratchAddr below) so a future accessor that
+ // reads loadedEntriesAddr()/loadedEntriesLen() post-close cannot
+ // dereference freed native memory; the getters are not closed-guarded.
+ loadedEntriesAddr = 0L;
+ loadedEntriesLen = 0;
+ }
+ if (scratchAddr != 0L) {
+ Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT);
+ scratchAddr = 0L;
+ scratchCap = 0;
+ }
+ if (fd >= 0) {
+ ff.close(fd);
+ }
+ }
+
+ /**
+ * Base address of the loaded entry region -- the concatenated
+ * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly
+ * as a delta section carries them. Zero when nothing was recovered.
+ *
+ * Construction-phase only. This hands out a raw pointer into native
+ * memory that {@link #close()} frees and nulls, with no closed-guard and no
+ * synchronization. It is safe to read only BEFORE the slot's I/O thread and
+ * any producer append start -- i.e. while the send loop is being constructed
+ * or an orphan-drain is seeding its mirror, both of which happen-before those
+ * threads. A caller that reads it from a running thread races {@code close()}
+ * and can dereference freed memory (use-after-free).
+ */
+ public long loadedEntriesAddr() {
+ return loadedEntriesAddr;
+ }
+
+ /**
+ * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for
+ * the same reason -- see {@link #loadedEntriesAddr()}.
+ */
+ public int loadedEntriesLen() {
+ return loadedEntriesLen;
+ }
+
+ /**
+ * Materialises the loaded entries as symbol strings in ascending-id order
+ * (entry {@code i} is symbol id {@code i}). Used once on recovery to
+ * repopulate the producer's global dictionary. Empty when nothing was
+ * recovered.
+ *
+ * Construction-phase only -- like {@link #loadedEntriesAddr()}, this
+ * walks the native entry region {@link #close()} frees, with no closed-guard,
+ * so it must run before the I/O thread and any producer append start.
+ */
+ public ObjList readLoadedSymbols() {
+ ObjList out = new ObjList<>(Math.max(size, 1));
+ long p = loadedEntriesAddr;
+ long limit = p + loadedEntriesLen;
+ for (int i = 0; i < size && p < limit; i++) {
+ long len = 0;
+ int shift = 0;
+ while (p < limit) {
+ byte b = Unsafe.getUnsafe().getByte(p++);
+ len |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ if (shift > 35) {
+ // Bound the varint like decodeVarint / appendRawEntries /
+ // readVarintAt: a canonical length is <= 5 bytes. open() already
+ // CRC-validated these bytes, so this is defensive only; the
+ // p + len > limit check below then rejects the over-long run.
+ break;
+ }
+ }
+ if (p + len > limit) {
+ break; // defensive: torn tail (should not happen past parse in open)
+ }
+ out.add(Utf8s.stringFromUtf8Bytes(p, p + len));
+ p += len;
+ }
+ return out;
+ }
+
+ /**
+ * Number of symbols the dictionary holds (highest id + 1).
+ */
+ public int size() {
+ return size;
+ }
+
+ /**
+ * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns
+ * {@code [value, newPos]} or {@code null} if the varint is truncated
+ * (torn tail).
+ */
+ private static long[] decodeVarint(long buf, int pos, int limit) {
+ long value = 0;
+ int shift = 0;
+ int cur = pos;
+ while (cur < limit) {
+ byte b = Unsafe.getUnsafe().getByte(buf + cur);
+ cur++;
+ value |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ return new long[]{value, cur};
+ }
+ shift += 7;
+ if (shift > 35) {
+ return null; // implausible for an entry length; treat as torn
+ }
+ }
+ return null;
+ }
+
+ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) {
+ int fd = ff.openRW(filePath);
+ if (fd < 0) {
+ LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd);
+ return null;
+ }
+ long buf = 0L;
+ long entriesAddr = 0L;
+ int entriesLen = 0;
+ try {
+ int len = (int) fileLen;
+ buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+ long read = ff.read(fd, buf, len, 0);
+ if (read != len
+ || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC
+ || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) {
+ LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath);
+ Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT);
+ buf = 0L; // null after free so the catch below cannot double-free if ff.close throws
+ ff.close(fd);
+ return null;
+ }
+ // Parse complete, CRC-valid entries after the header; stop at the first
+ // torn/incomplete OR crc-mismatched entry. The CRC turns an interior
+ // tear (a lost page reading back as zeroes) or a stale entry left past
+ // the end by a failed truncate into a clean stop point, so recovery
+ // trusts only the intact prefix instead of silently mis-parsing a
+ // corrupt entry and shifting the dense id->symbol map.
+ int diskPos = HEADER_SIZE; // walks the on-disk [len][utf8][crc] entries
+ int count = 0;
+ int wireLen = 0; // running size of the crc-stripped copy
+ while (diskPos < len) {
+ long[] vr = decodeVarint(buf, diskPos, len);
+ if (vr == null) {
+ break; // torn length varint
+ }
+ long symLen = vr[0];
+ int afterVar = (int) vr[1];
+ // [len varint][utf8] then a u32 CRC. symLen stays a long so a
+ // corrupt multi-gigabyte length cannot wrap an int back under the
+ // bound check. No fixed per-entry ceiling -- the write path applies
+ // none, so a legitimately large symbol must recover here.
+ long wireEnd = (long) afterVar + symLen; // end of [len][utf8]
+ if (wireEnd + CRC_SIZE > len) {
+ break; // torn/incomplete trailing entry (its CRC doesn't fit)
+ }
+ int wireEndI = (int) wireEnd;
+ int wireSpan = wireEndI - diskPos;
+ int crcStored = Unsafe.getUnsafe().getInt(buf + wireEndI);
+ int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, wireSpan);
+ if (crcCalc != crcStored) {
+ break; // corrupt/stale entry -- stop before it (fail-clean)
+ }
+ diskPos = wireEndI + CRC_SIZE;
+ wireLen += wireSpan;
+ count++;
+ }
+ int diskConsumed = diskPos - HEADER_SIZE; // valid entries incl. their CRCs
+ // Materialise the trusted entries as WIRE bytes ([len][utf8]..., no
+ // CRC) so loadedEntries*/readLoadedSymbols and the send-loop catch-up
+ // mirror stay wire-shaped -- the on-disk CRC is stripped here, once, at
+ // open. A second no-alloc walk over the already-validated region.
+ if (wireLen > 0) {
+ entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT);
+ // Record the length alongside the malloc so the catch below frees the
+ // right size (not 0) if this copy walk ever throws.
+ entriesLen = wireLen;
+ long dst = entriesAddr;
+ int p = HEADER_SIZE;
+ for (int i = 0; i < count; i++) {
+ int vp = p;
+ long symLen = 0;
+ int shift = 0;
+ while (true) {
+ byte b = Unsafe.getUnsafe().getByte(buf + vp++);
+ symLen |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ if (shift > 35) {
+ break; // corrupt run; these entries were CRC-validated above
+ }
+ }
+ int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC
+ Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan);
+ dst += wireSpan;
+ p += wireSpan + CRC_SIZE; // skip the entry's CRC
+ }
+ }
+ Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT);
+ buf = 0L;
+ // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
+ // leave residue past its own end. Unlike before, the truncate result IS
+ // checked: a file we cannot trim could still expose stale post-end bytes
+ // whose (self-consistent) per-entry CRC the parse would accept at a
+ // shifted position, so a failed truncate makes the file untrusted --
+ // open() then recreates it empty (fail-clean) rather than risk a silent
+ // misattribution.
+ long validLen = HEADER_SIZE + diskConsumed;
+ if (validLen < len && !ff.truncate(fd, validLen)) {
+ LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath);
+ if (entriesAddr != 0L) {
+ Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT);
+ // Null after freeing (like buf above) so the catch below cannot
+ // double-free entriesAddr if the following ff.close throws.
+ entriesAddr = 0L;
+ entriesLen = 0;
+ }
+ ff.close(fd);
+ return null;
+ }
+ return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen);
+ } catch (Throwable t) {
+ if (buf != 0L) {
+ Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT);
+ }
+ // Free entriesAddr if it was allocated and not yet handed off. The success
+ // path transfers it to the returned dict, and every path that frees it
+ // earlier (the truncate-failure branch above) also nulls it, so this cannot
+ // double-free. Keeps the error path leak-free on any throw between its
+ // malloc and the return.
+ if (entriesAddr != 0L) {
+ Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT);
+ }
+ ff.close(fd);
+ LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t));
+ return null;
+ }
+ }
+
+ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
+ int fd = ff.openCleanRW(filePath);
+ if (fd < 0) {
+ LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd);
+ return null;
+ }
+ long hdr = 0L;
+ try {
+ hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+ Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC);
+ Unsafe.getUnsafe().putByte(hdr + 4, VERSION);
+ Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0);
+ Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0);
+ Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0);
+ long written = ff.write(fd, hdr, HEADER_SIZE, 0);
+ if (written != HEADER_SIZE) {
+ ff.close(fd);
+ ff.remove(filePath);
+ LOG.warn("symbol dict {} header write failed; proceeding without it", filePath);
+ return null;
+ }
+ } catch (Throwable t) {
+ // Unreachable today (Files.write is native and returns -1 rather than
+ // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte
+ // malloc cannot realistically OOM), but close the fd against a future
+ // edit so it cannot leak -- mirroring openExisting's error handling.
+ ff.close(fd);
+ LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t));
+ return null;
+ } finally {
+ if (hdr != 0L) {
+ Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+ return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0);
+ }
+
+ private void ensureScratch(int required) {
+ if (scratchCap >= required) {
+ return;
+ }
+ // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and
+ // would make the realloc size negative. required is bounded by one frame's
+ // entries (the server batch cap), so this never actually caps -- it mirrors the
+ // long-math growth in CursorWebSocketSendLoop.ensureSentDictCapacity.
+ long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2));
+ if (newCap > Integer.MAX_VALUE - 8) {
+ newCap = Integer.MAX_VALUE - 8;
+ }
+ scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+ scratchCap = (int) newCap;
+ }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index c8516c4c..21cfa9d1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -546,6 +546,26 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag
return Math.max(best, fsn);
}
+ /**
+ * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below
+ * {@code maxFsnInclusive} references (0 when none). See
+ * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a
+ * persisted dictionary torn below the ids the surviving committed frames
+ * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred
+ * tail and are excluded.
+ */
+ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) {
+ long maxEnd = 0L;
+ for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+ long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive);
+ if (end > maxEnd) {
+ maxEnd = end;
+ }
+ }
+ long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive);
+ return Math.max(maxEnd, end);
+ }
+
public MmapSegment getActive() {
return active;
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java
new file mode 100644
index 00000000..132f2c06
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java
@@ -0,0 +1,589 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode).
+ *
+ * When a memory-mode sender reconnects, the server it lands on has an empty
+ * dictionary (the server discards it on every disconnect). Because the producer
+ * ships monotonic deltas -- each symbol id once -- a naive replay would leave the
+ * fresh server with a dictionary gap. The I/O thread prevents this by sending a
+ * full-dictionary catch-up frame before any post-reconnect traffic. This test
+ * reconstructs the server's per-connection dictionary from the captured wire
+ * bytes and asserts it stays complete and gap-free across the reconnect.
+ */
+public class DeltaDictCatchUpTest {
+
+ @Test
+ public void testReconnectCatchUpRebuildsDictionary() throws Exception {
+ // Connection 1: send "alpha" (id 0), ACK it, then drop the socket so the
+ // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1).
+ // Without catch-up, connection 2's first data frame would carry
+ // deltaStart=1 and the fresh server would never learn id 0.
+ assertMemoryLeak(() -> {
+ CatchUpHandler handler = new CatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+ sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow();
+ sender.flush();
+ waitFor(() -> handler.dictFor(1).size() >= 1, 5_000);
+
+ // Wait until the server has actually closed connection 1 before
+ // sending batch 2, so batch 2 cannot race into connection 1 and
+ // must drive the reconnect + catch-up.
+ waitFor(() -> handler.conn1Closed, 5_000);
+
+ sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.connectionsAccepted.get() >= 2
+ && handler.dictFor(2).size() >= 2, 5_000);
+ }
+
+ // The fresh (2nd) connection's dictionary, rebuilt purely from the
+ // frames it received, must hold both symbols contiguously with no
+ // null gap -- exactly what the catch-up frame guarantees.
+ List conn2 = handler.dictFor(2);
+ Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables",
+ handler.sawZeroTableFrameOnConn2);
+ Assert.assertTrue("the catch-up frame carries no rows, so it must defer its "
+ + "(empty) commit -- FLAG_DEFER_COMMIT set",
+ handler.catchUpDeferredOnConn2);
+ Assert.assertEquals("2nd connection dictionary size", 2, conn2.size());
+ Assert.assertEquals("alpha", conn2.get(0));
+ Assert.assertEquals("beta", conn2.get(1));
+ }
+ });
+ }
+
+ @Test
+ public void testReconnectPreservesMonotonicDeltaBaseline() throws Exception {
+ // Regression: the producer's sent-symbol watermark (sentMaxSymbolId) must
+ // SURVIVE a reconnect. resetSymbolDictStateForNewConnection deliberately
+ // leaves it untouched -- the I/O thread re-registers the whole dictionary via
+ // a catch-up frame before replay, so the producer keeps shipping deltas ABOVE
+ // the baseline across the wire boundary. If a regression reset it on
+ // reconnect, the first post-reconnect data frame would re-ship the whole
+ // dictionary inline (deltaStart=0), pure wasted bandwidth. The sibling
+ // testReconnectCatchUpRebuildsDictionary asserts only that the final
+ // dictionary is complete -- which a reset-then-redefine ALSO satisfies (the
+ // server tolerates the redefinition) -- so it does NOT catch that regression.
+ // This pins the baseline survival directly: connection 2's data frame must
+ // carry a delta starting at id 1 (above alpha), not 0.
+ assertMemoryLeak(() -> {
+ CatchUpHandler handler = new CatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+ // Connection 1 registers alpha (id 0); the server ACKs and drops it.
+ sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow();
+ sender.flush();
+ waitFor(() -> handler.dictFor(1).size() >= 1, 5_000);
+ waitFor(() -> handler.conn1Closed, 5_000);
+
+ // Connection 2 (fresh) registers beta (id 1). With the baseline
+ // preserved, beta ships as a delta ABOVE id 0 (deltaStart=1).
+ sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.connectionsAccepted.get() >= 2
+ && handler.dictFor(2).size() >= 2, 5_000);
+ }
+
+ Assert.assertTrue("connection 2 must re-register the dictionary via a catch-up first",
+ handler.sawZeroTableFrameOnConn2);
+ Assert.assertTrue("post-reconnect data frame must ship a delta ABOVE the surviving "
+ + "baseline (deltaStart >= 1); a reset baseline re-ships the whole "
+ + "dictionary from deltaStart 0",
+ handler.conn2SawDeltaAboveBaseline);
+ }
+ });
+ }
+
+ @Test
+ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception {
+ // Regression (homogeneous single cap): a symbol whose length sits just below
+ // the advertised cap is ACCEPTED into a data frame (messageSize <= cap) and
+ // enters the sent-dictionary mirror. On reconnect the catch-up must
+ // re-register it under the SAME cap -- the bare catch-up frame (header + two
+ // varints + the entry) is smaller than the data frame that already shipped
+ // it (which also carried the table schema + a row), so it fits.
+ //
+ // Pre-fix, the single-entry terminal used the conservative PACKING budget
+ // (cap - HEADER_SIZE - 16), which is stricter than the producer's publish
+ // gate (messageSize <= cap) by more than the minimal data-frame overhead. So
+ // a symbol accepted onto the wire under cap C could exceed that budget and
+ // trip a spurious "during catch-up" terminal, permanently hard-failing a
+ // running producer on its first transient reconnect. Concretely at cap=200:
+ // table("t").symbol("s", <173 chars>).atNow() encodes to 198 bytes (<=200,
+ // accepted), its dict entry is 2+173=175 bytes (> old budget 172 -> old
+ // terminal), while the real solo catch-up frame is 12+1+1+175=189 (<=200 ->
+ // fits). Unlike testCatchUpEntryTooLargeForCapFailsTerminally (a genuinely
+ // oversized entry on a shrunk cap, which MUST still terminate), this entry
+ // is legally shippable and must NOT terminate.
+ final int cap = 200;
+ final String nearCapSymbol = TestUtils.repeat("x", 173);
+ assertMemoryLeak(() -> {
+ CatchUpHandler handler = new CatchUpHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(cap); // same cap on every handshake (homogeneous)
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+ // Symbol-only row so the near-cap symbol drives the frame size.
+ sender.table("t").symbol("s", nearCapSymbol).atNow();
+ sender.flush();
+ waitFor(() -> handler.dictFor(1).size() >= 1, 5_000);
+ waitFor(() -> handler.conn1Closed, 5_000);
+
+ // The reconnect runs the catch-up under the SAME cap. Pre-fix
+ // this latched a terminal (surfacing on this flush); post-fix the
+ // catch-up ships the near-cap symbol and the flush goes through.
+ sender.table("t").symbol("s", "beta").atNow();
+ sender.flush();
+ waitFor(() -> handler.connectionsAccepted.get() >= 2
+ && handler.dictFor(2).size() >= 2, 5_000);
+ }
+
+ // Connection 2's dictionary, rebuilt purely from the frames it
+ // received, must hold the near-cap symbol (re-registered by the
+ // catch-up) and beta, gap-free -- proving the catch-up SHIPPED rather
+ // than terminated.
+ List conn2 = handler.dictFor(2);
+ Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables",
+ handler.sawZeroTableFrameOnConn2);
+ Assert.assertEquals("2nd connection dictionary size", 2, conn2.size());
+ Assert.assertEquals(nearCapSymbol, conn2.get(0));
+ Assert.assertEquals("beta", conn2.get(1));
+ }
+ });
+ }
+
+ @Test
+ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception {
+ // A dictionary entry that exceeds the reconnect server's per-chunk budget
+ // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. Every
+ // reachable node re-advertises the same small cap here, so the gap never
+ // resolves: sendDictCatchUp must retry across the settle budget and then
+ // latch a clean terminal ("... during catch-up") rather than call fail()
+ // (which from inside the catch-up re-enters connectLoop). Pre-fix it latched
+ // on the FIRST cap gap; the settle budget (MAX_CATCHUP_CAP_GAP_ATTEMPTS)
+ // rides out a transient smaller-cap window first (see the retry sibling in
+ // CursorWebSocketSendLoopCatchUpAlignmentTest), and only a persistent gap
+ // exhausts it. Small reconnect backoffs keep the budgeted attempts fast.
+ //
+ // Connection 1 advertises no cap, so the ~202-byte symbol registers and
+ // enters the sent-dictionary mirror. The handler then shrinks the
+ // advertised cap to 160 (catch-up budget 132) and drops the socket, so the
+ // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do
+ // this: the client refuses to SEND a single-table frame over the cap, and
+ // that data frame is always larger than the bare catch-up entry.)
+ assertMemoryLeak(() -> {
+ CapShrinkHandler handler = new CapShrinkHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ handler.setServer(server);
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry
+ LineSenderException terminal = null;
+ Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+ + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;");
+ try {
+ sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow();
+ sender.flush();
+ // The terminal latches on the I/O thread once the reconnect's
+ // catch-up hits the oversized entry; it surfaces to the producer
+ // on a subsequent flush. Poll a bounded time for it. The polling
+ // rows use a small symbol that fits the shrunk cap, so the
+ // producer-side cap check never fires and flush() surfaces the
+ // I/O thread's catch-up terminal via checkError.
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline && terminal == null) {
+ try {
+ sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow();
+ sender.flush();
+ Thread.sleep(20);
+ } catch (LineSenderException e) {
+ terminal = e;
+ }
+ }
+ } finally {
+ try {
+ sender.close();
+ } catch (LineSenderException e) {
+ if (terminal == null) {
+ terminal = e;
+ }
+ }
+ }
+ Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal);
+ Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(),
+ terminal.getMessage().contains("during catch-up"));
+ }
+ });
+ }
+
+ @Test
+ public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception {
+ // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes),
+ // then drops once the server has learned them all. On reconnect the fresh
+ // server has an empty dictionary, so the I/O thread must replay all 40 as a
+ // catch-up -- but the server advertises a 160-byte batch cap, so the whole
+ // dictionary cannot fit in a single frame. The catch-up therefore splits
+ // into several contiguous zero-table frames that the fresh server stitches
+ // back into a complete, gap-free dictionary.
+ final int symbolCount = 40;
+ assertMemoryLeak(() -> {
+ SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount);
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+ // One row per flush so each frame stays under the 160-byte cap; the
+ // sent dictionary still accumulates all 40 symbols on connection 1.
+ for (int i = 0; i < symbolCount; i++) {
+ sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow();
+ sender.flush();
+ }
+ // Wait until connection 1 has learned every symbol, so the sender's
+ // sent-dictionary mirror (the catch-up source) holds all of them.
+ waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000);
+
+ // Wait until the server has actually closed connection 1 before
+ // sending batch 2, so batch 2 drives the reconnect + split catch-up.
+ waitFor(() -> handler.conn1Closed, 5_000);
+
+ sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow();
+ sender.flush();
+ waitFor(() -> handler.connectionsAccepted.get() >= 2
+ && handler.dictFor(2).size() >= symbolCount + 1, 10_000);
+ }
+
+ // Connection 2's dictionary, rebuilt purely from the frames it received,
+ // must hold every symbol contiguously with no null gap -- the split
+ // catch-up frames reassemble exactly.
+ List conn2 = handler.dictFor(2);
+ Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size());
+ for (int i = 0; i <= symbolCount; i++) {
+ Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i));
+ }
+ // The catch-up had to span more than one zero-table frame to stay under
+ // the advertised cap -- that split is the behaviour under test.
+ Assert.assertTrue("catch-up split into multiple frames (saw "
+ + handler.zeroTableFramesOnConn2 + ")",
+ handler.zeroTableFramesOnConn2 >= 2);
+ }
+ });
+ }
+
+ private static String symbolName(int i) {
+ // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap.
+ return "symbol" + (1000 + i);
+ }
+
+ private static int readVarint(byte[] buf, int[] pos) {
+ int result = 0;
+ int shift = 0;
+ while (pos[0] < buf.length) {
+ int b = buf[pos[0]++] & 0xFF;
+ result |= (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) return result;
+ shift += 7;
+ if (shift > 28) throw new IllegalStateException("varint too long");
+ }
+ throw new IllegalStateException("varint truncated");
+ }
+
+ private static void waitFor(BoolCondition cond, long timeoutMillis) {
+ long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (System.currentTimeMillis() < deadline) {
+ if (cond.test()) return;
+ try {
+ Thread.sleep(20);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ Assert.fail("interrupted");
+ }
+ }
+ Assert.fail("waitFor timed out");
+ }
+
+ @FunctionalInterface
+ private interface BoolCondition {
+ boolean test();
+ }
+
+ /**
+ * Mirrors the server's per-connection delta-dictionary accumulation: the
+ * dictionary resets on every new connection, and each frame's delta section
+ * ({@code setQuick(deltaStart + i)}, null-padding to reach deltaStart) extends
+ * or overwrites it. A missing catch-up would show up here as a null gap.
+ */
+ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ // Set once the server has closed connection 1. A test waits on this
+ // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot
+ // race into connection 1's pre-close window and must land on the reconnect.
+ volatile boolean conn1Closed;
+ // Set from the flags byte of the zero-table catch-up frame on connection 2:
+ // the catch-up carries no rows and must defer its (empty) commit.
+ volatile boolean catchUpDeferredOnConn2;
+ // Set when connection 2 receives a DATA frame (tableCount > 0) whose delta
+ // starts ABOVE id 0 (deltaStart >= 1). This can only happen if the producer's
+ // monotonic baseline SURVIVED the reconnect: a reset would re-ship the whole
+ // dictionary from deltaStart 0. Robust to replay -- a replayed pre-reconnect
+ // frame carries its original deltaStart 0, so only a genuinely-above-baseline
+ // post-reconnect frame trips it.
+ volatile boolean conn2SawDeltaAboveBaseline;
+ volatile boolean sawZeroTableFrameOnConn2;
+ private final List> dictsByConn = new CopyOnWriteArrayList<>();
+ private TestWebSocketServer.ClientHandler currentClient;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ synchronized List dictFor(int connNumber) {
+ return connNumber <= dictsByConn.size()
+ // Copy under the lock: the caller iterates it unlocked while the
+ // server thread may still be appending to the live inner list.
+ ? new ArrayList<>(dictsByConn.get(connNumber - 1))
+ : new ArrayList<>();
+ }
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ boolean newConnection = currentClient != client;
+ if (newConnection) {
+ currentClient = client;
+ connectionsAccepted.incrementAndGet();
+ dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection
+ }
+ int connNumber = dictsByConn.size();
+ List dict = dictsByConn.get(connNumber - 1);
+ accumulate(data, dict);
+ if (connNumber == 2) {
+ if (tableCount(data) == 0) {
+ sawZeroTableFrameOnConn2 = true;
+ // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5).
+ catchUpDeferredOnConn2 = (data[5] & 0x01) != 0;
+ } else if (data.length >= 12 && (data[5] & 0x08) != 0) {
+ // A post-reconnect DATA frame carrying a delta section
+ // (FLAG_DELTA_SYMBOL_DICT = 0x08). A deltaStart >= 1 means the
+ // producer resumed the delta ABOVE the surviving baseline; a reset
+ // baseline would re-ship from deltaStart 0. Checking any frame (not
+ // just the first) keeps this robust to a replayed pre-reconnect
+ // frame arriving ahead of the new one -- that replay carries its
+ // original deltaStart 0 and does not trip the flag.
+ int[] pos = {12};
+ if (readVarint(data, pos) >= 1) {
+ conn2SawDeltaAboveBaseline = true;
+ }
+ }
+ }
+ try {
+ client.sendBinary(buildAck(nextSeq.getAndIncrement()));
+ // Drop the first connection right after ACKing its only frame,
+ // forcing the sender to reconnect onto a fresh dictionary.
+ if (connNumber == 1) {
+ Thread.sleep(50);
+ client.close();
+ conn1Closed = true;
+ }
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void accumulate(byte[] frame, List dict) {
+ final byte FLAG_DELTA_SYMBOL_DICT = 0x08;
+ if (frame.length < 12 || (frame[5] & FLAG_DELTA_SYMBOL_DICT) == 0) {
+ return;
+ }
+ int[] pos = {12}; // just past the 12-byte QWP header
+ int deltaStart = readVarint(frame, pos);
+ int deltaCount = readVarint(frame, pos);
+ while (dict.size() < deltaStart) {
+ dict.add(null); // null-pad, mirroring the server
+ }
+ for (int i = 0; i < deltaCount; i++) {
+ int len = readVarint(frame, pos);
+ String symbol = new String(frame, pos[0], len, StandardCharsets.UTF_8);
+ pos[0] += len;
+ int idx = deltaStart + i;
+ while (dict.size() <= idx) {
+ dict.add(null);
+ }
+ dict.set(idx, symbol);
+ }
+ }
+
+ private static byte[] buildAck(long seq) {
+ byte[] buf = new byte[1 + 8 + 2];
+ ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x00);
+ bb.putLong(seq);
+ bb.putShort((short) 0);
+ return buf;
+ }
+
+ private static int tableCount(byte[] frame) {
+ return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8);
+ }
+ }
+
+ /**
+ * ACKs connection 1's frames with no advertised cap (so an oversized symbol
+ * registers), then -- once connection 1 has sent something -- shrinks the
+ * advertised batch cap and drops the socket. The reconnect (connection 2)
+ * therefore advertises a cap whose catch-up budget is too small for the
+ * symbol, exercising the oversized-entry terminal in sendDictCatchUp.
+ */
+ private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ private final AtomicLong nextSeq = new AtomicLong(0);
+ private TestWebSocketServer.ClientHandler currentClient;
+ private volatile TestWebSocketServer server;
+
+ void setServer(TestWebSocketServer server) {
+ this.server = server;
+ }
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ connectionsAccepted.incrementAndGet();
+ }
+ try {
+ client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement()));
+ if (connectionsAccepted.get() == 1) {
+ // Connection 1 registered the big symbol. Shrink the cap so the
+ // reconnect's catch-up budget can't fit it, then drop to force
+ // the reconnect. Setting the cap before the close (not from the
+ // test thread after it) removes the set-vs-reconnect race.
+ server.setAdvertisedMaxBatchSize(160); // catch-up budget = 132
+ Thread.sleep(50);
+ client.close();
+ }
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Like {@link CatchUpHandler}, but drops connection 1 only after it has
+ * learned the whole batch, and counts the zero-table catch-up frames on
+ * connection 2 so a test can assert the dictionary replay split across
+ * several frames to respect the advertised batch cap.
+ */
+ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger connectionsAccepted = new AtomicInteger();
+ // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed).
+ volatile boolean conn1Closed;
+ volatile int zeroTableFramesOnConn2;
+ private final List> dictsByConn = new CopyOnWriteArrayList<>();
+ private final int dropConn1AtDictSize;
+ private final AtomicLong nextSeq = new AtomicLong(0);
+ private boolean conn1Dropped;
+ private TestWebSocketServer.ClientHandler currentClient;
+
+ SplitCatchUpHandler(int dropConn1AtDictSize) {
+ this.dropConn1AtDictSize = dropConn1AtDictSize;
+ }
+
+ synchronized List dictFor(int connNumber) {
+ return connNumber <= dictsByConn.size()
+ // Copy under the lock: the caller iterates it unlocked while the
+ // server thread may still be appending to the live inner list.
+ ? new ArrayList<>(dictsByConn.get(connNumber - 1))
+ : new ArrayList<>();
+ }
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ boolean newConnection = currentClient != client;
+ if (newConnection) {
+ currentClient = client;
+ connectionsAccepted.incrementAndGet();
+ dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection
+ }
+ int connNumber = dictsByConn.size();
+ List dict = dictsByConn.get(connNumber - 1);
+ CatchUpHandler.accumulate(data, dict);
+ if (connNumber == 2 && CatchUpHandler.tableCount(data) == 0) {
+ zeroTableFramesOnConn2++;
+ }
+ try {
+ client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement()));
+ // Drop connection 1 only once it has learned the entire batch, so
+ // the sender's sent-dictionary mirror is complete and the reconnect
+ // catch-up must replay a dictionary larger than the batch cap.
+ if (connNumber == 1 && !conn1Dropped && dict.size() >= dropConn1AtDictSize) {
+ conn1Dropped = true;
+ Thread.sleep(50);
+ client.close();
+ conn1Closed = true;
+ }
+ } catch (IOException | InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
new file mode 100644
index 00000000..fb355b82
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
@@ -0,0 +1,1045 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * End-to-end recovery for the delta symbol dictionary in store-and-forward mode.
+ *
+ * A file-mode sender writes delta-encoded SYMBOL frames (each frame carries only
+ * the ids it introduces) to a slot but never drains it -- simulating a crash. A
+ * fresh sender then recovers the slot and replays those non-self-sufficient
+ * frames to a brand-new server whose dictionary starts empty. Correctness hinges
+ * on the persisted {@code .symbol-dict}: the recovering sender loads it, the I/O
+ * thread re-registers the whole dictionary via a catch-up frame, and only then do
+ * the delta frames replay. This test reconstructs the server-side dictionary from
+ * the wire and asserts it comes out complete and gap-free.
+ */
+public class DeltaDictRecoveryTest {
+
+ private static final int DISTINCT_SYMBOLS = 8;
+ private static final int ROWS = 40;
+ private String sfDir;
+
+ @Before
+ public void setUp() {
+ sfDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-delta-recov-" + System.nanoTime()).toString();
+ }
+
+ @After
+ public void tearDown() {
+ TestUtils.removeTmpDirRec(sfDir);
+ }
+
+ @Test
+ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception {
+ assertMemoryLeak(() -> {
+ // Phase 1: silent server (no acks). Sender 1 writes symbol rows and
+ // close-fast (no drain), leaving unacked delta frames + a persisted
+ // dictionary in the slot.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+
+ String pad = TestUtils.repeat("x", 64);
+ String cfg = "ws::addr=localhost:" + port
+ + ";sf_dir=" + sfDir
+ + ";sf_max_bytes=4096"
+ + ";close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ for (int i = 0; i < ROWS; i++) {
+ s1.table("m")
+ .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS))
+ .stringColumn("p", pad)
+ .longColumn("v", i)
+ .atNow();
+ s1.flush();
+ }
+ }
+ }
+
+ // Ack a prefix so recovery does NOT replay from the self-sufficient head.
+ // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the
+ // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN
+ // DISTINCT_SYMBOLS onward -- frames whose delta starts at
+ // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle
+ // reuse existing ids). The early ids those frames reference then exist
+ // ONLY in the persisted dictionary, so the reconstructed dictionary below
+ // is complete solely because the catch-up frame re-registered them. That
+ // pins the content assertions to the catch-up: without it (or with a
+ // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1
+ // and the per-id checks would fail.
+ java.nio.file.Path slot = Paths.get(sfDir, "default");
+ writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1);
+
+ // Phase 2: fresh server that reconstructs its per-connection dictionary
+ // from the delta sections. Sender 2 recovers the slot and replays.
+ DictReconstructingHandler handler = new DictReconstructingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (Sender ignored = Sender.fromConfig(cfg)) {
+ long deadline = System.currentTimeMillis() + 5_000;
+ while (System.currentTimeMillis() < deadline
+ && handler.maxDictSize() < DISTINCT_SYMBOLS) {
+ Thread.sleep(20);
+ }
+ }
+
+ // The recovering sender must have re-registered the dictionary via a
+ // catch-up (0-table) frame before replaying delta frames.
+ Assert.assertTrue("recovery sent a full-dictionary catch-up frame",
+ handler.sawCatchUpFrame);
+ // The reconstructed dictionary must be complete and gap-free: exactly
+ // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id.
+ List dict = handler.dictSnapshot();
+ Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size());
+ for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+ Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i));
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception {
+ assertMemoryLeak(() -> {
+ // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i.
+ // Silent server + close-fast leaves all frames unacked in the slot.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ for (int i = 0; i < 6; i++) {
+ s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+ s1.flush();
+ }
+ }
+ }
+
+ // Simulate a host/power crash: the segment frames survive but the WHOLE
+ // persisted dictionary is lost (truncate .symbol-dict to its 8-byte header,
+ // 0 symbols), and the ack watermark was left mid-stream (FSN 2). The
+ // surviving frames still reference the lost ids.
+ java.nio.file.Path slot = Paths.get(sfDir, "default");
+ java.nio.file.Path dict = slot.resolve(".symbol-dict");
+ byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8);
+ java.nio.file.Files.write(dict, header);
+ writeAckWatermark(slot.resolve(".ack-watermark"), 2);
+
+ // Phase 2: recover against a fresh counting server. With the whole
+ // dictionary lost (size 0) while the surviving frames still reference its
+ // ids, seedGlobalDictionaryFromPersisted must refuse the resume at BUILD --
+ // its size==0 short-circuit must NOT skip the torn-dict guard -- rather than
+ // let the producer resume unseeded and silently misattribute reused ids.
+ CountingHandler handler = new CountingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ LineSenderException terminal = null;
+ try {
+ Sender s2 = Sender.fromConfig(cfg);
+ s2.close();
+ } catch (LineSenderException e) {
+ terminal = e;
+ }
+ Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary",
+ 0, handler.frames.get());
+ Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal);
+ Assert.assertTrue(terminal.getMessage(),
+ terminal.getMessage().contains("subset of the surviving frames")
+ && terminal.getMessage().contains("resend"));
+ }
+ });
+ }
+
+ @Test
+ public void testTornDictionaryOneIdGapFailsCleanly() throws Exception {
+ // One-id-gap boundary variant of
+ // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames: the first
+ // replayed frame starts EXACTLY ONE id past the empty mirror -- the tightest
+ // gap the I/O-thread replay guard must still reject (deltaStart ==
+ // sentDictCount + 1). It reaches that guard via the unopenable-dictionary path
+ // (deltaDictEnabled=false), NOT a size-0 openable dict, which now fails clean
+ // earlier in seedGlobalDictionaryFromPersisted. A one-entry-short tail is the
+ // common host-crash outcome, so this pins the guard's false-negative edge: a
+ // "deltaStart > size + 1" mutation would let this single-id gap through and
+ // null-pad the missing symbol on the server.
+ assertMemoryLeak(() -> {
+ // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i).
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ for (int i = 0; i < 6; i++) {
+ s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+ s1.flush();
+ }
+ }
+ }
+
+ // Make the persisted dictionary UNOPENABLE (replace .symbol-dict with a
+ // directory, so both openRW and openCleanRW fail and the engine reports
+ // deltaDictEnabled=false), then stamp the watermark at FSN 0 so recovery
+ // replays from FSN 1 -- a frame with deltaStart=1. The mirror is unseeded
+ // (size 0), so id 0 is the single missing entry: deltaStart(1) ==
+ // sentDictCount(0) + 1. No catch-up is sent, so any counted frame would be
+ // the gapped data frame.
+ java.nio.file.Path slot = Paths.get(sfDir, "default");
+ java.nio.file.Path dict = slot.resolve(".symbol-dict");
+ java.nio.file.Files.delete(dict);
+ java.nio.file.Files.createDirectory(dict);
+ writeAckWatermark(slot.resolve(".ack-watermark"), 0);
+
+ // Phase 2: recover against a fresh counting server. The guard must fire on
+ // the very first replay frame (deltaStart 1 > recovered size 0) and fail
+ // terminally rather than send a frame that null-pads id 0.
+ CountingHandler handler = new CountingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ LineSenderException terminal = null;
+ Sender s2 = Sender.fromConfig(cfg);
+ try {
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline && terminal == null) {
+ try {
+ s2.flush();
+ Thread.sleep(20);
+ } catch (LineSenderException e) {
+ terminal = e;
+ }
+ }
+ } finally {
+ try {
+ s2.close();
+ } catch (LineSenderException e) {
+ if (terminal == null) {
+ terminal = e;
+ }
+ }
+ }
+ Assert.assertEquals("a one-id gap must still block replay to a fresh server",
+ 0, handler.frames.get());
+ Assert.assertNotNull("a one-id-short dictionary must surface a terminal error", terminal);
+ Assert.assertTrue(terminal.getMessage(),
+ terminal.getMessage().contains("delta start 1 exceeds recovered dictionary size 0"));
+ }
+ });
+ }
+
+ @Test
+ public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception {
+ // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's
+ // dictionary and delta baseline from the persisted .symbol-dict, so a
+ // recovered sender that continues ingesting assigns the NEXT id, not a
+ // colliding low one. Without it the new symbol reuses a recovered id and the
+ // fresh server sees a redefinition -> silent misattribution. No prior test
+ // ingests on the recovered sender. Replay is from FSN 0 (no acks), so the
+ // recovered frames legitimately overlap the seeded dictionary -- this also
+ // pins that the redefinition guard does not false-positive on normal
+ // recovery.
+ assertMemoryLeak(() -> {
+ // Phase 1: ingest DISTINCT_SYMBOLS symbols, silent server, close-fast ->
+ // unacked frames + a full persisted dictionary.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";sf_max_bytes=4096;close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+ s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+ s1.flush();
+ }
+ }
+ }
+
+ // Phase 2: recover against a fresh server, then ingest a genuinely NEW
+ // symbol. The producer must continue at id DISTINCT_SYMBOLS.
+ DictReconstructingHandler handler = new DictReconstructingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (Sender s2 = Sender.fromConfig(cfg)) {
+ s2.table("m").symbol("s", "brand-new").longColumn("v", 99L).atNow();
+ s2.flush();
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline
+ && handler.maxDictSize() < DISTINCT_SYMBOLS + 1) {
+ Thread.sleep(20);
+ }
+ }
+ List dict = handler.dictSnapshot();
+ Assert.assertEquals("recovered sender must continue the dictionary, not collide: " + dict,
+ DISTINCT_SYMBOLS + 1, dict.size());
+ for (int i = 0; i < DISTINCT_SYMBOLS; i++) {
+ Assert.assertEquals("sym-" + i, dict.get(i));
+ }
+ Assert.assertEquals("brand-new", dict.get(DISTINCT_SYMBOLS));
+ }
+ });
+ }
+
+ @Test
+ public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception {
+ // C1 regression: when a recovered disk slot's persisted dictionary cannot be
+ // OPENED (fd exhaustion, a read-only remount, ENOSPC -- simulated here by a
+ // .symbol-dict that is a DIRECTORY, so both openRW and openCleanRW fail),
+ // CursorSendEngine.isDeltaDictEnabled() returns false. The recorded frames
+ // are still DELTA frames, and replaying them against a fresh
+ // empty-dictionary server would null-pad the missing ids and SILENTLY
+ // corrupt the table. The torn-dictionary guard must fire regardless of
+ // deltaDictEnabled -- pre-fix it was gated on that very flag, so the
+ // corrupting frame sailed through unguarded. Unlike
+ // testTornDictionaryFailsCleanlyInsteadOfCorrupting (dict present but empty,
+ // deltaDictEnabled=true), here the dict is UNOPENABLE (deltaDictEnabled=false).
+ assertMemoryLeak(() -> {
+ // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ for (int i = 0; i < 6; i++) {
+ s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+ s1.flush();
+ }
+ }
+ }
+
+ // Make the persisted dictionary UNOPENABLE: replace the .symbol-dict file
+ // with a directory of the same name so PersistedSymbolDict.open() returns
+ // null (both openRW and openCleanRW fail) and the engine reports
+ // deltaDictEnabled=false. Stamp the watermark at FSN 2 so replay starts
+ // at FSN 3 -- a frame whose delta starts at id 3, with ids 0..2 living
+ // only in the now-unreadable dictionary.
+ java.nio.file.Path slot = Paths.get(sfDir, "default");
+ java.nio.file.Path dict = slot.resolve(".symbol-dict");
+ java.nio.file.Files.delete(dict);
+ java.nio.file.Files.createDirectory(dict);
+ writeAckWatermark(slot.resolve(".ack-watermark"), 2);
+
+ // Phase 2: recover against a fresh counting server. The guard must fire
+ // (frame deltaStart 3 > recovered dictionary size 0) and fail terminally
+ // rather than send a gapped frame that would corrupt the table.
+ CountingHandler handler = new CountingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ LineSenderException terminal = null;
+ Sender s2 = Sender.fromConfig(cfg);
+ try {
+ long deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline && terminal == null) {
+ try {
+ s2.flush();
+ Thread.sleep(20);
+ } catch (LineSenderException e) {
+ terminal = e;
+ }
+ }
+ } finally {
+ try {
+ s2.close();
+ } catch (LineSenderException e) {
+ if (terminal == null) {
+ terminal = e;
+ }
+ }
+ }
+ Assert.assertEquals("no delta frame may be replayed when the persisted dictionary is unopenable",
+ 0, handler.frames.get());
+ Assert.assertNotNull("an unopenable dictionary must surface a terminal error", terminal);
+ Assert.assertTrue(terminal.getMessage(),
+ terminal.getMessage().contains("symbol dictionary is incomplete"));
+ }
+ });
+ }
+
+ @Test
+ public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception {
+ // C3 regression: sendCommitMessage does NOT write-ahead-persist the
+ // dictionary, so its frame must carry NO new symbol. A symbol left in the
+ // batch by a cancelled row -- cancelRow rolls back neither
+ // currentBatchMaxSymbolId nor the global-dictionary registration -- must not
+ // ride out on the commit frame: doing so puts an id on the wire that a
+ // recovered slot cannot rebuild from .symbol-dict, diverging the producer
+ // dictionary from the surviving frames and silently misattributing reused
+ // ids after a crash. The commit's delta must be bounded by sentMaxSymbolId
+ // (empty here), not currentBatchMaxSymbolId. Memory mode suffices to observe
+ // the wire behaviour; close() drains every frame to the server first.
+ assertMemoryLeak(() -> {
+ DictReconstructingHandler handler = new DictReconstructingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Transactional + autoFlushRows=1: every completed row auto-flushes
+ // as a DEFERRED batch (setting hasDeferredMessages); an explicit
+ // flush() then emits the commit message.
+ Sender sender = Sender.builder("ws::addr=localhost:" + port + ";")
+ .transactional(true)
+ .autoFlushRows(1)
+ .build();
+ try {
+ // Row 1 registers "a"@0 and auto-flushes it deferred.
+ sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow();
+ // Register "b"@1 on a row that is then cancelled: "b" stays in
+ // the global dictionary and currentBatchMaxSymbolId advances to
+ // 1, but nothing persists or sends it.
+ sender.table("m").symbol("s", "b");
+ sender.cancelRow();
+ // Commit the deferred batch. The commit frame must carry an
+ // EMPTY delta -- NOT "b"@1.
+ sender.flush();
+ } finally {
+ sender.close(); // drains every frame (incl. the commit) to the server
+ }
+
+ // The server's reconstructed dictionary must hold ONLY "a". Pre-fix
+ // the commit shipped "b"@1, so the server saw a second symbol.
+ List dict = handler.dictSnapshot();
+ Assert.assertEquals("commit frame must not ship the cancelled row's leaked symbol "
+ + "(recovery would then diverge from the persisted dictionary): " + dict,
+ 1, dict.size());
+ Assert.assertEquals("a", dict.get(0));
+ }
+ });
+ }
+
+ @Test
+ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception {
+ // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs
+ // BEFORE the frame is published (sealAndSwapBuffer -> appendBlocking). If
+ // publish fails (here PAYLOAD_TOO_LARGE, a frame bigger than the SF
+ // segment; a backpressure deadline in production), the frame's symbols are
+ // already on disk but sentMaxSymbolId is NOT advanced and the rows stay
+ // buffered -- so a retry re-runs the persist. Keying the persist range off
+ // pd.size() (not sentMaxSymbolId+1) makes it idempotent. Before that fix,
+ // the retry appended the symbol a SECOND time, breaking the dense
+ // id->position invariant; on recovery every later global id shifts by one
+ // and symbol column values are silently misattributed.
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+
+ // Small segment; a heavily padded row's frame cannot fit, so
+ // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no
+ // backpressure timing needed. The server never acks (SilentHandler).
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;";
+ String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment
+ Sender sender = Sender.fromConfig(cfg);
+ try {
+ // Buffer ONE new-symbol row, then flush it TWICE. Each flush
+ // runs the write-ahead persist and then fails to publish; the
+ // failed flush leaves the row buffered, so the second flush is
+ // the retry that (pre-fix) duplicated the persisted symbol.
+ sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ for (int attempt = 0; attempt < 2; attempt++) {
+ try {
+ sender.flush();
+ Assert.fail("oversized frame must fail to publish");
+ } catch (LineSenderException expected) {
+ // frame too large -- expected on every attempt
+ }
+ }
+
+ // The persisted dictionary must hold "s0" EXACTLY ONCE.
+ // Pre-fix, the retry duplicated it (size == 2).
+ PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString());
+ Assert.assertNotNull(pd);
+ try {
+ Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol",
+ 1, pd.size());
+ Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0));
+ } finally {
+ pd.close();
+ }
+ } finally {
+ try {
+ sender.close();
+ } catch (LineSenderException ignored) {
+ // close() re-flushes the still-buffered oversized row and
+ // fails again (PAYLOAD_TOO_LARGE); expected here and not
+ // what we assert. close() still runs its resource cleanup,
+ // so no native memory leaks.
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() throws Exception {
+ // Regression for the re-encode (else) branch of persistNewSymbolsBeforePublish.
+ // After a failed publish leaves the durable dictionary size ahead of the wire
+ // baseline (pd.size() > sentMaxSymbolId+1), a later flush that introduces a NEW
+ // symbol cannot reuse the frame's already-encoded fast-path bytes -- it
+ // re-encodes just the [pd.size() .. currentBatchMaxSymbolId] suffix via
+ // appendSymbols. Keying that off pd.size() (not sentMaxSymbolId+1) keeps it
+ // idempotent: the already-persisted prefix is NOT re-appended.
+ assertMemoryLeak(() -> {
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ // Small segment: any frame carrying the padded row fails to publish with
+ // PAYLOAD_TOO_LARGE deterministically (no backpressure timing).
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;";
+ String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment
+ Sender sender = Sender.fromConfig(cfg);
+ try {
+ // Flush 1: s0 is persisted (write-ahead) before the oversized frame
+ // fails to publish. pd.size()=1, sentMaxSymbolId stays -1.
+ sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("oversized frame must fail to publish");
+ } catch (LineSenderException expected) {
+ }
+
+ // Add a NEW symbol s1 (id 1). The failed s0 row is still buffered, so
+ // the batch is {s0, s1} and the durable size (1) has run ahead of the
+ // wire baseline (-1) -- the state that selects the appendSymbols branch.
+ sender.table("m").symbol("s", "s1").stringColumn("p", pad).longColumn("v", 2L).atNow();
+ try {
+ sender.flush();
+ Assert.fail("oversized frame must fail to publish");
+ } catch (LineSenderException expected) {
+ }
+
+ // The else branch persisted ONLY s1 (the suffix). The dictionary holds
+ // s0, s1 exactly once each. Pre-fix (appendSymbols from
+ // sentMaxSymbolId+1) re-appended s0, giving size 3.
+ PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString());
+ Assert.assertNotNull(pd);
+ try {
+ Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix",
+ 2, pd.size());
+ Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0));
+ Assert.assertEquals("s1", pd.readLoadedSymbols().getQuick(1));
+ } finally {
+ pd.close();
+ }
+ } finally {
+ try {
+ sender.close();
+ } catch (LineSenderException ignored) {
+ // close() re-flushes the still-buffered oversized rows and fails
+ // again (PAYLOAD_TOO_LARGE); expected, resources still released.
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception {
+ // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay.
+ // A failed publish persists the frame's symbol (write-ahead) but does NOT
+ // record the frame, so the persisted dictionary becomes a strict SUPERSET of
+ // the recorded frames' references. A recovering sender must still replay
+ // gap-free: it re-registers the whole (superset) dictionary via the catch-up
+ // -- including the symbol whose frame never reached disk -- so the fresh
+ // server reconstructs a complete, gap-free dictionary. The sibling
+ // testFailedPublishDoesNotDuplicatePersistedSymbols proves the dict has no
+ // duplicate after the failed publish; this proves the resulting slot then
+ // recovers and replays end-to-end against a real server.
+ assertMemoryLeak(() -> {
+ // Phase 1: a silent server (no acks) + a small SF segment. Four small
+ // rows register sym-0..sym-3 and their frames are recorded; a fifth,
+ // oversized row registers (persists) sym-4 but its frame is too large for
+ // the segment, so appendBlocking throws and the frame is NOT recorded.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port
+ + ";sf_dir=" + sfDir
+ + ";sf_max_bytes=4096"
+ + ";close_flush_timeout_millis=0;";
+ Sender s1 = Sender.fromConfig(cfg);
+ try {
+ for (int i = 0; i < 4; i++) {
+ s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow();
+ s1.flush();
+ }
+ // Oversized frame: sym-4 is persisted (write-ahead) before the
+ // publish fails, so the dictionary runs one ahead of the frames.
+ s1.table("m").symbol("s", "sym-4")
+ .stringColumn("p", TestUtils.repeat("x", 8000))
+ .longColumn("v", 4).atNow();
+ try {
+ s1.flush();
+ Assert.fail("oversized frame must fail to publish");
+ } catch (LineSenderException expected) {
+ // PAYLOAD_TOO_LARGE -- frame not recorded, sym-4 stays persisted
+ }
+ } finally {
+ try {
+ // Re-flushes the still-buffered oversized row and fails again
+ // (expected); resources are still released, and the idempotent
+ // write-ahead does not re-append sym-4.
+ s1.close();
+ } catch (LineSenderException ignored) {
+ }
+ }
+ }
+
+ // The persisted dictionary must hold the superset: sym-0..sym-4 (5 ids),
+ // one more than the four recorded frames reference, with no duplicate.
+ PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString());
+ Assert.assertNotNull(pd);
+ try {
+ Assert.assertEquals("failed publish must leave the dict a superset (sym-0..sym-4)",
+ 5, pd.size());
+ } finally {
+ pd.close();
+ }
+
+ // Phase 2: recover against a fresh server that reconstructs its
+ // dictionary from the wire. The recovering sender must re-register all 5
+ // symbols via a catch-up (sym-4 exists ONLY in the dictionary -- no frame
+ // carries it) and replay the 4 recorded frames, leaving a complete,
+ // gap-free server dictionary.
+ DictReconstructingHandler handler = new DictReconstructingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (Sender ignored = Sender.fromConfig(cfg)) {
+ long deadline = System.currentTimeMillis() + 5_000;
+ while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 5) {
+ Thread.sleep(20);
+ }
+ }
+ Assert.assertTrue("recovery must send a full-dictionary catch-up frame",
+ handler.sawCatchUpFrame);
+ List dict = handler.dictSnapshot();
+ Assert.assertEquals("recovered dictionary must include the failed-publish symbol",
+ 5, dict.size());
+ for (int i = 0; i < 5; i++) {
+ Assert.assertEquals("dictionary id " + i + " must be gap-free",
+ "sym-" + i, dict.get(i));
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Exception {
+ // M2 regression: two DISTINCT source symbols that collapse to the SAME UTF-8
+ // bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get
+ // distinct producer ids and persist as separate entries. On recovery the
+ // producer must rebuild its id space to match the persisted entry count
+ // exactly. Pre-fix, seedGlobalDictionaryFromPersisted used getOrAddSymbol,
+ // which de-duped the two decoded "?" strings, leaving the producer
+ // dictionary (and sentMaxSymbolId) one short of pd.size() -- desyncing from
+ // the send-loop catch-up mirror (which uses pd.size()) and silently
+ // misattributing later symbols after a reconnect. addRecoveredSymbol appends
+ // without de-duping, keeping producer and mirror in lockstep.
+ assertMemoryLeak(() -> {
+ // Phase 1: ingest two lone-surrogate symbols in file mode, close-fast.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ s1.table("m").symbol("s", "\uD800").longColumn("v", 1L).atNow(); // lone surrogate -> '?'
+ s1.flush();
+ s1.table("m").symbol("s", "\uD801").longColumn("v", 2L).atNow(); // a DIFFERENT one -> '?'
+ s1.flush();
+ }
+ }
+
+ // The persisted dictionary holds TWO entries (both encode to '?').
+ int persistedSize;
+ PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString());
+ Assert.assertNotNull(pd);
+ try {
+ persistedSize = pd.size();
+ } finally {
+ pd.close();
+ }
+ Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize);
+
+ // Phase 2: recover. The seeded producer id space must match pd.size().
+ DictReconstructingHandler handler = new DictReconstructingHandler();
+ try (TestWebSocketServer good = new TestWebSocketServer(handler)) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (Sender s2 = Sender.fromConfig(cfg)) {
+ Assert.assertEquals("recovered producer dictionary must match the persisted "
+ + "entry count (not de-duped), else the delta baseline desyncs from the mirror",
+ persistedSize, globalDictSize(s2));
+ Assert.assertEquals("delta baseline must resume at the persisted tip",
+ persistedSize - 1, intField(s2, "sentMaxSymbolId"));
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testTornDictSubsetFailsCleanOnResume() throws Exception {
+ // C1 regression: the persisted .symbol-dict is NOT fsync'd, so a host crash
+ // can lose its highest-id tail entries while the segment frames that
+ // introduced those ids survive (out-of-order page loss). Recovery then seeds
+ // the producer from the SHORTER dictionary, but the I/O-thread catch-up mirror
+ // rebuilds the missing ids from those surviving frames -- so a resumed producer
+ // would assign its next new symbol an id the frames already define, silently
+ // misattributing symbol values on the wire. The send loop's replay guard only
+ // catches a GAP (deltaStart > mirror); a frame introducing exactly the
+ // torn-off id slips through and self-heals the mirror, leaving only this
+ // producer diverged. seedGlobalDictionaryFromPersisted must detect that the
+ // surviving frames reference an id at/beyond the recovered dictionary size and
+ // fail clean. Without the fix the resume succeeds and the reuse corrupts
+ // silently -- so the build below would NOT throw and the test's fail() fires.
+ assertMemoryLeak(() -> {
+ // Phase 1: record three delta frames (a@0, b@1, c@2) to disk with nothing
+ // acked (silent server), so all three survive and replay on recovery.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ Sender s1 = Sender.fromConfig(cfg);
+ try {
+ s1.table("m").symbol("s", "a").longColumn("v", 0).atNow();
+ s1.flush();
+ s1.table("m").symbol("s", "b").longColumn("v", 1).atNow();
+ s1.flush();
+ s1.table("m").symbol("s", "c").longColumn("v", 2).atNow();
+ s1.flush();
+ } finally {
+ s1.close();
+ }
+ }
+
+ // Simulate the host-crash tear: rewrite the persisted dictionary to drop
+ // its highest entry (c@2), keeping a@0,b@1 -- while the frame referencing
+ // c@2 stays on disk. openClean truncates; the two appends leave the exact
+ // two-entry dictionary a torn tail would recover to.
+ String slotDir = Paths.get(sfDir, "default").toString();
+ try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) {
+ Assert.assertNotNull(torn);
+ torn.appendSymbol("a");
+ torn.appendSymbol("b");
+ Assert.assertEquals(2, torn.size());
+ }
+
+ // Phase 2: a resuming sender must refuse -- the surviving frames reference
+ // id 2 but the recovered dictionary holds only 2 id(s) [0,1].
+ try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try {
+ Sender resumed = Sender.fromConfig(cfg);
+ resumed.close();
+ Assert.fail("resume on a torn (subset) dictionary must fail clean");
+ } catch (LineSenderException expected) {
+ Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(),
+ expected.getMessage().contains("subset of the surviving frames")
+ && expected.getMessage().contains("resend"));
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testTornDictTotalLossFailsCleanOnResume() throws Exception {
+ // C1 regression (TOTAL-loss variant of testTornDictSubsetFailsCleanOnResume):
+ // a host crash can lose the ENTIRE persisted .symbol-dict (size 0) while the
+ // frames that introduced its ids survive; equivalently, a prior session whose
+ // openClean failed ran full-dict mode and never wrote the side-file, so this
+ // session recovers those frames against a FRESH EMPTY dictionary. The recovered
+ // frames start at deltaStart=0 and self-heal the I/O-thread catch-up mirror, so
+ // the send loop's replay guard (deltaStart > mirror) never fires -- only
+ // seedGlobalDictionaryFromPersisted can catch it. Its pd.size()==0 early return
+ // must NOT skip the torn-dict guard: the surviving frames reference id 2 while
+ // the recovered dictionary holds 0 id(s), so resuming unseeded would reuse those
+ // ids and silently misattribute symbol values on the wire. Without the fix the
+ // build below does NOT throw (the guard is bypassed) and the test's fail() fires.
+ assertMemoryLeak(() -> {
+ // Phase 1: record three delta frames (a@0, b@1, c@2) with nothing acked, so
+ // all three survive and replay -- from FSN 0 (deltaStart=0), the case the
+ // I/O-thread guard self-heals rather than rejects.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ Sender s1 = Sender.fromConfig(cfg);
+ try {
+ s1.table("m").symbol("s", "a").longColumn("v", 0).atNow();
+ s1.flush();
+ s1.table("m").symbol("s", "b").longColumn("v", 1).atNow();
+ s1.flush();
+ s1.table("m").symbol("s", "c").longColumn("v", 2).atNow();
+ s1.flush();
+ } finally {
+ s1.close();
+ }
+ }
+
+ // Total dictionary loss: openClean truncates .symbol-dict to its bare header
+ // (size 0) with zero appends, while every frame -- including the ones
+ // referencing a@0,b@1,c@2 -- stays on disk. This is the fresh-empty-dict
+ // state a full-dict-mode prior session or a total host-crash tear leaves.
+ String slotDir = Paths.get(sfDir, "default").toString();
+ try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) {
+ Assert.assertNotNull(torn);
+ Assert.assertEquals(0, torn.size());
+ }
+
+ // Phase 2: a resuming sender must refuse at build -- the surviving frames
+ // reference id 2 but the recovered dictionary holds 0 id(s). The I/O-thread
+ // guard cannot help (deltaStart 0 self-heals the mirror), so the seed-time
+ // guard is the sole defense.
+ try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) {
+ int port = good.getPort();
+ good.start();
+ Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try {
+ Sender resumed = Sender.fromConfig(cfg);
+ resumed.close();
+ Assert.fail("resume on a totally lost (size 0) dictionary must fail clean");
+ } catch (LineSenderException expected) {
+ Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(),
+ expected.getMessage().contains("subset of the surviving frames")
+ && expected.getMessage().contains("resend"));
+ }
+ }
+ });
+ }
+
+ private static int globalDictSize(Sender sender) throws Exception {
+ Field f = sender.getClass().getDeclaredField("globalSymbolDictionary");
+ f.setAccessible(true);
+ Object dict = f.get(sender);
+ return (int) dict.getClass().getMethod("size").invoke(dict);
+ }
+
+ private static int intField(Sender sender, String name) throws Exception {
+ Field f = sender.getClass().getDeclaredField(name);
+ f.setAccessible(true);
+ return f.getInt(sender);
+ }
+
+ private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException {
+ byte[] buf = new byte[16];
+ ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+ bb.putInt(0x31574B41); // 'AKW1'
+ bb.putInt(0); // reserved
+ bb.putLong(fsn);
+ java.nio.file.Files.write(path, buf);
+ }
+
+ private static int readVarint(byte[] buf, int[] pos) {
+ int result = 0;
+ int shift = 0;
+ while (pos[0] < buf.length) {
+ int b = buf[pos[0]++] & 0xFF;
+ result |= (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ return result;
+ }
+ shift += 7;
+ if (shift > 28) {
+ throw new IllegalStateException("varint too long");
+ }
+ }
+ throw new IllegalStateException("varint truncated");
+ }
+
+ /**
+ * Reconstructs the per-connection symbol dictionary from delta sections,
+ * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding.
+ */
+ private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler {
+ volatile boolean sawCatchUpFrame;
+ private final List dict = new ArrayList<>();
+ private final AtomicLong nextSeq = new AtomicLong(0);
+ private TestWebSocketServer.ClientHandler currentClient;
+
+ synchronized List dictSnapshot() {
+ return new ArrayList<>(dict);
+ }
+
+ synchronized int maxDictSize() {
+ return dict.size();
+ }
+
+ @Override
+ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ if (currentClient != client) {
+ currentClient = client;
+ dict.clear(); // fresh server dictionary per connection
+ }
+ accumulate(data);
+ if (tableCount(data) == 0 && hasDelta(data)) {
+ sawCatchUpFrame = true;
+ }
+ try {
+ client.sendBinary(buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static byte[] buildAck(long seq) {
+ byte[] buf = new byte[1 + 8 + 2];
+ ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x00);
+ bb.putLong(seq);
+ bb.putShort((short) 0);
+ return buf;
+ }
+
+ private static boolean hasDelta(byte[] frame) {
+ return frame.length >= 12 && (frame[5] & 0x08) != 0;
+ }
+
+ private static int tableCount(byte[] frame) {
+ return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8);
+ }
+
+ private void accumulate(byte[] frame) {
+ if (!hasDelta(frame)) {
+ return;
+ }
+ int[] pos = {12};
+ int deltaStart = readVarint(frame, pos);
+ int deltaCount = readVarint(frame, pos);
+ while (dict.size() < deltaStart) {
+ dict.add(null);
+ }
+ for (int i = 0; i < deltaCount; i++) {
+ int len = readVarint(frame, pos);
+ String sym = new String(frame, pos[0], len, StandardCharsets.UTF_8);
+ pos[0] += len;
+ int idx = deltaStart + i;
+ while (dict.size() <= idx) {
+ dict.add(null);
+ }
+ dict.set(idx, sym);
+ }
+ }
+ }
+
+ private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler {
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ // never acks -- sender leaves everything unacked in the slot
+ }
+ }
+
+ /** Counts every binary frame it receives and acks it. */
+ private static class CountingHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger frames = new AtomicInteger();
+ private final AtomicLong nextSeq = new AtomicLong(0);
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ frames.incrementAndGet();
+ try {
+ client.sendBinary(buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static byte[] buildAck(long seq) {
+ byte[] buf = new byte[1 + 8 + 2];
+ ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x00);
+ bb.putLong(seq);
+ bb.putShort((short) 0);
+ return buf;
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
index b8945d40..2f38ab02 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java
@@ -31,6 +31,36 @@
public class GlobalSymbolDictionaryTest {
+ @Test
+ public void testAddRecoveredSymbol_appendsWithoutDeduplicating() {
+ // Recovery replays persisted entries in id order. Distinct source strings
+ // that decode to the same characters -- lone UTF-16 surrogates both
+ // UTF-8-encode to '?', so they read back as the string "?" -- must keep
+ // DISTINCT ids, so the producer id space matches the persisted entry count.
+ // getOrAddSymbol de-dups them; addRecoveredSymbol must not.
+ GlobalSymbolDictionary dedup = new GlobalSymbolDictionary();
+ dedup.getOrAddSymbol("?");
+ dedup.getOrAddSymbol("?");
+ assertEquals("getOrAddSymbol de-dups colliding strings", 1, dedup.size());
+
+ GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+ assertEquals(0, recovered.addRecoveredSymbol("?"));
+ assertEquals(1, recovered.addRecoveredSymbol("?"));
+ assertEquals(2, recovered.addRecoveredSymbol("nvda"));
+ assertEquals("addRecoveredSymbol keeps colliding entries distinct", 3, recovered.size());
+
+ // Dense id -> symbol mapping is preserved position-for-position.
+ assertEquals("?", recovered.getSymbol(0));
+ assertEquals("?", recovered.getSymbol(1));
+ assertEquals("nvda", recovered.getSymbol(2));
+
+ // A later ingest of a colliding string reuses the highest recovered id
+ // (harmless -- both encode to identical bytes), and a genuinely new symbol
+ // continues past the recovered tip.
+ assertEquals(1, recovered.getOrAddSymbol("?"));
+ assertEquals(3, recovered.getOrAddSymbol("brand-new"));
+ }
+
@Test
public void testAddSymbol_assignsSequentialIds() {
GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java
index 36553388..7ba7a1fe 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java
@@ -52,9 +52,10 @@
* disconnect = sender broken, every subsequent batch threw. Reconnect
* machinery now handles transient drops: detect, build a fresh client
* via the registered factory, reset wire state, and reposition the replay
- * cursor at {@code engine.ackedFsn() + 1}. Cursor frames are self-sufficient
- * (every frame carries full schema + full symbol-dict delta), so post-reconnect
- * replay needs no producer-side schema-reset signal.
+ * cursor at {@code engine.ackedFsn() + 1}. Every frame carries its full schema
+ * inline; the symbol dictionary is either shipped in full per frame (file-mode
+ * store-and-forward) or re-registered by an I/O-thread catch-up frame before
+ * replay (memory-mode), so post-reconnect replay needs no producer-side reset.
*
* This commit covers the mechanics with a single-attempt retry; backoff,
* per-outage time cap, and auth-failure detection follow.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
index 45275528..5729aa44 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
@@ -25,30 +25,36 @@
package io.questdb.client.test.cutlass.qwp.client;
import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
/**
- * Pins down the "every frame on disk is self-sufficient" rule for the symbol
- * dictionary.
+ * Pins down how the symbol dictionary is framed on the wire.
*
- * The cursor SF path used to elide previously-sent symbols on subsequent
- * batches over the same connection, emitting a delta-dict that carried only
- * the new entries. That's wrong for SF: the bytes survive process restarts and
- * replay against fresh server connections (post-reconnect, or via a background
- * drainer adopting an orphan slot). A delta that references symbol ids the new
- * server has never seen is unrecoverable.
+ * Both engine modes ship monotonic deltas -- each symbol id travels once,
+ * not the whole dictionary per message -- which is the bandwidth win this feature
+ * adds. The I/O thread re-registers the dictionary with a catch-up frame whenever
+ * it (re)connects, so a fresh server can resolve the non-self-sufficient delta
+ * frames that follow.
*
- * Today every frame must carry a complete symbol-dict delta starting at id 0
- * (column schemas travel inline on the first batch too). This test asserts the
- * symbol-dict invariant on the wire.
+ * The modes differ only in where the catch-up's dictionary comes from: memory
+ * mode keeps it in an in-process mirror; file-backed store-and-forward keeps it in
+ * a per-slot {@code .symbol-dict} file so a recovered or orphan-drained slot (a
+ * fresh process with no in-memory mirror) can rebuild it. This test asserts the
+ * monotonic wire framing in both modes and the presence of that dictionary file.
*/
public class SelfSufficientFramesTest {
@@ -56,58 +62,394 @@ public class SelfSufficientFramesTest {
private static final int DELTA_START_OFFSET = 12;
@Test
- public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception {
- // Send two batches against the same connection, each with a
- // distinct symbol value. With the old schema-ref/delta encoding,
- // batch 2 would emit deltaStart=1, deltaCount=1 โ only the new
- // symbol. With self-sufficient frames, batch 2 must emit
- // deltaStart=0 covering BOTH symbols.
- CapturingHandler handler = new CapturingHandler();
- try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
- int port = server.getPort();
- server.start();
- Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
-
- try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
- sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
- sender.flush();
- waitFor(() -> handler.batches.size() >= 1, 5_000);
-
- sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
- sender.flush();
- waitFor(() -> handler.batches.size() >= 2, 5_000);
+ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception {
+ // File-backed SF also ships monotonic deltas now: batch 2 carries only
+ // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict
+ // so a recovered/orphan-drained slot can rebuild it.
+ Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient");
+ try {
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ // The engine places slot files under sf_dir/ (default "default").
+ Path dictFile = sfDir.resolve("default").resolve(".symbol-dict");
+ try (Sender sender = Sender.fromConfig(config)) {
+ sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 1, 5_000);
+
+ sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 2, 5_000);
+
+ // Check the persisted dictionary while the sender is live: a
+ // fully-drained close intentionally unlinks it (slot cleanup).
+ Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile));
+ byte[] dict = Files.readAllBytes(dictFile);
+ Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha"));
+ Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta"));
+ }
+
+ Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
+ byte[] b1 = handler.batches.get(0);
+ byte[] b2 = handler.batches.get(1);
+
+ Assert.assertEquals("batch 1 deltaStart must be 0",
+ 0, readVarint(b1, DELTA_START_OFFSET));
+ Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1));
+ // batch 2 ships ONLY beta as a delta from id 1.
+ Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)",
+ 1, readVarint(b2, DELTA_START_OFFSET));
+ Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)",
+ 1, readVarint(b2, DELTA_START_OFFSET + 1));
+ }
+ });
+ } finally {
+ rmDir(sfDir);
+ }
+ }
+
+ @Test
+ public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Exception {
+ // When the per-slot .symbol-dict cannot be opened in disk mode,
+ // isDeltaDictEnabled() is false and the sender must fall back to
+ // self-sufficient frames: every batch re-ships the WHOLE dictionary from
+ // id 0. A recovered / orphan-drained slot then has no dictionary to
+ // rebuild deltas from, so a monotonic delta would dangle ids on the fresh
+ // server -- the full-dict frame is the safe degradation. Force the open
+ // failure by planting a DIRECTORY where the dictionary file belongs:
+ // openRW / openCleanRW on a directory fails, so open() returns null.
+ Path sfDir = Files.createTempDirectory("qwp-sf-fallback");
+ Path dictPath = sfDir.resolve("default").resolve(".symbol-dict");
+ Files.createDirectories(dictPath); // a directory, not a file
+ Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd
+ try {
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (Sender sender = Sender.fromConfig(config)) {
+ sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 1, 5_000);
+
+ sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 2, 5_000);
+ }
+
+ // The planted directory is untouched -- the dictionary never
+ // opened, so delta encoding stayed disabled.
+ Assert.assertTrue("planted .symbol-dict directory must remain (open failed)",
+ Files.isDirectory(dictPath));
+
+ Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
+ byte[] b1 = handler.batches.get(0);
+ byte[] b2 = handler.batches.get(1);
+
+ // Full-dict fallback: BOTH batches start at id 0, and batch 2
+ // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic
+ // delta (which would be deltaStart=1, deltaCount=1 as above).
+ Assert.assertEquals("batch 1 deltaStart must be 0",
+ 0, readVarint(b1, DELTA_START_OFFSET));
+ Assert.assertEquals("batch 1 deltaCount must be 1",
+ 1, readVarint(b1, DELTA_START_OFFSET + 1));
+ Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)",
+ 0, readVarint(b2, DELTA_START_OFFSET));
+ Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)",
+ 2, readVarint(b2, DELTA_START_OFFSET + 1));
+ }
+ });
+ } finally {
+ rmDir(sfDir);
+ }
+ }
+
+ private static boolean containsUtf8(byte[] haystack, String needle) {
+ byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ outer:
+ for (int i = 0; i + n.length <= haystack.length; i++) {
+ for (int j = 0; j < n.length; j++) {
+ if (haystack[i + j] != n[j]) {
+ continue outer;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Test
+ public void testMemoryModeShipsMonotonicDelta() throws Exception {
+ // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries
+ // only "beta" as a delta starting at id 1, not the whole dictionary.
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+ sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 1, 5_000);
+
+ sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 2, 5_000);
+ }
+
+ Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
+ byte[] b1 = handler.batches.get(0);
+ byte[] b2 = handler.batches.get(1);
+
+ // Batch 1 introduces alpha at id 0.
+ Assert.assertEquals("batch 1 deltaStart must be 0",
+ 0, readVarint(b1, DELTA_START_OFFSET));
+ Assert.assertEquals("batch 1 deltaCount must be 1",
+ 1, readVarint(b1, DELTA_START_OFFSET + 1));
+
+ // Batch 2 ships ONLY beta as a delta from id 1.
+ Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)",
+ 1, readVarint(b2, DELTA_START_OFFSET));
+ Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)",
+ 1, readVarint(b2, DELTA_START_OFFSET + 1));
+ }
+ });
+ }
+
+ @Test
+ public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception {
+ // A single flush whose encoded size exceeds the server's batch cap is
+ // split into one frame per table (flushPendingRowsSplit). The FIRST split
+ // frame carries the whole batch's symbol-dict delta and advances the
+ // baseline; the remaining frames carry an EMPTY delta and only reference
+ // ids the first frame already registered. A regression that shipped each
+ // table's own symbols (wrong deltaStart) would dangle ids on the server.
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Padding inflates each table past half the cap, so the combined
+ // two-table message exceeds it while each single-table frame fits.
+ String pad = new String(new char[60]).replace('\0', 'x');
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) {
+ // Buffer TWO tables (symbols "a" id 0, "b" id 1), then ONE flush.
+ sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 2, 5_000);
+ }
+
+ Assert.assertEquals("the oversized two-table batch must split into 2 frames",
+ 2, handler.batches.size());
+ byte[] f1 = handler.batches.get(0);
+ byte[] f2 = handler.batches.get(1);
+
+ // First split frame ships the whole batch's dictionary (a + b).
+ Assert.assertEquals("first split frame deltaStart must be 0",
+ 0, readVarint(f1, DELTA_START_OFFSET));
+ Assert.assertEquals("first split frame ships both new symbols",
+ 2, readVarint(f1, DELTA_START_OFFSET + 1));
+ // Second split frame carries an empty delta above the advanced baseline.
+ Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)",
+ 2, readVarint(f2, DELTA_START_OFFSET));
+ Assert.assertEquals("second split frame carries no new symbols",
+ 0, readVarint(f2, DELTA_START_OFFSET + 1));
}
+ });
+ }
+
+ @Test
+ public void testOversizedTableSplitStrandsNothing() throws Exception {
+ // Regression: flushPendingRowsSplit publishes each table's frame one at a
+ // time (all but the last deferred, i.e. appended but uncommitted). If a LATER
+ // table's frame exceeds the cap, the split must not have already published an
+ // earlier table's frame -- otherwise that prefix strands on the ring, a later
+ // commit delivers it as a partial batch, and resetTableBuffersAfterFlush
+ // discards every source row, all while flush() reported failure to the
+ // caller. The pre-flight size pass makes the split all-or-nothing: an
+ // oversized table throws BEFORE any frame is published. Pre-fix, the "small"
+ // table's frame was published and committed on close, so the server saw it.
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(200);
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
- Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size());
- byte[] b1 = handler.batches.get(0);
- byte[] b2 = handler.batches.get(1);
-
- // The deltaStart varint sits right after the 12-byte header.
- // For self-sufficient frames it must be 0 (single byte 0x00)
- // in BOTH batches โ regardless of how many symbols the prior
- // batch already shipped.
- int deltaStart1 = readVarint(b1, DELTA_START_OFFSET);
- int deltaStart2 = readVarint(b2, DELTA_START_OFFSET);
- Assert.assertEquals("batch 1 deltaStart must be 0", 0, deltaStart1);
- Assert.assertEquals("batch 2 deltaStart must be 0 (self-sufficient)",
- 0, deltaStart2);
-
- // batch 2 must include >= 2 symbols in its delta dict (alpha
- // from the prior batch + beta from this one). The varint at
- // DELTA_START_OFFSET+1 is deltaCount.
- int deltaCount2 = readVarint(b2, DELTA_START_OFFSET + 1);
- Assert.assertTrue("batch 2 must redefine at least 2 symbols, got " + deltaCount2,
- deltaCount2 >= 2);
-
- // Sanity: batch 2 should NOT be much smaller than batch 1 โ
- // with schema-ref/delta encoding it would have been; with
- // self-sufficient frames the size is in the same ballpark.
- Assert.assertTrue("batch 2 (" + b2.length + " bytes) must not be drastically smaller than batch 1 ("
- + b1.length + ")",
- b2.length >= b1.length / 2);
+ // auto_flush_bytes=off lets "big" accumulate PAST the cap (byte-based
+ // auto-flush is otherwise clamped to 90% of the cap and would flush
+ // first); the row/interval limits are set high so nothing auto-flushes
+ // during the test. Each row stays under the per-row guard (< cap), but
+ // 12 rows make "big"'s single frame exceed the cap, which no split can
+ // shrink. "small" (added first) fits; "big" (added second) does not, so
+ // the split hits it AFTER publishing "small" pre-fix.
+ String pad = new String(new char[40]).replace('\0', 'x');
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+ + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) {
+ sender.table("small").symbol("s", "a").longColumn("v", 1L).atNow();
+ for (int i = 0; i < 12; i++) {
+ sender.table("big").stringColumn("p", pad).longColumn("v", (long) i).atNow();
+ }
+ try {
+ sender.flush();
+ Assert.fail("an oversized single-table split frame must throw");
+ } catch (LineSenderException e) {
+ Assert.assertTrue(e.getMessage(),
+ e.getMessage().contains("too large for server batch cap"));
+ }
+ // close() drains the ring: pre-fix, the stranded "small" frame
+ // would be sent (and committed) here.
+ }
+
+ // No DATA frame (tableCount > 0) may have reached the server: the
+ // oversized-table split published nothing. Pre-fix, "small" arrived.
+ long dataFrames = 0;
+ for (byte[] frame : handler.batches) {
+ if (frame.length >= 8 && (((frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8)) > 0)) {
+ dataFrames++;
+ }
+ }
+ Assert.assertEquals("an oversized-table split must publish NO data frame -- an "
+ + "earlier table's frame must not strand on the ring", 0, dataFrames);
+ }
+ });
+ }
+
+ @Test
+ public void testFileModeSplitPersistsDictBeforePublish() throws Exception {
+ // File-mode store-and-forward + a SPLIT flush: a two-table batch whose combined
+ // size exceeds the server cap splits into one frame per table
+ // (flushPendingRowsSplit). The first split frame's write-ahead persist
+ // (persistNewSymbolsBeforePublish, the appendRawEntries fast path) must record
+ // the batch's new symbols in .symbol-dict BEFORE the frames publish, so a
+ // recovered / orphan-drained slot can rebuild what the delta frames reference.
+ // The other split tests run in memory mode, so this is the only coverage of the
+ // split x persist path with a live PersistedSymbolDict.
+ Path sfDir = Files.createTempDirectory("qwp-sf-split-persist");
+ try {
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ Path dictFile = sfDir.resolve("default").resolve(".symbol-dict");
+ String pad = new String(new char[60]).replace('\0', 'x');
+ try (Sender sender = Sender.fromConfig(config)) {
+ // Two tables, two new symbols, ONE flush -> the combined message
+ // exceeds cap 150 and splits into two frames.
+ sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow();
+ sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow();
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 2, 5_000);
+
+ // Check .symbol-dict while the sender is live: a fully-drained
+ // close would unlink it. The split's first-frame write-ahead
+ // persist must have recorded BOTH new symbols.
+ Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile));
+ byte[] dict = Files.readAllBytes(dictFile);
+ Assert.assertTrue("split-flush persist must record alpha", containsUtf8(dict, "alpha"));
+ Assert.assertTrue("split-flush persist must record bravo", containsUtf8(dict, "bravo"));
+ }
+
+ Assert.assertEquals("the oversized two-table batch must split into 2 frames",
+ 2, handler.batches.size());
+ }
+ });
+ } finally {
+ rmDir(sfDir);
}
}
+ @Test
+ public void testSplitPreflightAdvancesBaselineSoLaterFramesArentSizedWithTheDelta() throws Exception {
+ // Regression for the split pre-flight baseline advance in flushPendingRowsSplit
+ // (the "Mirror advanceSentMaxSymbolId" step). Only the FIRST split frame ships
+ // the batch's symbol-dict delta; the rest ship an EMPTY delta and reference ids
+ // the first frame already registered. The pre-flight size pass must advance
+ // simBaseline after the first table so it STOPS adding combinedDeltaEntriesLen
+ // to the later frames' estimated sizes. Without that advance, a later table
+ // whose real (empty-delta) frame fits the cap is mis-estimated as still carrying
+ // the whole delta and wrongly rejected with "single table batch too large" --
+ // discarding a legitimately shippable batch (fail-closed data loss).
+ //
+ // Shape (memory mode, delta enabled): a LARGE combined delta (two ~64-char
+ // symbols) rides only the first split frame. The first table (added first, so
+ // the first split frame) has a tiny body, so delta + body1 fits the cap. The
+ // second table has a big body: body2 alone fits the cap, but delta + body2 does
+ // NOT. The real code splits and ships both frames; the un-advanced pre-flight
+ // would size the second frame WITH the delta and throw. Table order is
+ // insertion order (CharSequenceObjHashMap.keys()), so t1 is the delta frame.
+ assertMemoryLeak(() -> {
+ CapturingHandler handler = new CapturingHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ server.setAdvertisedMaxBatchSize(200);
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Two long symbols => a large combined delta section (~130 bytes) that
+ // rides ONLY the first split frame. The symbol STRINGS live in the delta,
+ // not in either table body (the body carries only the varint global id).
+ String longSymA = new String(new char[64]).replace('\0', 'a');
+ String longSymB = new String(new char[64]).replace('\0', 'b');
+ String bigPad = new String(new char[100]).replace('\0', 'x');
+ // auto_flush off so both rows batch into one flush (byte-based auto-flush
+ // is otherwise clamped under the cap and would flush before the split).
+ try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port
+ + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) {
+ // t1 (added first -> first split frame): carries the whole delta but a
+ // tiny body, so delta + body1 fits the 200-byte cap.
+ sender.table("t1").symbol("s", longSymA).longColumn("v", 1L).atNow();
+ // t2 (second split frame): empty delta but a big body. body2 alone
+ // fits the cap; delta + body2 does NOT -- the mis-size the advance
+ // prevents.
+ sender.table("t2").symbol("s", longSymB).stringColumn("p", bigPad).longColumn("v", 2L).atNow();
+ // Must NOT throw: with the baseline advanced, t2's frame is sized
+ // WITHOUT the delta and fits. A broken advance throws "too large" here.
+ sender.flush();
+ waitFor(() -> handler.batches.size() >= 2, 5_000);
+ }
+
+ Assert.assertEquals("the batch must split into 2 frames, neither spuriously rejected",
+ 2, handler.batches.size());
+ byte[] f1 = handler.batches.get(0);
+ byte[] f2 = handler.batches.get(1);
+ // First split frame ships the whole delta (both new symbols, ids 0 and 1).
+ Assert.assertEquals("first split frame deltaStart must be 0",
+ 0, readVarint(f1, DELTA_START_OFFSET));
+ Assert.assertEquals("first split frame ships both new symbols",
+ 2, readVarint(f1, DELTA_START_OFFSET + 1));
+ // Second split frame carries an EMPTY delta above the advanced baseline --
+ // the whole point: it is not re-sized (or re-sent) with the delta.
+ Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)",
+ 2, readVarint(f2, DELTA_START_OFFSET));
+ Assert.assertEquals("second split frame carries no new symbols",
+ 0, readVarint(f2, DELTA_START_OFFSET + 1));
+ }
+ });
+ }
+
private static int readVarint(byte[] buf, int offset) {
// Simple unsigned varint decode โ sufficient for small values.
int result = 0;
@@ -122,6 +464,10 @@ private static int readVarint(byte[] buf, int offset) {
throw new IllegalStateException("varint truncated");
}
+ private static void rmDir(Path dir) {
+ TestUtils.removeTmpDirRec(dir == null ? null : dir.toString());
+ }
+
private static void waitFor(BoolCondition cond, long timeoutMillis) {
long deadline = System.currentTimeMillis() + timeoutMillis;
while (System.currentTimeMillis() < deadline) {
@@ -157,6 +503,7 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat
}
}
+ // Mirrors WebSocketResponse STATUS_OK layout: status u8 | sequence u64 | table_count u16
static byte[] buildAck(long seq) {
byte[] buf = new byte[1 + 8 + 2];
ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
new file mode 100644
index 00000000..c21b7d82
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -0,0 +1,641 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.DefaultHttpClientConfiguration;
+import io.questdb.client.cutlass.http.client.WebSocketClient;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.network.PlainSocketFactory;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in
+ * {@link CursorWebSocketSendLoop#setWireBaselineWithCatchUp}.
+ *
+ * On a fresh connection the loop re-registers the whole dictionary with a
+ * catch-up frame BEFORE replaying data frames. Each catch-up frame consumes a
+ * wire sequence, so the loop anchors {@code fsnAtZero = replayStart - catchUpFrames}
+ * to keep every catch-up frame mapped to an already-acked FSN. Dropping the
+ * {@code - catchUpFrames} term is silent data loss: a server ACK for a catch-up
+ * frame then translates through {@code engine.acknowledge(fsnAtZero + wireSeq)}
+ * to an FSN at or above {@code replayStart}, trimming a not-yet-delivered data
+ * frame from the store-and-forward log.
+ *
+ * The loop is constructed but never {@link CursorWebSocketSendLoop#start started};
+ * the catch-up runs against a stub {@link WebSocketClient} that counts frames, and
+ * the OK is delivered straight into the inner {@code ResponseHandler} -- the same
+ * white-box idiom {@code CursorWebSocketSendLoopDurableAckTest} uses, because
+ * {@code setWireBaselineWithCatchUp} and the wire ports have no public entry point.
+ * {@link CursorSendEngine#ackedFsn()} is the authoritative trim watermark asserted
+ * against.
+ */
+public class CursorWebSocketSendLoopCatchUpAlignmentTest {
+
+ private String tmpDir;
+
+ @Before
+ public void setUp() {
+ tmpDir = TestUtils.createTmpDir("qdb-cursor-catchup-");
+ }
+
+ @After
+ public void tearDown() {
+ TestUtils.removeTmpDir(tmpDir);
+ }
+
+ @Test
+ public void testCatchUpFrameAckDoesNotAdvanceTrimWatermark() throws Exception {
+ // Single catch-up frame (server advertises no cap). Two frames were
+ // acked before the reconnect (ackedFsn=1), FSN 2 is unacked. The catch-up
+ // frame's OK must NOT advance the watermark past 1 -- it carries no data,
+ // only the dictionary the fresh server needs before replay.
+ TestUtils.assertMemoryLeak(() -> {
+ CatchUpCapturingClient client = new CatchUpCapturingClient(0); // 0 => no cap => one frame
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 3); // FSN 0,1,2 published
+ engine.acknowledge(1); // ackedFsn=1 => replayStart=2, FSN 2 still unacked
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ seedMirror(loop, "s0", "s1"); // sentDictCount=2 => catch-up fires
+ long replayStart = engine.ackedFsn() + 1L; // = 2
+
+ invokeSetWireBaselineWithCatchUp(loop, replayStart);
+
+ assertEquals("whole dictionary fits one frame under no cap",
+ 1, client.framesSent);
+
+ // Behavioural (the harm): the catch-up frame (wire seq 0) is
+ // OK'd by the fresh server. It carries no data, so it must
+ // resolve to an already-acked FSN and leave the trim watermark
+ // untouched -- advancing it would trim the undelivered FSN 2.
+ deliverOk(loop, 0);
+ assertEquals("catch-up frame ACK must not advance the trim watermark "
+ + "(would trim an undelivered data frame -> silent data loss)",
+ 1L, engine.ackedFsn());
+ // Mechanism: the catch-up frames are anchored below replayStart.
+ assertEquals("fsnAtZero must be anchored catchUpFrames below replayStart",
+ replayStart - client.framesSent, readLong(loop, "fsnAtZero"));
+ } finally {
+ loop.close(); // frees the seeded mirror + the stub client's buffers
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Exception {
+ // A small advertised cap splits the dictionary across several catch-up
+ // frames, so the fsnAtZero offset must subtract the full frame count. Ack
+ // the LAST catch-up wire sequence: it still maps below replayStart. With
+ // the offset dropped it would translate to replayStart+1 and over-trim.
+ TestUtils.assertMemoryLeak(() -> {
+ CatchUpCapturingClient client = new CatchUpCapturingClient(40); // budget 12 => one 11-byte symbol per frame
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 5); // FSN 0..4 published
+ engine.acknowledge(2); // ackedFsn=2 => replayStart=3, FSN 3,4 unacked
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ seedMirror(loop, "symbol0000", "symbol0001"); // 11 bytes each -> two frames
+ long replayStart = engine.ackedFsn() + 1L; // = 3
+
+ invokeSetWireBaselineWithCatchUp(loop, replayStart);
+
+ assertEquals("cap must split the two symbols across two frames",
+ 2, client.framesSent);
+
+ // ACK the highest catch-up wire sequence (the last catch-up
+ // frame). It too must map below replayStart -- with the offset
+ // dropped it translates to replayStart+1 and over-trims.
+ deliverOk(loop, client.framesSent - 1);
+ assertEquals("no catch-up frame ACK may advance the trim watermark",
+ 2L, engine.ackedFsn());
+ assertEquals("fsnAtZero must subtract the full split frame count",
+ replayStart - client.framesSent, readLong(loop, "fsnAtZero"));
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception {
+ // A transient wire failure WHILE shipping the catch-up (the fresh
+ // connection drops mid-handshake) must surface as a retriable
+ // CatchUpSendException for the reconnect loop to handle -- it must NOT
+ // call fail(). From inside the catch-up fail() re-enters connectLoop
+ // (corrupting the fsnAtZero/nextWireSeq mapping, or overflowing the stack
+ // on a flapping connection) or, with no reconnect attempt reachable,
+ // latches a terminal -- turning a transient outage into a hard failure and
+ // breaking store-and-forward. Only the oversized-entry (non-retriable)
+ // terminal was covered; this pins the retriable path.
+ TestUtils.assertMemoryLeak(() -> {
+ CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); // sendBinary throws
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 2);
+ engine.acknowledge(0); // ackedFsn=0 => a real unacked frame exists behind the catch-up
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ seedMirror(loop, "s0", "s1"); // non-empty dict => catch-up fires and hits the failing send
+ try {
+ invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+ fail("a transient catch-up send failure must raise a retriable "
+ + "CatchUpSendException, not be swallowed into fail()/a terminal");
+ } catch (InvocationTargetException e) {
+ assertEquals("transient catch-up send failure must surface as CatchUpSendException",
+ "CatchUpSendException", e.getCause().getClass().getSimpleName());
+ }
+ // Retriable, not terminal: the producer-facing error latch stays clear.
+ loop.checkError();
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testAccumulateSentDictPartialOverlapExtendsMirror() throws Exception {
+ // M3: accumulateSentDict must handle a delta that STRADDLES the mirror tip
+ // (deltaStart < sentDictCount < deltaStart+deltaCount) by copying only the
+ // new tail, not dropping the whole frame. The monotonic producer never emits
+ // a straddling delta in steady state (so the pre-fix drop-whole-frame guard
+ // passed every test), but a torn-dict replay can seed the mirror smaller than
+ // a frame's coverage. Seed the mirror with 1 symbol, feed a [0..2] delta, and
+ // assert the mirror extends to all 3 -- pre-fix it stayed at 1, leaving the
+ // reconnect catch-up incomplete and shifting server-side ids.
+ TestUtils.assertMemoryLeak(() -> {
+ CatchUpCapturingClient client = new CatchUpCapturingClient(0);
+ try (CursorSendEngine engine = newEngine()) {
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ seedMirror(loop, "aa"); // sentDictCount = 1, mirror holds "aa"
+ int[] frameLen = new int[1];
+ long frame = buildDeltaFrame(0, new String[]{"aa", "bb", "cc"}, frameLen);
+ try {
+ Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(
+ "accumulateSentDict", long.class, int.class, int.class);
+ m.setAccessible(true);
+ m.invoke(loop, frame, frameLen[0], 0);
+ } finally {
+ Unsafe.free(frame, frameLen[0], MemoryTag.NATIVE_DEFAULT);
+ }
+ assertEquals("straddling delta must extend the mirror to all 3 ids",
+ 3, readInt(loop, "sentDictCount"));
+ assertEquals("mirror must hold the two new tail symbols after the "
+ + "already-held prefix, gap-free",
+ Arrays.asList("aa", "bb", "cc"), readMirrorSymbols(loop));
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception {
+ // M3: sendDictCatchUp caps each chunk under the budget, so the single-frame
+ // catch-up path cannot overflow its int frameLen at any real cardinality. The
+ // guard must still be LOCAL -- a future caller must not be able to feed a
+ // wrapped-negative frameLen to Unsafe.malloc. An oversized symbolsLen must
+ // fail loud (CatchUpSendException) BEFORE the malloc; the guard fires before
+ // symbolsAddr is read, so a dummy address is fine.
+ TestUtils.assertMemoryLeak(() -> {
+ CatchUpCapturingClient client = new CatchUpCapturingClient(0);
+ try (CursorSendEngine engine = newEngine()) {
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(
+ "sendCatchUpChunk", int.class, int.class, long.class, int.class);
+ m.setAccessible(true);
+ // symbolsLen past the mirror ceiling: HEADER + varints + symbolsLen
+ // overflows an int, so the guard must reject it before malloc.
+ m.invoke(loop, 0, 1, 0L, Integer.MAX_VALUE - 4);
+ fail("an overflowing catch-up frame size must fail loud, not malloc negative");
+ } catch (InvocationTargetException e) {
+ assertEquals("overflow must surface as CatchUpSendException",
+ "CatchUpSendException", e.getCause().getClass().getSimpleName());
+ assertTrue("message must name the frame-size guard: " + e.getCause().getMessage(),
+ e.getCause().getMessage().contains("catch-up frame exceeds the maximum size"));
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception {
+ // M1: an entry too large for the fresh server's cap during catch-up (a
+ // heterogeneous / rolling-cap failover to a smaller-cap node) must NOT latch
+ // on first sight. sendDictCatchUp throws a RETRIABLE CatchUpSendException so
+ // the reconnect loop rides it out -- a larger-cap node may return -- and only
+ // after MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive cap gaps does it recordFatal.
+ // Pre-fix the first cap gap latched a terminal, so one transient failover to a
+ // smaller-cap node killed the sender. (A successful catch-up resets the budget;
+ // the other catch-up tests, which use a fitting cap, never trip it.)
+ TestUtils.assertMemoryLeak(() -> {
+ Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+ maxField.setAccessible(true);
+ int maxAttempts = maxField.getInt(null);
+ // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol.
+ CatchUpCapturingClient client = new CatchUpCapturingClient(160);
+ try (CursorSendEngine engine = newEngine()) {
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ seedMirror(loop, TestUtils.repeat("x", 200));
+ // Attempts 1 .. max-1 are retriable: no terminal is latched.
+ for (int i = 1; i < maxAttempts; i++) {
+ try {
+ invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+ fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+ } catch (InvocationTargetException e) {
+ assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+ assertTrue("attempt " + i + " must name the catch-up cap gap: "
+ + e.getCause().getMessage(),
+ e.getCause().getMessage().contains("during catch-up"));
+ }
+ loop.checkError(); // under budget => retriable => no terminal
+ }
+ // The exhausting attempt still throws, and now latches the terminal.
+ try {
+ invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+ fail("the exhausting cap gap must still raise CatchUpSendException");
+ } catch (InvocationTargetException e) {
+ assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+ }
+ try {
+ loop.checkError();
+ fail("exhausting the cap-gap settle budget must latch a terminal");
+ } catch (LineSenderException terminal) {
+ assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(),
+ terminal.getMessage().contains("during catch-up")
+ && terminal.getMessage().contains("must be resent"));
+ }
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception {
+ // The cap-gap settle budget (catchUpCapGapAttempts) counts CONSECUTIVE cap
+ // gaps across reconnects; a successful catch-up ends the episode and MUST reset
+ // it to 0 (sendDictCatchUp's final line). Otherwise cap gaps interspersed with
+ // successful catch-ups -- a rolling-cap cluster where a larger-cap node comes
+ // and goes -- would accumulate to a spurious terminal over a long-lived sender.
+ // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one
+ // fixed cap with no success interleaved, so it cannot pin the reset.
+ TestUtils.assertMemoryLeak(() -> {
+ Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS");
+ maxField.setAccessible(true);
+ int maxAttempts = maxField.getInt(null);
+ CatchUpCapturingClient client = new CatchUpCapturingClient(160); // too small for a 200-char symbol
+ try (CursorSendEngine engine = newEngine()) {
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ seedMirror(loop, TestUtils.repeat("x", 200));
+ // Accrue max-1 consecutive cap gaps (each retriable, no terminal).
+ for (int i = 1; i < maxAttempts; i++) {
+ try {
+ invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+ fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')');
+ } catch (InvocationTargetException e) {
+ assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+ }
+ }
+ assertEquals("precondition: budget accrued to max-1",
+ maxAttempts - 1, readInt(loop, "catchUpCapGapAttempts"));
+
+ // A larger-cap node returns: the whole dictionary re-registers with
+ // no cap gap, so the settle budget must reset to 0.
+ client.cap = 0; // no cap => the 200-char symbol fits one frame
+ invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+ assertEquals("a successful catch-up must reset the cap-gap settle budget",
+ 0, readInt(loop, "catchUpCapGapAttempts"));
+
+ // Behavioural proof the budget is genuinely fresh: max-1 more cap
+ // gaps still latch NO terminal (they would if the counter had stayed
+ // at max-1 -- one more gap would have hit the cap and killed the sender).
+ client.cap = 160;
+ for (int i = 1; i < maxAttempts; i++) {
+ try {
+ invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L);
+ fail("post-reset cap gap must be retriable (attempt " + i + ')');
+ } catch (InvocationTargetException e) {
+ assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName());
+ }
+ loop.checkError(); // fresh budget => still under max => no terminal
+ }
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testMirrorOverflowFailsLoud() throws Exception {
+ // ensureSentDictCapacity must latch a terminal -- not silently overflow the
+ // int capacity math into a heap-corrupting copyMemory -- when the sent-dict
+ // mirror would exceed MAX_SENT_DICT_BYTES. Unreachable at real cardinality
+ // (~200M+ symbols on one connection), so drive the guard directly with an
+ // oversized required, mirroring testCatchUpChunkFrameSizeOverflowFailsLoud.
+ TestUtils.assertMemoryLeak(() -> {
+ Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_SENT_DICT_BYTES");
+ maxField.setAccessible(true);
+ long overCeiling = (long) maxField.getInt(null) + 1L;
+ CatchUpCapturingClient client = new CatchUpCapturingClient(0);
+ try (CursorSendEngine engine = newEngine()) {
+ CursorWebSocketSendLoop loop = newLoop(engine, client);
+ try {
+ Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("ensureSentDictCapacity", long.class);
+ m.setAccessible(true);
+ try {
+ m.invoke(loop, overCeiling);
+ fail("a mirror capacity past MAX_SENT_DICT_BYTES must fail loud, not overflow");
+ } catch (InvocationTargetException e) {
+ assertEquals("overflow must surface as LineSenderException",
+ "LineSenderException", e.getCause().getClass().getSimpleName());
+ assertTrue("message must name the mirror ceiling: " + e.getCause().getMessage(),
+ e.getCause().getMessage().contains("mirror exceeds the maximum size"));
+ }
+ // recordFatal (not a bare throw) latched the terminal, so the loop
+ // winds down instead of reconnecting into the same overflow.
+ try {
+ loop.checkError();
+ fail("mirror overflow must latch a terminal");
+ } catch (LineSenderException terminal) {
+ assertTrue(terminal.getMessage().contains("mirror exceeds the maximum size"));
+ }
+ } finally {
+ loop.close();
+ }
+ }
+ });
+ }
+
+ private static void appendFrames(CursorSendEngine engine, int count) {
+ long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+ try {
+ byte[] payload = "frame-bytes-padd".getBytes(StandardCharsets.US_ASCII);
+ for (int i = 0; i < payload.length; i++) {
+ Unsafe.getUnsafe().putByte(buf + i, payload[i]);
+ }
+ for (int i = 0; i < count; i++) {
+ engine.appendBlocking(buf, 16);
+ }
+ } finally {
+ Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
+ // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount
+ // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict
+ // skips the header, so its content is irrelevant; the caller frees the frame.
+ private static long buildDeltaFrame(int deltaStart, String[] symbols, int[] outLen) {
+ int deltaCount = symbols.length;
+ int size = 12 + varintSize(deltaStart) + varintSize(deltaCount);
+ for (String s : symbols) {
+ size += varintSize(s.getBytes(StandardCharsets.UTF_8).length)
+ + s.getBytes(StandardCharsets.UTF_8).length;
+ }
+ long addr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+ long p = writeVarint(addr + 12, deltaStart);
+ p = writeVarint(p, deltaCount);
+ for (String s : symbols) {
+ byte[] b = s.getBytes(StandardCharsets.UTF_8);
+ p = writeVarint(p, b.length);
+ for (byte x : b) {
+ Unsafe.getUnsafe().putByte(p++, x);
+ }
+ }
+ outLen[0] = size;
+ return addr;
+ }
+
+ private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+ f.setAccessible(true);
+ return f.getInt(loop);
+ }
+
+ // Parses the loop's native sent-dictionary mirror ([len varint][utf8]...) back
+ // into the symbol strings a reconnect catch-up would re-register.
+ private static List readMirrorSymbols(CursorWebSocketSendLoop loop) throws Exception {
+ long addr = readLong(loop, "sentDictBytesAddr");
+ int len = readInt(loop, "sentDictBytesLen");
+ List out = new ArrayList<>();
+ long p = addr;
+ long limit = addr + len;
+ while (p < limit) {
+ long l = 0;
+ int shift = 0;
+ while (p < limit) {
+ byte b = Unsafe.getUnsafe().getByte(p++);
+ l |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ }
+ byte[] bytes = new byte[(int) l];
+ for (int i = 0; i < l; i++) {
+ bytes[i] = Unsafe.getUnsafe().getByte(p++);
+ }
+ out.add(new String(bytes, StandardCharsets.UTF_8));
+ }
+ return out;
+ }
+
+ // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response
+ // handler, mimicking the server acking a catch-up frame (which carries no tables).
+ private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception {
+ int size = 11; // status(1) + sequence(8) + tableCount(2)
+ long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+ try {
+ Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK);
+ Unsafe.getUnsafe().putLong(ptr + 1, wireSeq);
+ Unsafe.getUnsafe().putShort(ptr + 9, (short) 0);
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler");
+ f.setAccessible(true);
+ Object handler = f.get(loop);
+ Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class);
+ m.setAccessible(true);
+ m.invoke(handler, ptr, size);
+ } finally {
+ Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
+ private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) throws Exception {
+ Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("setWireBaselineWithCatchUp", long.class);
+ m.setAccessible(true);
+ m.invoke(loop, replayStart);
+ }
+
+ private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) {
+ return new CursorWebSocketSendLoop(
+ client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS,
+ () -> {
+ throw new UnsupportedOperationException("test loop is never started");
+ },
+ 5_000L, 100L, 5_000L, false);
+ }
+
+ private CursorSendEngine newEngine() {
+ return new CursorSendEngine(tmpDir, 16384);
+ }
+
+ private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+ f.setAccessible(true);
+ return f.getLong(loop);
+ }
+
+ // Populates the loop's native sent-dictionary mirror with {@code symbols} in
+ // the on-wire [len varint][utf8] layout, so setWireBaselineWithCatchUp sees a
+ // non-empty dictionary to re-register. loop.close() frees it.
+ private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) throws Exception {
+ int total = 0;
+ for (String s : symbols) {
+ int len = s.getBytes(StandardCharsets.UTF_8).length;
+ total += varintSize(len) + len;
+ }
+ long addr = Unsafe.malloc(total, MemoryTag.NATIVE_DEFAULT);
+ long p = addr;
+ for (String s : symbols) {
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ p = writeVarint(p, bytes.length);
+ for (byte b : bytes) {
+ Unsafe.getUnsafe().putByte(p++, b);
+ }
+ }
+ setField(loop, "sentDictBytesAddr", addr);
+ setIntField(loop, "sentDictBytesCapacity", total);
+ setIntField(loop, "sentDictBytesLen", total);
+ setIntField(loop, "sentDictCount", symbols.length);
+ }
+
+ private static void setField(Object target, String name, long value) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+ f.setAccessible(true);
+ f.setLong(target, value);
+ }
+
+ private static void setIntField(Object target, String name, int value) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+ f.setAccessible(true);
+ f.setInt(target, value);
+ }
+
+ private static int varintSize(long value) {
+ int n = 1;
+ while (value > 0x7F) {
+ value >>>= 7;
+ n++;
+ }
+ return n;
+ }
+
+ private static long writeVarint(long addr, long value) {
+ while (value > 0x7F) {
+ Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80));
+ value >>>= 7;
+ }
+ Unsafe.getUnsafe().putByte(addr++, (byte) value);
+ return addr;
+ }
+
+ // Stub transport: completes no real I/O. getServerMaxBatchSize drives the
+ // catch-up split; sendBinary counts the frames the catch-up emitted, or --
+ // when throwOnSend is set -- raises a transient wire error to model the fresh
+ // connection dropping mid-catch-up.
+ private static final class CatchUpCapturingClient extends WebSocketClient {
+ // Mutable so a test can model a rolling-cap cluster: raise it for a node that
+ // accepts the dictionary, lower it for a smaller-cap node that cap-gaps.
+ private int cap;
+ private final boolean throwOnSend;
+ private int framesSent;
+
+ CatchUpCapturingClient(int cap) {
+ this(cap, false);
+ }
+
+ CatchUpCapturingClient(int cap, boolean throwOnSend) {
+ super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE);
+ this.cap = cap;
+ this.throwOnSend = throwOnSend;
+ }
+
+ @Override
+ public int getServerMaxBatchSize() {
+ return cap;
+ }
+
+ @Override
+ public int getServerQwpVersion() {
+ return 1;
+ }
+
+ @Override
+ public void sendBinary(long dataPtr, int length) {
+ if (throwOnSend) {
+ throw new RuntimeException("transient wire failure during catch-up");
+ }
+ framesSent++;
+ }
+
+ @Override
+ protected void ioWait(int timeout, int op) {
+ }
+
+ @Override
+ protected void setupIoWait() {
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
new file mode 100644
index 00000000..5cff81ba
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
@@ -0,0 +1,208 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+/**
+ * Guards the recovery-seeded symbol-dictionary mirror against leaking when the
+ * I/O loop is constructed but never run.
+ *
+ * On recovery / orphan-drain the {@link CursorWebSocketSendLoop} constructor
+ * seeds a native mirror ({@code sentDictBytesAddr}) from the slot's persisted
+ * dictionary so the first connection can re-register it. That mirror is freed on
+ * the I/O thread's exit path -- so if the loop is closed WITHOUT ever starting
+ * (start() never called, or Thread.start() failing before the loop runs), the
+ * free never happens. {@code close()} must free it in that case.
+ */
+public class CursorWebSocketSendLoopMirrorLeakTest {
+
+ private static final int DISTINCT_SYMBOLS = 8;
+ private static final int ROWS = 40;
+
+ @Test
+ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception {
+ Path sfDir = Files.createTempDirectory("qwp-mirror-leak");
+ try {
+ // Populate a slot with delta frames + a non-empty .symbol-dict, then
+ // abandon it (silent server, close-fast) -- outside assertMemoryLeak,
+ // because a full Sender+server round trip is not net-zero on its own.
+ populateRecoverableSlot(sfDir);
+
+ Path slot = sfDir.resolve("default");
+ Assert.assertTrue("populate must leave a persisted dictionary",
+ Files.exists(slot.resolve(".symbol-dict")));
+
+ // Only the recovery construct + close is leak-checked: the engine
+ // recovers (loading the dict), the loop ctor seeds the mirror from it,
+ // and close() -- with NO start() -- must free every native allocation.
+ // Pre-fix the seeded mirror leaks here and this assertion fails.
+ assertMemoryLeak(() -> {
+ try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) {
+ PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+ Assert.assertNotNull("disk-mode engine must open a persisted dict", pd);
+ Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)",
+ pd.size() > 0 && pd.loadedEntriesLen() > 0);
+
+ CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
+ null, engine, 0, 1_000_000L,
+ () -> {
+ throw new IOException("no reconnect in this test");
+ },
+ 0, 0, 1);
+ // Close without start(): the ctor-seeded mirror is this
+ // thread's to free, since the I/O loop never ran.
+ Assert.assertTrue("precondition: the ctor seeded a non-empty mirror",
+ readInt(loop, "sentDictCount") > 0);
+ loop.close();
+ // close() must reset sentDictCount alongside freeing the buffer,
+ // so the mirror stays all-or-nothing: a hypothetical post-close
+ // start() (no closed guard) cannot read a stale count against a
+ // freed buffer and drive a null-mirror catch-up.
+ Assert.assertEquals("close() must reset sentDictCount to 0",
+ 0, readInt(loop, "sentDictCount"));
+ }
+ });
+ } finally {
+ rmDir(sfDir);
+ }
+ }
+
+ @Test
+ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
+ // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW
+ // CursorWebSocketSendLoop per wire session against the SAME, persistent
+ // engine when a durable-ack capability gap forces a mid-drain recycle. The
+ // recovery mirror seed must survive that recycle. If the first loop CONSUMED
+ // the persisted dictionary's loaded entries (a one-shot ownership transfer),
+ // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no
+ // reconnect catch-up, and the first replayed delta frame (deltaStart > 0)
+ // trips the torn-dict guard -- falsely quarantining a healthy slot with a
+ // bogus "resend required" terminal. Copying the entries (leaving the
+ // dictionary intact for the engine's lifetime) lets every recycled loop
+ // re-seed. Pre-fix, loop2's sentDictCount is 0 and this assertion fails.
+ Path sfDir = Files.createTempDirectory("qwp-mirror-reseed");
+ try {
+ populateRecoverableSlot(sfDir);
+ Path slot = sfDir.resolve("default");
+ assertMemoryLeak(() -> {
+ try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) {
+ PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+ Assert.assertNotNull(pd);
+ int dictSize = pd.size();
+ Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0);
+
+ // Session 1 seeds its mirror from the persisted dictionary.
+ CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine);
+ try {
+ Assert.assertEquals("session-1 mirror must seed from the persisted dict",
+ dictSize, readInt(loop1, "sentDictCount"));
+ } finally {
+ loop1.close();
+ }
+
+ // Session 2 against the SAME engine (the drainer recycle): the
+ // seed must NOT have been consumed -- the mirror must re-seed to
+ // the full dictionary so the reconnect catch-up is complete.
+ CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine);
+ try {
+ Assert.assertEquals("recycled session-2 mirror must re-seed from the "
+ + "persisted dict (pre-fix it was 0)",
+ dictSize, readInt(loop2, "sentDictCount"));
+ } finally {
+ loop2.close();
+ }
+ }
+ });
+ } finally {
+ rmDir(sfDir);
+ }
+ }
+
+ // Constructs a recovery send loop but does NOT start it: the ctor seeds the
+ // catch-up mirror synchronously, which is all these tests observe. The
+ // reconnect factory is never invoked.
+ private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine) {
+ return new CursorWebSocketSendLoop(
+ null, engine, 0, 1_000_000L,
+ () -> {
+ throw new IOException("no reconnect in this test");
+ },
+ 0, 0, 1);
+ }
+
+ private static void populateRecoverableSlot(Path sfDir) throws Exception {
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port
+ + ";sf_dir=" + sfDir
+ + ";sf_max_bytes=4096"
+ + ";close_flush_timeout_millis=0;";
+ try (Sender s1 = Sender.fromConfig(cfg)) {
+ for (int i = 0; i < ROWS; i++) {
+ s1.table("m")
+ .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS))
+ .longColumn("v", i)
+ .atNow();
+ s1.flush();
+ }
+ }
+ }
+ }
+
+ private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+ f.setAccessible(true);
+ return f.getInt(loop);
+ }
+
+ private static void rmDir(Path dir) throws IOException {
+ TestUtils.removeTmpDirRec(dir == null ? null : dir.toString());
+ }
+
+ private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler {
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ // never acks -- the sender leaves everything unacked in the slot
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
index ff710fe1..bf16f3b0 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
@@ -75,7 +75,9 @@
public class CursorWebSocketSendLoopOrphanTailTest {
private static final int FLAG_DEFER_COMMIT = 0x01;
+ private static final int FLAG_DELTA_SYMBOL_DICT = 0x08;
private static final int HEADER_OFFSET_FLAGS = 5;
+ private static final int HEADER_SIZE = 12;
private static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" little-endian
private String tmpDir;
@@ -221,6 +223,83 @@ public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exce
});
}
+ @Test
+ public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception {
+ // recoveredMaxSymbolId must reflect only COMMITTED (transmitted) frames, not
+ // the aborted orphan-tail frames trySendOne retires without ever sending. A
+ // host crash that tears the persisted dictionary down to the committed ids
+ // while an orphan-tail frame introduced a HIGHER id must NOT over-reject the
+ // resume: the producer never reuses an orphan id on the wire (the tail retires
+ // first), so counting it would inflate recoveredMaxSymbolId and make
+ // seedGlobalDictionaryFromPersisted fail a fully-recoverable slot. The
+ // maxSymbolDeltaEnd walk is therefore bounded to recoveredCommitBoundaryFsn.
+ TestUtils.assertMemoryLeak(() -> {
+ try (CursorSendEngine engine = newEngine()) {
+ // fsn 0: commit-bearing delta frame registering ids 0,1.
+ appendDeltaFrame(engine, false, 0, 2);
+ // fsn 1: DEFERRED delta frame registering id 2 -- the orphan tail.
+ appendDeltaFrame(engine, true, 2, 1);
+ }
+ try (CursorSendEngine engine = newEngine()) {
+ assertTrue(engine.wasRecoveredFromDisk());
+ assertEquals("last commit-bearing frame", 0L, engine.recoveredCommitBoundaryFsn());
+ assertEquals("orphan tail tip", 1L, engine.recoveredOrphanTipFsn());
+ // Only the committed frame's ids (0,1) count -> highest id 1. The
+ // orphan-tail frame's id 2 is excluded, so a resume whose recovered
+ // dictionary holds ids 0,1 (size 2) is NOT over-rejected.
+ assertEquals("orphan-tail id 2 must be excluded from recoveredMaxSymbolId",
+ 1L, engine.recoveredMaxSymbolId());
+ }
+ });
+ }
+
+ @Test
+ public void testZeroCountDeltaFrameAnchorsRecoveredMaxSymbolIdAtItsBaseline() throws Exception {
+ // A committed delta frame that introduces NO new symbol (deltaCount == 0 -- a
+ // commit frame, or one whose rows only re-use existing ids) still carries
+ // deltaStart == the producer's baseline at encode time, because beginMessage
+ // ALWAYS sets FLAG_DELTA_SYMBOL_DICT. maxSymbolDeltaEnd counts it as
+ // deltaStart + deltaCount == deltaStart (NOT 0), so recoveredMaxSymbolId ==
+ // deltaStart - 1 even though the frame introduces nothing.
+ //
+ // This is the mechanism behind the torn-dict guard's deliberate CONSERVATIVE
+ // over-strand (see seedGlobalDictionaryFromPersisted): if a host crash tears
+ // the persisted dictionary below such a frame's baseline while its
+ // symbol-introducing predecessors were already acked and trimmed, both the
+ // seed-time guard (recoveredMaxSymbolId >= pd.size()) and the drainer's replay
+ // guard (deltaStart > sentDictCount) fire and quarantine the slot -- fail-clean
+ // "resend required" -- even though the frame's rows may reference only ids the
+ // truncated dictionary still holds.
+ //
+ // Counting the zero-count frame's baseline is load-bearing SAFETY: a "fix" that
+ // skipped zero-count frames (returning 0 for them) would UNDER-strand and let a
+ // genuinely torn dictionary through, silently shifting the dense id map. This
+ // pins that a zero-count delta frame is anchored at its baseline, not skipped.
+ TestUtils.assertMemoryLeak(() -> {
+ try (CursorSendEngine engine = newEngine()) {
+ // fsn 0: commit-bearing delta frame that genuinely registers ids 0..4.
+ appendDeltaFrame(engine, false, 0, 5);
+ // fsn 1: commit-bearing delta frame with deltaStart 10, deltaCount 0 --
+ // introduces NOTHING, but its baseline (10) sits ABOVE every id any
+ // surviving frame actually introduces (max 4). Models a commit /
+ // symbol-reusing frame emitted after ids 5..9 were registered by
+ // predecessor frames that have since been acked and trimmed away.
+ appendDeltaFrame(engine, false, 10, 0);
+ }
+ try (CursorSendEngine engine = newEngine()) {
+ assertTrue(engine.wasRecoveredFromDisk());
+ assertEquals("both frames are commit-bearing", 1L, engine.recoveredCommitBoundaryFsn());
+ // The zero-count frame drives recoveredMaxSymbolId to 9 (its baseline
+ // 10, minus 1), NOT to 4 (the highest id any surviving frame actually
+ // introduces) and NOT to 0 (which skipping it would yield). This
+ // inflation is exactly what makes seedGlobalDictionaryFromPersisted
+ // over-reject a dictionary holding ids 0..4 (size 5).
+ assertEquals("a zero-count delta frame anchors recoveredMaxSymbolId at its baseline-1",
+ 9L, engine.recoveredMaxSymbolId());
+ }
+ });
+ }
+
// ---------------------------------------------------------------------
// harness
// ---------------------------------------------------------------------
@@ -291,6 +370,27 @@ private static void appendFrame(CursorSendEngine engine, boolean defer) {
}
}
+ // Appends a QWP frame carrying a symbol-dict delta section ([deltaStart varint]
+ // [deltaCount varint]) so the recovery walk's maxSymbolDeltaEnd counts it.
+ // deltaStart/deltaCount stay < 128 so each encodes in a single LEB128 byte.
+ private static void appendDeltaFrame(CursorSendEngine engine, boolean defer, int deltaStart, int deltaCount) {
+ int size = HEADER_SIZE + 2;
+ long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT);
+ try {
+ for (int i = 0; i < size; i++) {
+ Unsafe.getUnsafe().putByte(buf + i, (byte) 0);
+ }
+ Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE);
+ Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS,
+ (byte) (FLAG_DELTA_SYMBOL_DICT | (defer ? FLAG_DEFER_COMMIT : 0)));
+ Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart);
+ Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) deltaCount);
+ engine.appendBlocking(buf, size);
+ } finally {
+ Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT);
+ }
+ }
+
private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException {
long deadline = System.nanoTime() + 10_000_000_000L;
while (engine.ackedFsn() < target) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java
index 34ae5518..d63887db 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java
@@ -244,6 +244,77 @@ public void testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine() throws Exception
});
}
+ @Test
+ public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception {
+ // C2 regression: the dictionary catch-up advances nextWireSeq WITHOUT
+ // sending a data frame. A non-orderly close in that window -- a flapping
+ // LB/middlebox that completes the upgrade, accepts the catch-up, then drops
+ // before the first replay frame -- must be strike-EXEMPT. Keying the
+ // poison-strike gate off nextWireSeq > 0 (rather than
+ // dataFrameSentThisConnection) charges a strike on a frame that was never
+ // sent; MAX_REJECTIONS such closes then escalate a TRANSIENT outage to a
+ // PROTOCOL_VIOLATION terminal, hard-failing the producer and quarantining an
+ // orphan drainer -- exactly what store-and-forward's retry-forever contract
+ // forbids. Mirror image of testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine,
+ // which sends a real data frame (setSentCount) and DOES escalate.
+ TestUtils.assertMemoryLeak(() -> {
+ List clients = new ArrayList<>();
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 2);
+ CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+ for (int i = 0; i < MAX_REJECTIONS + 2; i++) {
+ // Model the catch-up: nextWireSeq advanced, but NO data frame
+ // sent. swapClient resets both on every recycle, so re-apply
+ // before each close. Pre-fix, each of these lands a strike on the
+ // never-sent head frame and the loop terminals by now.
+ setCatchUpWireSeqOnly(loop, 2);
+ deliverNonOrderlyClose(loop);
+ }
+ // No strike was ever charged, so nothing escalated: the loop stays
+ // retriable and the producer-facing error latch is clear.
+ loop.checkError();
+ } finally {
+ closeAll(clients);
+ }
+ });
+ }
+
+ @Test
+ public void testNonOrderlyRejectionAfterOnlyCatchUpDoesNotStrike() throws Exception {
+ // C2 regression: the server-NACK twin of
+ // testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike. handleServerRejection's
+ // pre-send gate keys off dataFrameSentThisConnection, NOT nextWireSeq -- the
+ // dictionary catch-up advances nextWireSeq WITHOUT sending a data frame. A
+ // server NACK of a catch-up frame (nextWireSeq>0, no data frame sent) must
+ // take the pre-send path (surface + recycle, no strike), not the post-send
+ // poison-strike path. Pre-fix (gate on highestSent >= 0, i.e. nextWireSeq>0)
+ // each catch-up NACK strikes the never-sent head frame; MAX_REJECTIONS such
+ // strikes escalate a TRANSIENT outage to a producer-fatal PROTOCOL_VIOLATION
+ // terminal -- exactly what store-and-forward's retry-forever contract forbids.
+ // WRITE_ERROR (RETRIABLE) is the discriminating category: it DOES accrue
+ // strikes on the post-send path (see testNackRecycleIsPacedAgainstHealthyServer),
+ // so a regressed gate escalates here and checkError() throws.
+ TestUtils.assertMemoryLeak(() -> {
+ List clients = new ArrayList<>();
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 2);
+ CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+ for (int i = 0; i < MAX_REJECTIONS + 2; i++) {
+ // Model the catch-up: nextWireSeq advanced, but NO data frame
+ // sent. The pre-send recycle (swapClient) resets nextWireSeq, so
+ // re-apply before each NACK (mirrors the onClose twin).
+ setCatchUpWireSeqOnly(loop, 2);
+ deliverRetriableNack(loop, 1, "disk full");
+ }
+ // No strike was ever charged, so nothing escalated: the loop stays
+ // retriable and the producer-facing error latch is clear.
+ loop.checkError();
+ } finally {
+ closeAll(clients);
+ }
+ });
+ }
+
@Test
public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception {
// A reachable, healthy server that NACKs the head frame (RETRIABLE)
@@ -585,6 +656,88 @@ public void testPoisonDwellHoldsEscalationUntilWallClockWindowElapses() throws E
});
}
+ @Test
+ public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Exception {
+ // Regression: after a reconnect the loop ships the head DATA frame
+ // (dataFrameSentThisConnection=true) BEFORE tryReceiveAcks reads the server's
+ // NACK of a dictionary CATCH-UP frame in the same loop iteration. That NACK
+ // names a wire seq below the replay head, so its fsn = fsnAtZero+cappedSeq is
+ // at or below ackedFsn -- negative when replayStart < catchUpFrames. It must
+ // NOT charge a poison strike: recordHeadRejectionStrike(fsn) would set
+ // poisonFsn to that value (the common shape yields -1, the "no suspect"
+ // sentinel), laundering any genuine in-progress poison run and reporting a
+ // bogus fsn. The fix routes an fsn <= ackedFsn rejection to the pre-send path
+ // (surface + recycle, no strike), symmetric with the success path's
+ // engine.acknowledge() no-op below ackedFsn.
+ TestUtils.assertMemoryLeak(() -> {
+ List clients = new ArrayList<>();
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 2);
+ CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+ // Model: 2 catch-up frames (wire seq 0,1) + 1 data frame (wire seq 2)
+ // sent, nothing acked (ackedFsn = -1), so fsnAtZero = replayStart(0) -
+ // catchUpFrames(2) = -2. A NACK of catch-up wire seq 0 maps to fsn -2,
+ // strictly below the -1 sentinel so the assertion discriminates.
+ setPostCatchUpDataFrameBaseline(loop, -2L, 3L);
+ assertEquals("precondition: no poison suspect yet",
+ -1L, getLongField(loop, "poisonFsn"));
+
+ deliverRetriableNack(loop, 0, "disk full");
+
+ // The catch-up NACK was surfaced + recycled but charged NO strike, so
+ // the sentinel is intact. Pre-fix it was set to -2, laundering the
+ // poison detector's state.
+ assertEquals("catch-up NACK must not set/launder the poison sentinel",
+ -1L, getLongField(loop, "poisonFsn"));
+ loop.checkError();
+ } finally {
+ closeAll(clients);
+ }
+ });
+ }
+
+ @Test
+ public void testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike() throws Exception {
+ // Boundary companion to testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState:
+ // the COMMON single-frame catch-up maps its only catch-up frame to
+ // fsn == ackedFsn EXACTLY (the last catch-up frame lands on replayStart-1 ==
+ // ackedFsn). The pre-send routing guard is "fsn <= ackedFsn", so this equality
+ // case must also skip the poison strike. Without pinning it a "<=" -> "<"
+ // mutation would slip the single/last catch-up frame's NACK onto the post-send
+ // poison-strike path and launder the detector -- exactly the hazard the guard
+ // closes -- yet the strictly-below test above would still pass.
+ TestUtils.assertMemoryLeak(() -> {
+ List clients = new ArrayList<>();
+ try (CursorSendEngine engine = newEngine()) {
+ appendFrames(engine, 2);
+ CursorWebSocketSendLoop loop = newDurableLoop(engine, clients);
+ // 1 catch-up frame (wire seq 0) + 1 data frame (wire seq 1), nothing
+ // acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - catchUpFrames(1)
+ // = -1. A NACK of catch-up wire seq 0 maps to fsn -1 == ackedFsn exactly.
+ setPostCatchUpDataFrameBaseline(loop, -1L, 2L);
+ assertEquals("precondition: no strikes yet",
+ 0L, getLongField(loop, "poisonStrikes"));
+
+ deliverRetriableNack(loop, 0, "disk full");
+
+ // poisonStrikes -- NOT poisonFsn -- is the discriminator at this exact
+ // boundary: a wrongly-charged strike calls recordHeadRejectionStrike(-1),
+ // which sets poisonFsn to the rejected fsn -1 -- the "no suspect"
+ // sentinel -- so a poisonFsn check cannot see the laundering here. It
+ // still bumps poisonStrikes 0 -> 1, so the correct pre-send routing (no
+ // strike) shows as poisonStrikes staying 0. A "<=" -> "<" mutation of the
+ // fsn-vs-ackedFsn guard charges the strike and fails this.
+ assertEquals("catch-up NACK at fsn == ackedFsn must not charge a poison strike",
+ 0L, getLongField(loop, "poisonStrikes"));
+ assertEquals("... and must leave the poison sentinel intact",
+ -1L, getLongField(loop, "poisonFsn"));
+ loop.checkError();
+ } finally {
+ closeAll(clients);
+ }
+ });
+ }
+
@Test
public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception {
// RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame
@@ -969,6 +1122,44 @@ private static void setSentCount(CursorWebSocketSendLoop loop, long count) throw
Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq");
f.setAccessible(true);
f.setLong(loop, count);
+ // A non-zero sent count models DATA frames sent on this connection. The
+ // poison-strike / pre-send gates key off dataFrameSentThisConnection, not
+ // nextWireSeq (the dictionary catch-up advances nextWireSeq without sending
+ // a data frame), so keep the two in sync for these white-box scenarios.
+ Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection");
+ d.setAccessible(true);
+ d.setBoolean(loop, count > 0);
+ }
+
+ // Sets ONLY nextWireSeq -- deliberately NOT dataFrameSentThisConnection -- to
+ // model the dictionary catch-up having advanced the wire sequence with no data
+ // frame sent yet. Contrast setSentCount, which sets both.
+ private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long count) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq");
+ f.setAccessible(true);
+ f.setLong(loop, count);
+ }
+
+ // Models a fresh connection that has shipped the dictionary catch-up (advancing
+ // fsnAtZero below the replay head) AND the head data frame: sets fsnAtZero,
+ // nextWireSeq, and dataFrameSentThisConnection=true together.
+ private static void setPostCatchUpDataFrameBaseline(CursorWebSocketSendLoop loop,
+ long fsnAtZero, long nextWireSeq) throws Exception {
+ Field z = CursorWebSocketSendLoop.class.getDeclaredField("fsnAtZero");
+ z.setAccessible(true);
+ z.setLong(loop, fsnAtZero);
+ Field w = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq");
+ w.setAccessible(true);
+ w.setLong(loop, nextWireSeq);
+ Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection");
+ d.setAccessible(true);
+ d.setBoolean(loop, true);
+ }
+
+ private static long getLongField(CursorWebSocketSendLoop loop, String name) throws Exception {
+ Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+ f.setAccessible(true);
+ return f.getLong(loop);
}
private static long[] txns(long... v) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java
index 5c7607a6..2efee2e4 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java
@@ -25,6 +25,7 @@
package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
import io.questdb.client.std.Files;
import io.questdb.client.test.tools.TestUtils;
import org.junit.After;
@@ -35,6 +36,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
/**
* Regression test for M6 โ drainer adopting an empty orphan slot would
@@ -87,6 +89,48 @@ public void tearDown() {
Files.remove(sfDir);
}
+ @Test
+ public void testFreshStartDiscardsSurvivingStaleDictionary() throws Exception {
+ // Regression: a prior fully-drained lifecycle can leave a stale
+ // .symbol-dict behind (a best-effort delete that failed, or a crash in the
+ // close window) with NO segments. A fresh start must DISCARD it -- the
+ // dictionary is load-bearing and the fresh-start producer is not seeded
+ // from it, so trusting a survivor would diverge the producer ids from the
+ // dictionary the send loop replays and misattribute symbols on reconnect.
+ TestUtils.assertMemoryLeak(() -> {
+ // Pre-seed a stale dictionary in the slot, with no segments behind it.
+ PersistedSymbolDict stale = PersistedSymbolDict.open(sfDir);
+ assertNotNull(stale);
+ try {
+ stale.appendSymbol("staleX");
+ stale.appendSymbol("staleY");
+ assertEquals(2, stale.size());
+ } finally {
+ stale.close();
+ }
+
+ // A fresh start (no recovered segments) must open a CLEAN, empty
+ // dictionary -- not inherit the survivor.
+ try (CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024)) {
+ assertFalse("fresh start must not report a disk recovery",
+ engine.wasRecoveredFromDisk());
+ PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+ assertNotNull(pd);
+ assertEquals("fresh start must discard the surviving stale dictionary",
+ 0, pd.size());
+ }
+
+ // The survivor's bytes are physically gone, not just hidden.
+ PersistedSymbolDict reopened = PersistedSymbolDict.open(sfDir);
+ assertNotNull(reopened);
+ try {
+ assertEquals(0, reopened.size());
+ } finally {
+ reopened.close();
+ }
+ });
+ }
+
@Test
public void testNeverPublishedCloseLeavesNoSfaFiles() throws Exception {
TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
new file mode 100644
index 00000000..eb79e016
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
@@ -0,0 +1,779 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak;
+
+public class PersistedSymbolDictTest {
+
+ @Test
+ public void testAppendPersistsAcrossReopen() throws Exception {
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(d);
+ try {
+ Assert.assertEquals(0, d.size());
+ d.appendSymbol("AAPL");
+ d.appendSymbol("GOOG");
+ d.appendSymbol("MSFT");
+ Assert.assertEquals(3, d.size());
+ } finally {
+ d.close();
+ }
+
+ // Reopen: entries recovered in id order.
+ PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(reopened);
+ try {
+ Assert.assertEquals(3, reopened.size());
+ ObjList symbols = reopened.readLoadedSymbols();
+ Assert.assertEquals(3, symbols.size());
+ Assert.assertEquals("AAPL", symbols.getQuick(0));
+ Assert.assertEquals("GOOG", symbols.getQuick(1));
+ Assert.assertEquals("MSFT", symbols.getQuick(2));
+ Assert.assertTrue(reopened.loadedEntriesLen() > 0);
+
+ // Appending after recovery continues from the recovered tip.
+ reopened.appendSymbol("TSLA");
+ Assert.assertEquals(4, reopened.size());
+ } finally {
+ reopened.close();
+ }
+
+ PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString());
+ try {
+ Assert.assertEquals(4, third.size());
+ Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3));
+ } finally {
+ third.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception {
+ // M1: the producer persists the frame's already-encoded delta bytes via
+ // appendRawEntries instead of re-encoding the symbols. Those bytes are the
+ // same [len][utf8]... layout appendSymbols writes, so both must produce an
+ // identical, recoverable dictionary. Encode a range with appendSymbols,
+ // reopen to grab its on-disk entry bytes, replay them through
+ // appendRawEntries into a fresh dict, and assert the recovered symbols
+ // match -- including an empty entry mid-range.
+ assertMemoryLeak(() -> {
+ Path src = Files.createTempDirectory("qwp-symdict-src");
+ Path dst = Files.createTempDirectory("qwp-symdict-dst");
+ try {
+ GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+ dict.getOrAddSymbol("AAPL"); // id 0
+ dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range
+ dict.getOrAddSymbol("MSFT"); // id 2
+
+ PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString());
+ encoded.appendSymbols(dict, 0, 2);
+ encoded.close();
+
+ // Reopen to obtain the on-disk entry region [len][utf8]... verbatim,
+ // then replay it byte-for-byte into a fresh dict via appendRawEntries.
+ PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString());
+ try {
+ PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString());
+ try {
+ raw.appendRawEntries(reopened.loadedEntriesAddr(),
+ reopened.loadedEntriesLen(), reopened.size());
+ Assert.assertEquals(3, raw.size());
+ } finally {
+ raw.close();
+ }
+ } finally {
+ reopened.close();
+ }
+
+ // The raw-appended dict must recover the same dense symbols.
+ PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString());
+ try {
+ Assert.assertEquals(3, recovered.size());
+ ObjList symbols = recovered.readLoadedSymbols();
+ Assert.assertEquals("AAPL", symbols.getQuick(0));
+ Assert.assertEquals("", symbols.getQuick(1));
+ Assert.assertEquals("MSFT", symbols.getQuick(2));
+ } finally {
+ recovered.close();
+ }
+ } finally {
+ rmDir(src);
+ rmDir(dst);
+ }
+ });
+ }
+
+ @Test
+ public void testAppendSymbolsBatchWritesDenseRange() throws Exception {
+ // appendSymbols persists a whole id range in one write (the hot-path
+ // syscall reduction). It must produce the same dense, id-ordered file a
+ // per-symbol loop would, including an empty symbol mid-range, and a second
+ // batched call keyed off size() must continue densely (the resume-from-
+ // durable-size contract the producer relies on).
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ GlobalSymbolDictionary dict = new GlobalSymbolDictionary();
+ dict.getOrAddSymbol("AAPL"); // id 0
+ dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range
+ dict.getOrAddSymbol("MSFT"); // id 2
+ dict.getOrAddSymbol("TSLA"); // id 3
+
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ try {
+ d.appendSymbols(dict, 0, 3); // one write for all four ids
+ Assert.assertEquals(4, d.size());
+ d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op
+ Assert.assertEquals(4, d.size());
+ } finally {
+ d.close();
+ }
+
+ PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString());
+ try {
+ Assert.assertEquals(4, reopened.size());
+ ObjList symbols = reopened.readLoadedSymbols();
+ Assert.assertEquals(4, symbols.size());
+ Assert.assertEquals("AAPL", symbols.getQuick(0));
+ Assert.assertEquals("", symbols.getQuick(1));
+ Assert.assertEquals("MSFT", symbols.getQuick(2));
+ Assert.assertEquals("TSLA", symbols.getQuick(3));
+
+ // A follow-on batch keyed off the recovered size continues
+ // the dense sequence without a gap or duplicate.
+ dict.getOrAddSymbol("NVDA"); // id 4
+ reopened.appendSymbols(dict, reopened.size(), 4);
+ Assert.assertEquals(5, reopened.size());
+ } finally {
+ reopened.close();
+ }
+
+ PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString());
+ try {
+ Assert.assertEquals(5, third.size());
+ Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4));
+ } finally {
+ third.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testBadMagicIsRecreatedEmpty() throws Exception {
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ // A file with the right size but garbage content (bad magic).
+ Path f = dir.resolve(".symbol-dict");
+ Files.write(f, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(d);
+ try {
+ Assert.assertEquals("bad-magic file recreated empty", 0, d.size());
+ d.appendSymbol("X");
+ Assert.assertEquals(1, d.size());
+ } finally {
+ d.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testBadVersionIsRecreatedEmpty() throws Exception {
+ // A file with correct 'SYD1' magic and a VALID entry but an unknown version
+ // byte must be recreated empty -- its entries belong to a foreign/future
+ // format and must not be parsed as v2. Covers the version sub-condition
+ // specifically (testBadMagicIsRecreatedEmpty covers the magic one). Because
+ // the seeded "a" is a valid v2 entry, without the version check it would parse
+ // back (size 1), so this fails.
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ Path f = dir.resolve(".symbol-dict");
+ // Build a real v2 dictionary with one valid entry...
+ PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(seed);
+ seed.appendSymbol("a");
+ seed.close();
+ // ...then corrupt ONLY the version byte (offset 4) to an unknown value.
+ byte[] bytes = Files.readAllBytes(f);
+ bytes[4] = (byte) 99;
+ Files.write(f, bytes);
+
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(d);
+ try {
+ Assert.assertEquals("bad-version file recreated empty (entries must not parse)", 0, d.size());
+ d.appendSymbol("X");
+ Assert.assertEquals(1, d.size());
+ } finally {
+ d.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testCloseNullsLoadedEntries() throws Exception {
+ // close() must null loadedEntriesAddr/Len after freeing them (like
+ // scratchAddr), so an accidental post-close read of the getters cannot
+ // dereference freed native memory. Pre-fix the pointer survived close()
+ // non-zero.
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ d.appendSymbol("AAPL");
+ d.close();
+
+ // Reopen so recovery loads the entries into native memory.
+ PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString());
+ Assert.assertTrue("recovery must load entries into native memory",
+ re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0);
+ re.close();
+ Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr());
+ Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen());
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testEmptySymbolRoundTrips() throws Exception {
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ try {
+ d.appendSymbol("");
+ d.appendSymbol("nonempty");
+ } finally {
+ d.close();
+ }
+ PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString());
+ try {
+ Assert.assertEquals(2, re.size());
+ ObjList s = re.readLoadedSymbols();
+ Assert.assertEquals("", s.getQuick(0));
+ Assert.assertEquals("nonempty", s.getQuick(1));
+ } finally {
+ re.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception {
+ // A host-crash interior tear (a lost page reading back as zeroes) or a
+ // stale entry left past the end by a failed truncate can change the bytes
+ // of a NON-trailing entry. Without the per-entry CRC the parse would
+ // accept those bytes, shifting the dense id->symbol map and silently
+ // misattributing symbol-column values on replay. With the CRC the corrupt
+ // entry fails verification and the parse stops there, so recovery trusts
+ // only the intact prefix (fail-clean: the send loop's torn-dict guard then
+ // forces a resend of the rest).
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ try {
+ d.appendSymbol("s0");
+ d.appendSymbol("s1");
+ d.appendSymbol("s2");
+ d.appendSymbol("s3");
+ d.appendSymbol("s4");
+ Assert.assertEquals(5, d.size());
+ } finally {
+ d.close();
+ }
+
+ // Corrupt one byte inside the 3rd entry's UTF-8 (id 2). On-disk
+ // entry layout is [len varint][utf8][crc32c u32]; a 2-byte ASCII
+ // symbol is 1 + 2 + 4 = 7 bytes, after the 8-byte header:
+ // header[0,8) e0[8,15) e1[15,22) e2[22,29) ...
+ // Offset 23 is "s2"'s first UTF-8 byte; flipping it leaves e2's
+ // stored CRC stale.
+ Path f = dir.resolve(".symbol-dict");
+ byte[] bytes = Files.readAllBytes(f);
+ bytes[23] ^= 0x7F;
+ Files.write(f, bytes);
+
+ PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(re);
+ try {
+ // Only the intact prefix [s0, s1] is trusted; the corrupt e2
+ // and everything after it are dropped. No recovered symbol is
+ // the corrupted string -- the tear is DETECTED, never silently
+ // misattributed.
+ Assert.assertEquals("parse must stop at the corrupt interior entry", 2, re.size());
+ ObjList s = re.readLoadedSymbols();
+ Assert.assertEquals("s0", s.getQuick(0));
+ Assert.assertEquals("s1", s.getQuick(1));
+ } finally {
+ re.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testLargeSymbolRoundTripsAcrossReopen() throws Exception {
+ // C1 regression: the write path caps nothing, so a symbol larger than the
+ // old fixed 1 MB read ceiling must still recover intact. Before the fix,
+ // appendSymbol wrote the oversized entry but openExisting rejected it as
+ // "oversized", truncated the dictionary at that id (dropping it and every
+ // higher id), and a normal process-crash recovery then hard-failed with a
+ // spurious "host crash / resend required" terminal -- defeating store-and-
+ // forward's process-crash durability for large symbols. The file length is
+ // now the only bound, so the write and read paths agree.
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ // Just over the old 1 << 20 (1 MB) ceiling.
+ String big = TestUtils.repeat("x", (1 << 20) + 17);
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ try {
+ d.appendSymbol("before");
+ d.appendSymbol(big);
+ d.appendSymbol("after");
+ Assert.assertEquals(3, d.size());
+ } finally {
+ d.close();
+ }
+
+ // Recovery must load ALL three; pre-fix the reopen truncated at the
+ // big entry and came back with size 1 (only "before" survived).
+ PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString());
+ try {
+ Assert.assertEquals("large entry must survive recovery, not be truncated",
+ 3, re.size());
+ ObjList s = re.readLoadedSymbols();
+ Assert.assertEquals("before", s.getQuick(0));
+ Assert.assertEquals(big, s.getQuick(1));
+ Assert.assertEquals("after", s.getQuick(2));
+ } finally {
+ re.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testOpenCleanDiscardsSurvivingDictionary() throws Exception {
+ // A fresh start must NOT inherit a dictionary left by a prior lifecycle:
+ // openClean() truncates any survivor to empty, where open() would recover
+ // (and TRUST) it. Trusting a survivor whose segments are gone -- the
+ // fresh-start producer is not seeded from it -- shifts the dense id->symbol
+ // mapping and misattributes symbols on the next reconnect.
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict-clean");
+ try {
+ PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(stale);
+ try {
+ stale.appendSymbol("staleX");
+ stale.appendSymbol("staleY");
+ Assert.assertEquals(2, stale.size());
+ } finally {
+ stale.close();
+ }
+
+ // Fresh start: openClean yields an EMPTY dictionary regardless of
+ // the survivor, and appends from id 0 again.
+ PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString());
+ Assert.assertNotNull(fresh);
+ try {
+ Assert.assertEquals(0, fresh.size());
+ Assert.assertEquals(0, fresh.readLoadedSymbols().size());
+ fresh.appendSymbol("freshA");
+ Assert.assertEquals(1, fresh.size());
+ } finally {
+ fresh.close();
+ }
+
+ // The survivor's bytes are physically gone, not just hidden: a
+ // subsequent recovery open() sees only the post-clean content.
+ PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(reopened);
+ try {
+ Assert.assertEquals(1, reopened.size());
+ Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0));
+ } finally {
+ reopened.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testRemoveOrphanDeletesFile() throws Exception {
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ d.appendSymbol("A");
+ d.close();
+ Path f = dir.resolve(".symbol-dict");
+ Assert.assertTrue(Files.exists(f));
+ PersistedSymbolDict.removeOrphan(dir.toString());
+ Assert.assertFalse(Files.exists(f));
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testTooLargeToReopenRecreatesEmpty() throws Exception {
+ // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be read
+ // into one int-sized buffer; open() must recreate it empty (fail-clean, like
+ // every other unreadable-file case) rather than truncate the multi-GB file
+ // via a negative/zero (int) length cast. A fault facade reports the huge
+ // length without needing a real 2GB file on disk.
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ // Seed a small real file so exists() is true and the facade's huge
+ // length() drives the > Integer.MAX_VALUE reopen guard.
+ PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString());
+ Assert.assertNotNull(seed);
+ seed.appendSymbol("a");
+ seed.close();
+
+ PersistedSymbolDict d = PersistedSymbolDict.open(new HugeLengthFacade(), dir.toString());
+ Assert.assertNotNull(d);
+ try {
+ Assert.assertEquals("a >2GB dictionary must recreate empty", 0, d.size());
+ d.appendSymbol("X");
+ Assert.assertEquals(1, d.size());
+ } finally {
+ d.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testTornTrailingEntrySelfHeals() throws Exception {
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ // Write two complete entries, then a torn trailing record: a
+ // length prefix of 5 followed by only 2 bytes (crash mid-append).
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ d.appendSymbol("one");
+ d.appendSymbol("two");
+ d.close();
+
+ Path f = dir.resolve(".symbol-dict");
+ long cleanLen = Files.size(f); // header + "one" + "two", no tail
+ Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'},
+ StandardOpenOption.APPEND);
+ Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f));
+
+ // Reopen: the torn tail is ignored; only the two complete entries load.
+ PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString());
+ try {
+ // open() physically truncates the torn tail: the file returns
+ // to its clean length, so a later SHORTER append can never
+ // leave residue past its end that a future recovery mis-parses
+ // as a ghost symbol.
+ Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f));
+ Assert.assertEquals(2, re.size());
+ ObjList s = re.readLoadedSymbols();
+ Assert.assertEquals("one", s.getQuick(0));
+ Assert.assertEquals("two", s.getQuick(1));
+ // The next append continues from the truncated tail cleanly.
+ re.appendSymbol("three");
+ Assert.assertEquals(3, re.size());
+ } finally {
+ re.close();
+ }
+
+ PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString());
+ try {
+ Assert.assertEquals(3, re2.size());
+ Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2));
+ } finally {
+ re2.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ @Test
+ public void testTruncateFailureRecreatesEmpty() throws Exception {
+ // A host crash can leave a torn/stale tail past the last complete entry.
+ // open() drops it with a truncate; if that truncate FAILS (a read-only
+ // remount, a Windows share lock), the file still exposes the stale bytes,
+ // whose self-consistent per-entry CRC a later shifted parse could accept as
+ // a real symbol. So a failed truncate must make the file UNTRUSTED --
+ // open() recreates it empty (fail-clean) rather than returning a dict laid
+ // over stale bytes. Drive the truncate failure with a facade and assert the
+ // reopened dictionary is empty, not the [one, two] prefix.
+ assertMemoryLeak(() -> {
+ Path dir = Files.createTempDirectory("qwp-symdict");
+ try {
+ PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString());
+ d.appendSymbol("one");
+ d.appendSymbol("two");
+ d.close();
+
+ // Append a torn trailing record (length prefix 5, only 2 bytes) so
+ // the reopen parses [one, two], then finds validLen < len and tries
+ // to truncate the tail -- the branch under test.
+ Path f = dir.resolve(".symbol-dict");
+ long cleanLen = Files.size(f); // header + "one" + "two", no tail
+ Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND);
+ Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f));
+
+ // Reopen through a facade whose truncate() fails.
+ PersistedSymbolDict re = PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString());
+ Assert.assertNotNull(re);
+ try {
+ // The failed truncate made the file untrusted, so open() recreated
+ // it empty rather than trusting the [one, two] prefix over a stale
+ // tail: size()==0, not 2, and no recovered symbols.
+ Assert.assertEquals("failed truncate must recreate the dictionary empty", 0, re.size());
+ Assert.assertEquals(0, re.readLoadedSymbols().size());
+ // openFresh rewrote a bare 8-byte header (magic + version), so both
+ // the two entries and the torn tail are gone.
+ Assert.assertEquals("recreated file is a bare header", 8L, Files.size(f));
+ } finally {
+ re.close();
+ }
+ } finally {
+ rmDir(dir);
+ }
+ });
+ }
+
+ private static void rmDir(Path dir) {
+ TestUtils.removeTmpDirRec(dir == null ? null : dir.toString());
+ }
+
+ /**
+ * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE}.
+ * Subclasses inject a single fault; every other call hits the real filesystem.
+ */
+ private static class DelegatingFilesFacade implements FilesFacade {
+ @Override
+ public long allocNativePath(String path) {
+ return INSTANCE.allocNativePath(path);
+ }
+
+ @Override
+ public boolean allocate(int fd, long size) {
+ return INSTANCE.allocate(fd, size);
+ }
+
+ @Override
+ public int close(int fd) {
+ return INSTANCE.close(fd);
+ }
+
+ @Override
+ public boolean exists(String path) {
+ return INSTANCE.exists(path);
+ }
+
+ @Override
+ public void findClose(long findPtr) {
+ INSTANCE.findClose(findPtr);
+ }
+
+ @Override
+ public long findFirst(String dir) {
+ return INSTANCE.findFirst(dir);
+ }
+
+ @Override
+ public long findName(long findPtr) {
+ return INSTANCE.findName(findPtr);
+ }
+
+ @Override
+ public int findNext(long findPtr) {
+ return INSTANCE.findNext(findPtr);
+ }
+
+ @Override
+ public int findType(long findPtr) {
+ return INSTANCE.findType(findPtr);
+ }
+
+ @Override
+ public void freeNativePath(long pathPtr) {
+ INSTANCE.freeNativePath(pathPtr);
+ }
+
+ @Override
+ public int fsync(int fd) {
+ return INSTANCE.fsync(fd);
+ }
+
+ @Override
+ public long length(int fd) {
+ return INSTANCE.length(fd);
+ }
+
+ @Override
+ public long length(String path) {
+ return INSTANCE.length(path);
+ }
+
+ @Override
+ public long length(long pathPtr) {
+ return INSTANCE.length(pathPtr);
+ }
+
+ @Override
+ public int lock(int fd) {
+ return INSTANCE.lock(fd);
+ }
+
+ @Override
+ public int mkdir(String path, int mode) {
+ return INSTANCE.mkdir(path, mode);
+ }
+
+ @Override
+ public int openCleanRW(String path) {
+ return INSTANCE.openCleanRW(path);
+ }
+
+ @Override
+ public int openCleanRW(long pathPtr) {
+ return INSTANCE.openCleanRW(pathPtr);
+ }
+
+ @Override
+ public int openRW(String path) {
+ return INSTANCE.openRW(path);
+ }
+
+ @Override
+ public int openRW(long pathPtr) {
+ return INSTANCE.openRW(pathPtr);
+ }
+
+ @Override
+ public long read(int fd, long addr, long len, long offset) {
+ return INSTANCE.read(fd, addr, len, offset);
+ }
+
+ @Override
+ public boolean remove(String path) {
+ return INSTANCE.remove(path);
+ }
+
+ @Override
+ public boolean remove(long pathPtr) {
+ return INSTANCE.remove(pathPtr);
+ }
+
+ @Override
+ public int rename(String oldPath, String newPath) {
+ return INSTANCE.rename(oldPath, newPath);
+ }
+
+ @Override
+ public boolean truncate(int fd, long size) {
+ return INSTANCE.truncate(fd, size);
+ }
+
+ @Override
+ public long write(int fd, long addr, long len, long offset) {
+ return INSTANCE.write(fd, addr, len, offset);
+ }
+ }
+
+ /**
+ * Fails every {@link #truncate(int, long)} -- reproducing a host where the
+ * torn/stale-tail truncate cannot succeed (read-only remount, Windows share
+ * lock) so {@code open()}'s fail-clean recreate path runs.
+ */
+ private static final class FailingTruncateFacade extends DelegatingFilesFacade {
+ @Override
+ public boolean truncate(int fd, long size) {
+ return false;
+ }
+ }
+
+ /**
+ * Reports a dictionary length past {@link Integer#MAX_VALUE} -- reproducing a
+ * dictionary that legitimately grew beyond 2GB, which {@code open()} cannot read
+ * into one int-sized buffer and must recreate empty.
+ */
+ private static final class HugeLengthFacade extends DelegatingFilesFacade {
+ @Override
+ public long length(String path) {
+ return (long) Integer.MAX_VALUE + 1L;
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
index 6d4c5ef0..5e009d67 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java
@@ -108,6 +108,11 @@ public class TestWebSocketServer implements Closeable {
// 401, 403, 404, 426, 503, etc. that the failover loop should
// classify per failover.md ยง6.
private volatile String rejectingStatusReason;
+ // When > 0, 101 upgrade responses advertise this value as the
+ // X-QWP-Max-Batch-Size header, capping the QWP message size the client
+ // builds. Lets a test force the delta-dictionary catch-up to split across
+ // several frames. Live-updatable via setAdvertisedMaxBatchSize().
+ private volatile int advertisedMaxBatchSize;
public TestWebSocketServer(WebSocketServerHandler handler) throws IOException {
this(handler, false);
@@ -221,6 +226,14 @@ public int liveConnectionCount() {
return liveConnections.get();
}
+ /**
+ * Advertises {@code X-QWP-Max-Batch-Size: } on subsequent
+ * handshakes (live update). Pass {@code 0} to stop advertising a cap.
+ */
+ public void setAdvertisedMaxBatchSize(int maxBatchSize) {
+ this.advertisedMaxBatchSize = maxBatchSize;
+ }
+
/**
* Replaces the advertised role for subsequent handshakes (live update).
*/
@@ -603,6 +616,10 @@ private boolean performHandshake() throws IOException {
if (role != null) {
sb.append("X-QuestDB-Role: ").append(role).append("\r\n");
}
+ int maxBatch = advertisedMaxBatchSize;
+ if (maxBatch > 0) {
+ sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n");
+ }
sb.append("\r\n");
out.write(sb.toString().getBytes(StandardCharsets.US_ASCII));
out.flush();
diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
index 7572880b..39755426 100644
--- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
+++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
@@ -215,6 +215,36 @@ public static void removeTmpDir(String tmpDir) {
Files.remove(tmpDir);
}
+ /**
+ * Recursive counterpart to {@link #removeTmpDir(String)} for tests whose temp
+ * directory has subdirectories -- e.g. the store-and-forward slot layout
+ * {@code /default/...}, which the flat variant cannot clean up. A
+ * {@code null} argument is a no-op, so it is safe from {@code tearDown} before
+ * {@code setUp} ran. Uses {@code java.nio.file} (fully qualified to avoid the
+ * {@code io.questdb.client.std.Files} import clash) so subdirectories delete
+ * bottom-up.
+ */
+ public static void removeTmpDirRec(String tmpDir) {
+ if (tmpDir == null) {
+ return;
+ }
+ java.nio.file.Path root = java.nio.file.Paths.get(tmpDir);
+ if (!java.nio.file.Files.exists(root)) {
+ return;
+ }
+ // try-with-resources: the walk Stream holds an open directory handle that
+ // must be closed, or each call leaks a descriptor.
+ try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) {
+ walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> {
+ try {
+ java.nio.file.Files.deleteIfExists(p);
+ } catch (java.io.IOException ignored) {
+ }
+ });
+ } catch (java.io.IOException ignored) {
+ }
+ }
+
/**
* Java 8 stand-in for {@code String.repeat(int)} (added in Java 11).
*/