diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 75f228fa..01e18a6b 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -65,6 +65,13 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 return (jint) fd; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_CREAT | O_EXCL | O_RDWR, 0644), fd); + return (jint) fd; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { int fd; @@ -124,6 +131,21 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_RDONLY), fd); + if (fd < 0) { + return -1; + } + int res; + RESTARTABLE(fsync(fd), res); + int saved_errno = errno; + close(fd); + errno = saved_errno; + return res; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { int res; @@ -236,6 +258,18 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return flock((int) fd, LOCK_EX | LOCK_NB); } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + if (flock((int) fd, LOCK_UN) != 0) { + return -1; + } + /* Unlock success confirms that the slot is reusable. close() is one-shot: + * POSIX leaves descriptor state unspecified on EINTR, so retrying its + * numeric value could close an unrelated descriptor after reuse. */ + (void) close((int) fd); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { return mkdir((const char *) (uintptr_t) lpszPath, (mode_t) mode); diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 3b0162fc..f3bc1ddd 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -128,6 +128,12 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send return com_questdb_network_Net_EOTHERDISCONNECT; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + const int result = shutdown((int) fd, SHUT_RDWR); + return result == -1 && errno == ENOTCONN ? 0 : result; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong ptr, jint len) { ssize_t n; diff --git a/core/src/main/c/share/net.h b/core/src/main/c/share/net.h index 27143639..d80617e2 100644 --- a/core/src/main/c/share/net.h +++ b/core/src/main/c/share/net.h @@ -80,6 +80,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send (JNIEnv *, jclass, jint, jlong, jint); +/* + * Class: io_questdb_client_network_Net + * Method: shutdown + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *, jclass, jint); + /* * Class: io_questdb_client_network_Net * Method: getSndBuf diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 119f6315..7e57de94 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -103,6 +103,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 FILE_ATTRIBUTE_NORMAL); } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + return open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL); +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { jint fd = open_file((const char *) (uintptr_t) lpszName, @@ -215,6 +224,25 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + jint fd = open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + if (fd < 0) { + return -1; + } + if (!FlushFileBuffers(FD_TO_HANDLE(fd))) { + SaveLastError(); + CloseHandle(FD_TO_HANDLE(fd)); + return -1; + } + CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { FILE_END_OF_FILE_INFO eof; @@ -331,6 +359,20 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!UnlockFileEx(FD_TO_HANDLE(fd), 0, MAXDWORD, MAXDWORD, &ov)) { + SaveLastError(); + return -1; + } + /* Unlock success confirms that the slot is reusable. Match POSIX by + * closing once without making handle cleanup part of that signal. */ + (void) CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { (void) mode; diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index fd290629..d105b504 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -230,6 +230,20 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_configureNonBlocking return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + const int result = shutdown((SOCKET) fd, SD_BOTH); + if (result == SOCKET_ERROR) { + const int error = WSAGetLastError(); + if (error == WSAENOTCONN) { + return 0; + } + WSASetLastError(error); + SaveLastError(); + } + return result; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong addr, jint len) { const int n = recv((SOCKET) fd, (char *) addr, len, 0); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..94018fff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -248,6 +248,15 @@ public void close() { } } + /** + * Shuts down socket traffic without releasing the descriptor or freeing + * buffers that a concurrent I/O worker may still access. The owner must + * call {@link #close()} after joining the worker to complete cleanup. + */ + public void closeTraffic() { + socket.closeTraffic(); + } + /** * Connects to a WebSocket server. * diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..3f272eb9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -189,11 +189,19 @@ public class QwpWebSocketSender implements Sender { // Null in production; set reflectively by tests. @TestOnly private volatile java.util.function.Supplier clientFactoryOverride; + // Test-only lifecycle witness. drainOnClose() invokes and clears it after + // confirming a real unacknowledged close target, immediately before waiting. + private volatile Runnable closeDrainWaitingHook; // close() drain timeout in millis. Default applied at construction. // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. + private volatile boolean closeCleanupComplete; + private boolean closeCleanupStarted; private long closeFlushTimeoutMillis = 5_000L; private volatile boolean closed; + // Test-only lifecycle witness. close() invokes and clears it strictly after + // publishing closed=true and before starting any drain or teardown work. + private volatile Runnable closeStartedHook; private boolean connected; private SenderConnectionDispatcher connectionDispatcher; // Async-delivery sink for SenderConnectionEvent notifications. Default @@ -278,11 +286,23 @@ public class QwpWebSocketSender implements Sender { private boolean ownsCursorEngine; private long pendingBytes; // Set true by close() once the SF slot flock has been released (the normal - // teardown path). Stays false if close() bailed early with the I/O thread - // still running -- then cursorEngine.close() never ran and the flock is - // still held, so the owning pool MUST keep the slot reserved rather than - // hand the still-locked dir to the next borrow ("sf slot already in use"). - private boolean slotLockReleased; + // teardown path). Stays false if an I/O or manager worker did not stop and + // cursorEngine retained the flock, so the owning pool MUST keep the slot + // reserved rather than hand the still-locked dir to the next borrow + // ("sf slot already in use"). May flip to true LATER via the getter's + // re-probe of retainedEngine, once the deferred cleanup (manager-worker + // exit path or delegated I/O-thread close) releases the flock — pools + // re-probe retired slots to recover their capacity. volatile: written on + // the closing thread, read by pool threads. + private volatile boolean slotLockReleased; + // Optional owning-pool notification, relayed after the engine confirms the + // flock release and slotLockReleased is visible to pool re-probes. + private volatile Runnable slotLockReleaseListener; + // Engine whose close() could not complete during sender close() — its + // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() + // re-probes it so a late flock release becomes visible to the owning pool. + // Only ever set inside close(); null for a sender that closed cleanly. + private volatile CursorSendEngine retainedEngine; private int pendingRowCount; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a @@ -1038,13 +1058,26 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { * replaying frames get a 2.5s grace window plus a 0.5s stop window * — worst case ~3s when a drainer sits in a blocking native * connect (15s background deadline) and must be abandoned to exit - * on its own. + * on its own; + *
  • SF manager stop: normally immediate, bounded by 5s when its worker + * is stuck in a filesystem operation. On timeout the slot remains + * locked rather than being exposed to a stale worker.
    • * */ @Override public void close() { if (!closed) { closed = true; + Runnable hook = closeStartedHook; + closeStartedHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-started test hook: {}", String.valueOf(t)); + } + } boolean ioThreadStopped = true; // Captures the first error from the flush/drain path AND any // secondary errors from cleanup steps (added via addSuppressed). @@ -1175,106 +1208,26 @@ public void close() { } if (!ioThreadStopped) { - // The I/O thread may still be using the socket and microbatch - // buffers. Freeing them would risk SIGSEGV. - LOG.error("I/O thread is still running, leaking WebSocket client and microbatch buffers"); - // The engine, however, need not leak: delegate its close to - // the I/O thread's exit path, which runs it strictly after - // the thread's last engine access — the mapping and slot - // lock release as soon as the stuck wire call resolves - // (bounded by OS timeouts). slotLockReleased intentionally - // stays false: the lock is released only when the delegated - // close actually runs, so the pool must not reuse the slot - // meanwhile. A false return means the thread exited between - // the failed close() and now — then closing here is safe. - if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null - && !cursorSendLoop.delegateEngineClose()) { - try { - cursorEngine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; + // The worker may still touch every resource below. Hand the + // complete sender-owned tail to its exit path rather than + // permanently leaking everything except the engine. The + // callback is idempotence-gated by closeRemainingResources(). + if (ownsCursorEngine && cursorEngine != null) { + retainedEngine = cursorEngine; } - rethrowTerminal(terminalError); - return; - } - - if (buffer0 != null) { - try { - buffer0.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer0: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (buffer1 != null) { - try { - buffer1.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer1: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - - if (client != null) { - try { - client.close(); - } catch (Throwable t) { - LOG.error("Error closing WebSocket client: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - client = null; - } - - if (ownsCursorEngine && cursorEngine != null) { - try { - cursorEngine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - cursorEngine = null; - ownsCursorEngine = false; - } - // Past the ioThreadStopped guard => cursorEngine.close() ran and - // released the SF flock in its finally (or this sender owned no - // engine holding one). Signal the pool it may reuse the slot. - slotLockReleased = true; - - // Shutdown order: dispatcher last, after the I/O loop has stopped - // producing into it. close() drains pending entries with a short - // deadline so any final errors land in the user's handler. - if (errorDispatcher != null) { - try { - errorDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing error dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (progressDispatcher != null) { - try { - progressDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing progress dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (connectionDispatcher != null) { - try { - connectionDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing connection dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); + Runnable closeCallback = () -> closeRemainingResources(null); + if (cursorSendLoop != null && cursorSendLoop.delegateClose(closeCallback)) { + rethrowTerminal(terminalError); + return; } + // The worker exited between close() failing and delegation. + // Cleanup is safe here and its failures remain suppressed on + // the original close error. + terminalError = closeRemainingResources(terminalError); + } else { + terminalError = closeRemainingResources(terminalError); } - LOG.info("QwpWebSocketSender closed"); - // If close() ended up holding the same instance the user already // caught earlier, suppress the rethrow. The user's catch block // wraps close() (try-with-resources), and Throwable refuses @@ -1286,14 +1239,57 @@ public void close() { } } + @TestOnly + public boolean isCloseCleanupComplete() { + return closeCleanupComplete; + } + /** - * True once {@link #close()} has released the store-and-forward slot - * flock. False means close() leaked the still-running I/O thread (and its - * resources), so the flock is still held; the owning pool must keep the - * slot index reserved instead of reusing the still-locked slot dir. + * True once the store-and-forward slot flock has been released. False + * means an I/O or manager worker did not stop and close() retained the + * lock and worker-reachable resources; the owning pool must keep the slot + * index reserved instead of reusing the still-locked dir. + *

    + * Not a one-shot snapshot: when close() left engine cleanup pending on a + * manager-worker quiescence or I/O-thread exit path, this re-probes the + * retained engine and latches true the moment that cleanup completes — pools re-probe retired + * slots through this getter to recover their capacity. Monotonic: + * false→true only, never back. Cheap (volatile reads on every common + * path) so pools may call it under their capacity lock; only the rare + * orphaned-retry state below does more. + *

    + * The probe is also the recovery surface for a retained engine whose + * flock-release retry fell off the shared driver because the driver + * thread could not start (e.g. OOM at thread creation): close() is + * one-shot, so without the re-arm below that slot's capacity would stay + * lost until process exit. */ public boolean isSlotLockReleased() { - return slotLockReleased; + if (slotLockReleased) { + return true; + } + CursorSendEngine engine = retainedEngine; + if (engine != null) { + if (engine.isCloseCompleted()) { + // Benign latch race: concurrent callers may both observe the + // completed cleanup and both write true. + slotLockReleased = true; + return true; + } + engine.ensureFlockReleaseRetryScheduled(); + } + return false; + } + + /** + * Registers a callback for confirmed SF slot-lock release. Pools use this + * to wake borrowers without waiting for a timeout or housekeeper tick. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && slotLockReleased) { + listener.run(); + } } @Override @@ -1709,6 +1705,21 @@ public long getDroppedConnectionNotifications() { return d == null ? 0L : d.getDroppedNotifications(); } + @TestOnly + public SenderConnectionDispatcher getConnectionDispatcherForTesting() { + return connectionDispatcher; + } + + @TestOnly + public SenderErrorDispatcher getErrorDispatcherForTesting() { + return errorDispatcher; + } + + @TestOnly + public SenderProgressDispatcher getProgressDispatcherForTesting() { + return progressDispatcher; + } + /** * Number of {@link SenderError} notifications dropped because the * bounded inbox was full. Non-zero means the user-supplied @@ -2135,6 +2146,31 @@ public void setClientFactoryOverride(java.util.function.Suppliercancel() could closeTraffic() a client the walk no longer owns; + // proven a no-op on both production transports today (closed-fd closeTraffic + // no-ops), but a future custom transport that threw on a closed socket would + // spuriously loud-fail close(). Null-guarded: the no-arg reconnect() path + // and the Unsafe.allocateInstance bare-loop tests pass a null handle. Pairs + // with the success-path clear() after upgrade(). + private static void clearInFlight(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + if (cancellation != null) { + cancellation.clear(); + } + } + + private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { // Background (drainer) factories share this connect walk -- endpoint // list and hostTracker HEALTH state (never the shared round: a // background sweep walks its own RoundCursor and records with @@ -2808,9 +2886,34 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); + if (cancellation != null) { + // Publish the client we are about to block on so a + // concurrent CursorWebSocketSendLoop.close() can break its + // traffic and unwind a black-holed native connect + // (connect_timeout=0 => OS SYN-retry) instead of hanging on + // the untimed shutdown-latch await. The publish-then-check + // handshake pairs with ConnectCancellation.cancel(): if we + // observe cancellation here we skip the blocking connect + // entirely; otherwise cancel() observed this client and + // breaks it. The walk's per-attempt catch disposes the + // client and, since running has flipped false, the + // top-of-loop ctx.isAborted() gate ends the walk. + cancellation.publish(newClient); + if (cancellation.isCancelled()) { + throw new LineSenderException(ctx.abortMessage()); + } + } newClient.connect(ep.host, ep.port); int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); + if (cancellation != null) { + // connect()+upgrade() completed: this client is no longer + // blocking, so drop it from the in-flight handle before it + // becomes the loop's `client` field. close() must then break + // its traffic via the field path exactly once -- leaving it + // in the handle too would double-shut-down the socket. + cancellation.clear(); + } } catch (HttpClientException e) { // Close BEFORE classify: the sibling catch (Error) below does not // guard catch-arm bodies, so an Error thrown inside classify() @@ -2820,6 +2923,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // upgradeStatusCode) that are set during upgrade() and survive // close(). newClient.close(); + clearInFlight(cancellation); HttpClientException classified = QwpUpgradeFailures.classify(newClient, ep.host, ep.port, e); if (classified instanceof QwpIngressRoleRejectedException) { QwpIngressRoleRejectedException re = (QwpIngressRoleRejectedException) classified; @@ -2866,6 +2970,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { continue; } catch (Exception e) { newClient.close(); + clearInFlight(cancellation); hostTracker.recordTransportError(idx, !background); lastError = e; if (!background) { @@ -2888,6 +2993,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // the cursor reconnect loop, BackgroundDrainer) rethrows Error // rather than retrying, so this stays a loud one-shot failure. closeQuietlyOnError(newClient); + clearInFlight(cancellation); throw e; } // Guard the post-upgrade tail: from here until newClient is @@ -3084,6 +3190,83 @@ private void checkTableSelected() { } } + private synchronized Throwable closeRemainingResources(Throwable terminalError) { + if (closeCleanupStarted) { + return terminalError; + } + closeCleanupStarted = true; + try { + try { + buffer0.close(); + } catch (Throwable t) { + LOG.error("Error closing buffer0: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + try { + buffer1.close(); + } catch (Throwable t) { + LOG.error("Error closing buffer1: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + if (client != null) { + try { + client.close(); + } catch (Throwable t) { + LOG.error("Error closing WebSocket client: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + client = null; + } + if (ownsCursorEngine && cursorEngine != null) { + CursorSendEngine engine = cursorEngine; + try { + engine.close(); + } catch (Throwable t) { + LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + slotLockReleased = false; + retainedEngine = engine; + } + } else { + slotLockReleased = true; + } + if (errorDispatcher != null) { + try { + errorDispatcher.close(); + } catch (Throwable t) { + LOG.error("Error closing error dispatcher: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + } + if (progressDispatcher != null) { + try { + progressDispatcher.close(); + } catch (Throwable t) { + LOG.error("Error closing progress dispatcher: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + } + if (connectionDispatcher != null) { + try { + connectionDispatcher.close(); + } catch (Throwable t) { + LOG.error("Error closing connection dispatcher: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + } + LOG.info("QwpWebSocketSender closed"); + return terminalError; + } finally { + closeCleanupComplete = true; + } + } + private int countNonEmptyTables(ObjList keys) { int tableCount = 0; for (int i = 0, n = keys.size(); i < n; i++) { @@ -3185,6 +3368,16 @@ private void drainOnClose(boolean errorOwnedByCustomHandler) { if (cursorEngine.ackedFsn() >= target) { return; } + Runnable hook = closeDrainWaitingHook; + closeDrainWaitingHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); + } + } long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; while (cursorEngine.ackedFsn() < target) { // Stop on a latched terminal (acks will never reach target); @@ -3545,6 +3738,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm resetTableBuffersAfterFlush(keys); } + private void onSlotLockReleased() { + slotLockReleased = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { + listener.run(); + } + } + private void resetTableBuffersAfterFlush(ObjList keys) { for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); @@ -3850,7 +4051,12 @@ boolean isAborted() { @Override public WebSocketClient reconnect() { - return buildAndConnect(this); + return buildAndConnect(this, null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + return buildAndConnect(this, cancellation); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 469c91ea..6e7ebc8c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; @@ -49,11 +50,11 @@ * offset 8: i64 fsn * *

    - * Zero-alloc, zero-syscall writes: {@link #open(String)} opens the + * Zero-alloc, store-only ACK writes: {@link #open(String)} opens the * file and maps the 16 bytes once. {@link #write(long)} is a single - * 8-byte aligned {@code Unsafe.putLong} into the mapped region. No - * malloc/free, no read/write syscalls, no rename, on the manager's hot - * tick path. An 8-byte aligned store is hardware-atomic on x86_64 and + * 8-byte aligned {@code Unsafe.putLong} into the mapped region. Ordinary + * ACK-only manager updates require no malloc/free, read/write syscalls, or + * rename. An 8-byte aligned store is hardware-atomic on x86_64 and * arm64, and disk-atomic within one sector (the file is 16 bytes — * trivially within one sector), so a torn FSN across a crash boundary * is not a concern. @@ -67,11 +68,13 @@ * complexity (multi-store with fence + read-side validate) for * marginal additional safety. *

    - * fsync cadence: intentionally NOT performed. Host crash falls - * back to recovery's {@code lowestBase - 1} seed (same as before this - * feature, no regression), at the cost of a bounded re-replay window - * for whatever durable-acks landed since the last successful page - * cache flush. + * fsync cadence: ordinary ACK-only manager updates call + * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim + * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs + * the slot directory before unlinking, and fsyncs it again after the batch. A + * fully drained close uses the same + * covering order so the durable watermark guards any acknowledged segment that + * a host crash restores. *

    * Lifecycle: single-writer (the {@link SegmentManager} worker * thread) after construction. Read once at engine startup (any thread, @@ -98,6 +101,7 @@ public final class AckWatermark implements QuietCloseable { private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class); private static final int MAGIC_OFFSET = 0; private final int fd; + private final FilesFacade filesFacade; private final long mmapAddress; private boolean closed; // Stamped once per process either at open() (if the file already @@ -108,8 +112,9 @@ public final class AckWatermark implements QuietCloseable { // construction; no synchronisation needed. private boolean magicWritten; - private AckWatermark(int fd, long mmapAddress, boolean magicAlreadyWritten) { + private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, boolean magicAlreadyWritten) { this.fd = fd; + this.filesFacade = filesFacade; this.mmapAddress = mmapAddress; this.magicWritten = magicAlreadyWritten; } @@ -122,7 +127,7 @@ public void close() { Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); } } @@ -136,6 +141,10 @@ public void close() { * stamps the magic and the new FSN atomically. */ public static AckWatermark open(String slotDir) { + return open(FilesFacade.INSTANCE, slotDir); + } + + static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; // Decide by size: existing-and-correct -> openRW preserves the // previous session's watermark (defeating which is the whole @@ -143,17 +152,17 @@ public static AckWatermark open(String slotDir) { // wrong-sized -> openCleanRW + allocate creates a fresh // FILE_SIZE-byte file (zero magic, read() reports INVALID until // the first write). - long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; if (existing == FILE_SIZE) { - fd = Files.openRW(filePath); + fd = filesFacade.openRW(filePath); } else { - fd = Files.openCleanRW(filePath); - if (fd >= 0 && !Files.allocate(fd, FILE_SIZE)) { + fd = filesFacade.openCleanRW(filePath); + if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) { // FilesFacade.allocate contract on a false return: // close the fd AND unlink the partial file. - Files.close(fd); - Files.remove(filePath); + filesFacade.close(fd); + filesFacade.remove(filePath); fd = -1; } } @@ -165,7 +174,7 @@ public static AckWatermark open(String slotDir) { long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath); - Files.close(fd); + filesFacade.close(fd); return null; } // Inspect the existing magic once at open time. If it's already @@ -173,7 +182,7 @@ public static AckWatermark open(String slotDir) { // shutdown), the first write() can skip the magic store // entirely and degenerate to a single 8-byte FSN put. int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET); - return new AckWatermark(fd, addr, magic == FILE_MAGIC); + return new AckWatermark(filesFacade, fd, addr, magic == FILE_MAGIC); } /** @@ -183,7 +192,11 @@ public static AckWatermark open(String slotDir) { * would only confuse the next session's seed. */ public static void removeOrphan(String slotDir) { - Files.remove(slotDir + "/" + FILE_NAME); + removeOrphan(FilesFacade.INSTANCE, slotDir); + } + + static boolean removeOrphan(FilesFacade filesFacade, String slotDir) { + return filesFacade.remove(slotDir + "/" + FILE_NAME); } /** @@ -203,6 +216,23 @@ public long read() { return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET); } + /** + * Flushes the mapped bytes and then the backing fd. The caller must sync + * the slot directory after this succeeds so a newly-created watermark's + * directory entry is durable before segment deletion begins. + */ + public void sync() { + if (closed) { + throw new IllegalStateException("ack watermark is closed"); + } + if (filesFacade.msync(mmapAddress, FILE_SIZE, false) != 0) { + throw new IllegalStateException("could not msync ack watermark"); + } + if (filesFacade.fsync(fd) != 0) { + throw new IllegalStateException("could not fsync ack watermark"); + } + } + /** * Atomically updates the persisted FSN. Single 8-byte aligned * store to the mapped region. The first write also stamps the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..9e606b94 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -26,10 +26,17 @@ import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongConsumer; /** * Facade that bundles a {@link SegmentRing} with a {@link SegmentManager} and @@ -63,6 +70,18 @@ public final class CursorSendEngine implements QuietCloseable { public static final long DEFAULT_APPEND_DEADLINE_NANOS = 30_000_000_000L; private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class); + private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY = + runnable -> new Thread(runnable, "qdb-sf-flock-release-retry"); + private static final long FLOCK_RELEASE_RETRY_BASE_PARK_NANOS = 100_000_000L; // 100 ms + private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object(); + private static final long FLOCK_RELEASE_RETRY_MAX_PARK_NANOS = 5_000_000_000L; // 5 s + private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>(); + private static volatile Runnable afterFlockReleaseRetryFailureHook; + private static volatile Runnable beforeDeferredCloseCreationHook; + private static volatile LongConsumer flockReleaseRetryParkOverride; + private static Thread flockReleaseRetryThread; + private static volatile ThreadFactory flockReleaseRetryThreadFactory = + DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY; private final long appendDeadlineNanos; // Number of times appendBlocking observed BACKPRESSURE_NO_SPARE on its first // ring.appendOrFsn attempt. One increment per blocking-call that had to wait @@ -70,6 +89,11 @@ public final class CursorSendEngine implements QuietCloseable { // writer; volatile because the user may sample it from any thread. private final java.util.concurrent.atomic.AtomicLong backpressureStallCount = new java.util.concurrent.atomic.AtomicLong(); + // Constructed before an owned manager acquires its native path scratch, so + // callback allocation failure cannot orphan manager resources. A timed-out + // close can then hand it to either manager path without allocating. + private final Runnable deferredClose; + private final FilesFacade filesFacade; private final SegmentManager manager; // We own the manager iff the user constructed us with no manager — in that // case close() also stops the manager. When the manager is shared across @@ -114,6 +138,51 @@ public final class CursorSendEngine implements QuietCloseable { // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. private volatile boolean closed; + // True once close() has run its full cleanup sequence INCLUDING a + // CONFIRMED slot-flock release — finishClose() publishes this strictly + // after SlotLock.release() reports success, never before. Pool threads + // treat the flip as "the slot dir is reusable" and free the slot index + // the moment they observe it (QwpWebSocketSender.isSlotLockReleased -> + // SenderPool.reprobeRetiredSlots), so publishing before the release + // would let a replacement engine's SlotLock.acquire collide with the + // still-open fd. Stays false when a close attempt could not confirm + // manager-worker quiescence (or the flock release itself failed) and had + // to leak the ring/watermark/slot lock — in that case a later close() + // call retries the cleanup (the worker may have exited by then). + // volatile: latched by finishClose(), but read lock-free by + // isCloseCompleted() from pool threads re-probing a retired slot (see the + // getter for why it must not synchronize). + private volatile boolean closeCompleted; + // Invoked after closeCompleted publishes a confirmed flock release. Used + // by an owning sender pool to wake capacity-starved borrowers immediately. + private volatile Runnable slotLockReleaseListener; + // Test-only hook run by finishClose() between the terminal cleanup and + // the flock release. Lets a test park the releasing thread inside the + // cleanup/release window and assert that closeCompleted stays false — + // i.e. that completion is never observable while the flock is still + // held. volatile: finishClose may run on the manager worker's exit + // thread while the hook is installed from a test thread. + private volatile Runnable beforeFlockReleaseHook; + // Ensures this engine has at most one entry in the shared flock-release + // retry driver. The error path only: ordinary closes never enqueue work. + private final AtomicBoolean flockReleaseRetryStarted = new AtomicBoolean(); + // Published before deferredClose is registered. The manager lock provides + // the callback handoff fence; volatile also covers a direct test/retry read. + private volatile boolean fullyDrainedForDeferredClose; + // Exactly-once claim on the terminal cleanup (finishClose). Contended by + // close() and a worker-exit handoff (completeDeferredClose); whoever wins + // the CAS runs the cleanup, the loser never touches ring/watermark/flock. + // Deliberately NOT the engine monitor: a retried close() holds the + // monitor while joining the manager worker, and the worker cannot die + // until its exit-path cleanup finishes — monitor-based exclusion would + // stall that close() for its full join budget (test-visible livelock). + // With the CAS the worker's cleanup never blocks, so the join returns as + // soon as the pass ends. + private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean(); + // Published only after ring/watermark/unlink cleanup is finished. A close + // that loses terminalCleanupClaimed may retry the flock only after this + // becomes true, otherwise it could expose the slot while cleanup is live. + private volatile boolean terminalResourcesCleaned; // Producer-thread-only: timestamp of the last "we're backpressured" log // line, used to throttle. Plain long is fine. private long lastBackpressureLogNs; @@ -137,9 +206,7 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { - this(sfDir, segmentSizeBytes, - new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + this(sfDir, segmentSizeBytes, null, true, maxTotalBytes, appendDeadlineNanos); } /** @@ -153,6 +220,22 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager mana private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, boolean ownsManager, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, manager, ownsManager, + SegmentManager.UNLIMITED_TOTAL_BYTES, appendDeadlineNanos); + } + + private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos) { + // Allocate the bound callback before constructing an owned manager. + // Field initializers have completed, but no engine-owned native/disk + // resource exists yet. If callback allocation throws, construction + // stops without a manager whose native path scratch could be orphaned. + this.deferredClose = createDeferredClose(); + if (ownsManager && manager == null) { + manager = new SegmentManager( + segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes); + } + // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. @@ -172,11 +255,9 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // separate mkdir step needed here. acquiredLock = SlotLock.acquire(sfDir); } catch (Throwable t) { - // The delegating constructors evaluate `new SegmentManager(...)` - // BEFORE this body runs, so on a pre-try throw (e.g. slot lock - // collision) an owned manager is already alive and would leak - // its native path-scratch sink -- 256 bytes per failed - // construction attempt. Close it before propagating. + // Callback creation and owned-manager construction have already + // completed. A slot-lock failure must close the owned manager's + // native path scratch before propagating. if (ownsManager) { try { manager.close(); @@ -190,6 +271,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.sfDir = sfDir; this.segmentSizeBytes = segmentSizeBytes; this.manager = manager; + this.filesFacade = manager.filesFacade(); this.ownsManager = ownsManager; this.appendDeadlineNanos = appendDeadlineNanos; @@ -204,9 +286,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs // already on disk and corrupting ACK translation, trim, and replay. - SegmentRing recovered = memoryMode ? null - : SegmentRing.openExisting(sfDir, segmentSizeBytes); - this.wasRecoveredFromDisk = recovered != null; + SegmentRing.Recovery recovery = memoryMode ? null + : SegmentRing.recover(filesFacade, sfDir, segmentSizeBytes); + SegmentRing recovered = recovery == null ? null : recovery.ring(); + this.wasRecoveredFromDisk = recovery != null + && recovery.status() == SegmentRing.RecoveryStatus.RECOVERED; if (recovered != null) { ringInProgress = recovered; // Seed ackedFsn to one below the lowest segment's baseSeq. @@ -256,7 +340,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // open() returns null on any setup failure so a missing // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. - watermarkInProgress = AckWatermark.open(sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -307,23 +391,54 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // so the new session's first read() correctly reports // INVALID (magic=0 on a freshly zero-filled file). if (!memoryMode) { - AckWatermark.removeOrphan(sfDir); - watermarkInProgress = AckWatermark.open(sfDir); + AckWatermark.removeOrphan(filesFacade, sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); } MmapSegment initial; String initialPath = null; + SfManifest initialManifest = null; if (memoryMode) { initial = MmapSegment.createInMemory(0L, segmentSizeBytes); } else { + // Created WITHOUT the manifest-required flag: the flag is + // a durable promise that sf-manifest.bin exists, so it + // must only be stamped after the manifest itself is on + // disk. Stamping it at create time would leave a crash + // window (initial durable, manifest not yet created) + // whose recovery hard-fails with "segment requires + // missing manifest" even though nothing was lost. initialPath = sfDir + "/sf-initial.sfa"; - initial = MmapSegment.create(initialPath, 0L, segmentSizeBytes); + initial = MmapSegment.create(filesFacade, initialPath, 0L, segmentSizeBytes); } try { - ringInProgress = new SegmentRing(initial, segmentSizeBytes); + if (!memoryMode) { + // Ordering is load-bearing for crash recovery: + // 1. initial segment durable (header + dirent), + // 2. manifest created (its own create fsyncs), + // 3. flag stamped on the segment. + // A crash after (1) recovers via the legacy + // (manifest-less) path; after (2) via the manifest + // path with an unflagged-but-valid empty active; + // after (3) it is the steady state. Every window + // self-heals without operator action. + initial.syncHeader(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new MmapSegmentException( + "could not sync fresh SF segment directory " + sfDir); + } + initialManifest = SfManifest.create(filesFacade, sfDir, 0L, 0L); + initial.markManifestRequired(); + } + ringInProgress = new SegmentRing(initial, segmentSizeBytes, initialManifest); + initialManifest = null; } catch (Throwable t) { initial.close(); + if (initialManifest != null) { + initialManifest.close(); + SfManifest.removeFile(filesFacade, sfDir); + } if (initialPath != null) { - Files.remove(initialPath); + filesFacade.remove(initialPath); } throw t; } @@ -480,7 +595,7 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos @Override public synchronized void close() { - if (closed) return; + if (closed && closeCompleted) return; closed = true; // Capture drain state BEFORE closing the ring — once the ring is // closed, its accessors aren't safe to read. The active segment is @@ -491,36 +606,209 @@ public synchronized void close() { // against potentially-fresh server state — duplicate writes when // the server has no dedup state for those messageSequences. // Memory mode has no files to unlink. - // The whole close sequence runs under try/finally so the slot lock - // is ALWAYS released, even if manager/ring close or unlink throws — - // otherwise a kernel-held flock outlives the engine and the next - // sender for the same slot collides on a lock the dead engine - // never released. + // + // Cleanup is gated on worker quiescence: releasing the ring, + // watermark, segment files or the slot lock while the manager worker + // is still mid service pass would let a replacement engine acquire + // the slot and have its files unlinked by the stale worker — + // store-and-forward data loss after restart. When the bounded worker + // join times out on an owned manager, cleanup OWNERSHIP TRANSFERS to + // the worker's exit path (deferUntilWorkerExit): the worker is + // provably the last thread able to touch the slot directory, so it + // releases everything the moment its in-flight pass finishes instead + // of leaking the slot until process exit. + // + // "Fully drained" includes BOTH the obvious case (every published + // FSN has been acked) AND the never-published case (publishedFsn + // < 0). The latter matters because a drainer adopting an empty + // orphan slot — segments filtered as empty by recovery, engine + // recreates a fresh sf-initial.sfa — would otherwise leave that + // fresh empty file behind, the next scanner finds it, adopts the + // slot again, and the cycle repeats forever (M6). + // Own try/catch so sabotaged/broken ring state cannot skip the + // quiescence barrier below or the slot-lock release in finishClose. + boolean drained = false; try { - // "Fully drained" includes BOTH the obvious case (every published - // FSN has been acked) AND the never-published case (publishedFsn - // < 0). The latter matters because a drainer adopting an empty - // orphan slot — segments filtered as empty by recovery, engine - // recreates a fresh sf-initial.sfa — would otherwise leave that - // fresh empty file behind, the next scanner finds it, adopts the - // slot again, and the cycle repeats forever (M6). - boolean fullyDrained = sfDir != null + drained = sfDir != null && (ring.publishedFsn() < 0 || ring.ackedFsn() >= ring.publishedFsn()); - // Each cleanup step in its own try/catch so a single failure - // doesn't strand later cleanups — mirrors the constructor's - // catch block. Without this, a throw from manager.deregister - // or manager.close() would leave the ring mmap'd and any - // residual .sfa files on disk, where the next sender can - // adopt them and replay already-acked data. + } catch (Throwable ignored) { + } + final boolean fullyDrained = drained; + fullyDrainedForDeferredClose = fullyDrained; + // Each cleanup step in its own try/catch so a single failure + // doesn't strand later cleanups — mirrors the constructor's + // catch block. Without this, a throw from manager.deregister + // or manager.close() would leave the ring mmap'd and any + // residual .sfa files on disk, where the next sender can + // adopt them and replay already-acked data. + try { + manager.deregister(ring); + } catch (Throwable ignored) { + } + // Quiescence barrier. deregister alone only removes the entry + // from the registry — the worker may still be mid service pass + // for this ring (creating a spare file, trimming, unlinking). + boolean workerQuiescent = false; + if (ownsManager) { + // Stopping and reaping a private manager is a stronger barrier + // than waiting for this ring alone. Do it directly so a stuck + // worker consumes at most one workerJoinTimeoutMillis budget, + // rather than one here and a second one in manager.close(). try { - manager.deregister(ring); + manager.close(); } catch (Throwable ignored) { } - if (ownsManager) { + try { + workerQuiescent = manager.isWorkerReaped(); + } catch (Throwable ignored) { + } + } else { + // A shared manager must keep serving its other rings, so wait + // only for the deregistered ring's current pass to finish. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { + } + } + if (!workerQuiescent && ownsManager) { + // Ownership handoff: manager.close() already stopped the worker + // loop (running=false), so the worker exits as soon as its + // in-flight pass finishes — it is merely slow, or wedged in a + // syscall. Either way it is the last thread able to touch the + // slot, so hand it the terminal cleanup instead of leaking the + // slot until process exit. completeDeferredClose and a retried + // close() race through the terminalCleanupClaimed CAS, so the + // cleanup runs exactly once and the worker never blocks on the + // engine monitor a retried close() holds while joining it. + boolean handedOff = false; + boolean registrationFailed = false; + try { + handedOff = manager.deferOwnedEngineCloseUntilWorkerExit(deferredClose); + } catch (Throwable ignored) { + // Unexpected monitor/VM failure carries no worker-liveness + // information. Ordinary handoff cannot allocate: both the + // callback and the manager's single slot were preallocated. + registrationFailed = true; + } + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the worker's exit path and run the " + + "moment its in-flight service pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can race " + + "the stale worker on slot {}", sfDir == null ? "" : sfDir); + return; + } + if (registrationFailed) { + // The handoff never registered and the worker was never + // observed — it must be presumed live and mid service pass. + // Retain every worker-reachable resource (ring, watermark, + // segment files, slot flock) and leave terminalCleanupClaimed + // unclaimed and closeCompleted false, exactly like the + // shared-manager leak branch: manager.close() above already + // stopped the worker loop, so a retried close() converges via + // isWorkerReaped() once the in-flight pass ends. The kernel + // releases the slot flock on process exit regardless. + LOG.error("SF worker-exit handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live " + + "worker cannot corrupt a future engine on slot {}. close() may be " + + "invoked again to retry cleanup once the worker has exited.", + sfDir == null ? "" : sfDir); + return; + } + // Handoff rejected: the worker loop exited between the failed + // bounded join and the registration attempt (both sides share the + // manager's lock, so this observation is exact). A worker past + // its loop can never touch slot paths again — inline cleanup is + // as safe as a reaped worker. + workerQuiescent = true; + } + if (!workerQuiescent) { + // A shared manager keeps serving sibling rings, so it cannot use + // the whole-worker exit handoff above. Transfer cleanup to this + // ring's current pass instead. Registration and pass completion + // share the manager lock: true means the worker owns cleanup; + // false means the pass already finished and inline cleanup is safe. + boolean handedOff = false; + boolean registrationFailed = false; + try { + handedOff = manager.deferUntilRingQuiescent(ring, deferredClose); + } catch (Throwable ignored) { + registrationFailed = true; + } + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the current ring pass and run the " + + "moment that pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can " + + "race the stale worker on slot {}", + sfDir == null ? "" : sfDir); + return; + } + if (registrationFailed) { + LOG.error("SF ring-pass cleanup handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live worker " + + "cannot corrupt a future engine on slot {}. The kernel releases the " + + "flock on process exit.", sfDir == null ? "" : sfDir); + return; + } + workerQuiescent = true; + } + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A worker-exit handoff or earlier close owns the one-time + // ring/watermark cleanup. Once that work is published complete, + // this close may still retry an unconfirmed flock release. + retryFlockReleaseIfReady(); + return; + } + finishClose(fullyDrained); + } + + /** + * Terminal cleanup: closes the ring and watermark, unlinks drained + * segment files, releases the slot flock, and — only once the release + * is confirmed — latches {@link #closeCompleted}. Publish order + * is load-bearing: pools free the slot index the instant they observe + * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must + * never be visible while the flock is still held, or a replacement + * engine races the release and fails acquisition on a live slot. + * The caller must hold the engine monitor and + * must have established that the manager worker can no longer touch the + * slot directory (reaped, provably exited, or running this on its own + * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the + * claim token, not the engine monitor, is the exclusion between a + * worker-exit handoff and a retried close(), so ring/watermark/flock are + * never double-released and the worker's cleanup can never deadlock + * against a close() that holds the monitor while joining the worker. + */ + private void finishClose(boolean fullyDrained) { + try { + RuntimeException durabilityFailure = null; + // On a fully-drained close, persist the final acked FSN through + // the still-mapped watermark BEFORE closing the ring/watermark + // and BEFORE unlinking any segment file. The manager persists + // the watermark only on its own tick, so it may lag the final + // ack. If the unlink below then fails (or the process dies + // mid-unlink), residual acknowledged .sfa files without an + // up-to-date watermark would seed the successor's recovery at + // lowestBase - 1 and replay already-acknowledged rows -- + // duplicates on a non-DEDUP table. The write is a single mmap + // store, so it succeeds even when the unlink is about to fail + // (e.g. the slot dir turned read-only). Quiescence is already + // established here, so no manager tick can race this write. + if (fullyDrained && watermark != null) { try { - manager.close(); - } catch (Throwable ignored) { + long finalAckedFsn = ring.ackedFsn(); + if (finalAckedFsn >= 0) { + watermark.write(finalAckedFsn); + watermark.sync(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new IllegalStateException( + "could not fsync SF slot directory before segment cleanup"); + } + } + } catch (RuntimeException e) { + durabilityFailure = e; } } try { @@ -541,25 +829,334 @@ public synchronized void close() { } catch (Throwable ignored) { } } - if (fullyDrained) { + if (fullyDrained && watermark != null && durabilityFailure == null) { + boolean segmentsRemoved = false; try { - unlinkAllSegmentFiles(sfDir); + segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir); } catch (Throwable ignored) { } + // Remove the watermark ONLY once every segment unlink is + // durable. Until the directory fsync succeeds, stable storage + // may still contain acknowledged segments and therefore still + // needs the durable watermark. + if (segmentsRemoved) { + if (filesFacade.fsyncDir(sfDir) != 0) { + durabilityFailure = new IllegalStateException( + "could not fsync SF slot directory after segment cleanup"); + } else { + AckWatermark.removeOrphan(filesFacade, sfDir); + } + } else { + LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack " + + "watermark so residual acknowledged segments stay covered -- the next " + + "engine on this slot recovers them as fully acked and retries the " + + "unlink on its own close", sfDir); + } + } else if (fullyDrained && watermark == null && sfDir != null) { + LOG.warn("close-time segment cleanup skipped on slot {} because no ack watermark " + + "is available to cover a host crash during unlink", sfDir); + } + if (durabilityFailure != null) { + throw durabilityFailure; + } + } finally { + // Reaching finishClose at all requires established quiescence, so + // releasing the flock is safe even if a step above threw. Leaking + // it would strand the slot until process exit for no reason. + // + // ORDER MATTERS: explicitly release the flock, verify it, and + // only then publish closeCompleted. Pools read isCloseCompleted() + // as "the slot dir is reusable" and free the slot index the moment + // it flips. SlotLock closes the fd once after confirmed unlock, + // but that close result cannot safely control publication because + // POSIX may consume the fd even when close reports failure. + Runnable hook = beforeFlockReleaseHook; + if (hook != null) { try { - AckWatermark.removeOrphan(sfDir); + hook.run(); } catch (Throwable ignored) { + // test-only; must never block the release } } - } finally { - if (slotLock != null) { + terminalResourcesCleaned = true; + retryFlockReleaseIfReady(); + } + } + + /** + * Installs a hook invoked after each failed shared-driver flock release. + * Test-only: makes persistent retry progress observable without sleeps. + */ + @TestOnly + public static void setAfterFlockReleaseRetryFailureHook(Runnable hook) { + afterFlockReleaseRetryFailureHook = hook; + } + + /** + * Installs a constructor fault hook immediately before the bound deferred + * close callback is created. Test-only: proves callback allocation failure + * occurs before an owned manager acquires native resources. + */ + @TestOnly + public static void setBeforeDeferredCloseCreationHook(Runnable hook) { + beforeDeferredCloseCreationHook = hook; + } + + /** + * Replaces the shared retry driver's inter-round park with a callback + * receiving the park duration the driver would have used. Test-only: + * makes the retry cadence inspectable and rounds coordinatable without + * wall-clock waits. + */ + @TestOnly + public static void setFlockReleaseRetryParkOverride(LongConsumer override) { + flockReleaseRetryParkOverride = override; + } + + /** + * Overrides creation of the single shared flock-release retry thread. + * Test-only: makes thread creation/start failure and persistent retry + * scalability deterministic without relying on scheduler timing. + */ + @TestOnly + public static void setFlockReleaseRetryThreadFactory(ThreadFactory factory) { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + if (flockReleaseRetryThread != null || !FLOCK_RELEASE_RETRY_QUEUE.isEmpty()) { + throw new IllegalStateException("flock-release retry driver is active"); + } + flockReleaseRetryThreadFactory = factory == null + ? DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY + : factory; + } + } + + /** + * Installs a hook that {@link #finishClose} runs between the terminal + * cleanup and the slot-flock release. Test-only: makes the otherwise + * microsecond-wide cleanup/release window deterministic so tests can + * assert {@link #isCloseCompleted()} stays false until the release is + * confirmed. + */ + @TestOnly + public void setBeforeFlockReleaseHook(Runnable hook) { + this.beforeFlockReleaseHook = hook; + } + + /** + * Runs on a safe manager-worker handoff path when {@link #close()} moved + * cleanup ownership to either a shared manager's ring-pass completion or + * an owned manager's worker exit. + * Deliberately does NOT take the engine monitor: a retried close() can + * hold it while joining this very worker, and the thread cannot die until + * this method returns — the {@link #terminalCleanupClaimed} CAS provides + * the exactly-once exclusion instead, so the worker always exits promptly + * and the racing close() converges via {@code isWorkerReaped()}. + */ + private void completeDeferredClose() { + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A retried close() (or an earlier duplicate handoff) already ran + // the terminal cleanup. + return; + } + finishClose(fullyDrainedForDeferredClose); + LOG.info("deferred SF engine resource cleanup completed after manager-worker quiescence; " + + "slot release confirmed: {} [slot={}]", + closeCompleted, sfDir == null ? "" : sfDir); + } + + private Runnable createDeferredClose() { + Runnable hook = beforeDeferredCloseCreationHook; + if (hook != null) { + hook.run(); + } + return this::completeDeferredClose; + } + + private boolean retryFlockReleaseIfReady() { + if (closeCompleted || !terminalResourcesCleaned) { + return closeCompleted; + } + boolean released; + if (slotLock != null) { + try { + released = slotLock.release(); + } catch (Throwable ignored) { + released = false; + } + } else { + released = true; + } + if (released) { + closeCompleted = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { try { - slotLock.close(); + listener.run(); } catch (Throwable ignored) { - // best-effort; flock is also released by kernel on process exit + // A notification failure must not invalidate a confirmed release. + } + } + return true; + } + startFlockReleaseRetry(); + return false; + } + + private static void runFlockReleaseRetryDriver() { + // Capped exponential backoff: a persistent unlock failure must not + // burn a fixed 10 rounds of native release syscalls per second + // forever, but a transient failure must still recover promptly. The + // ramp doubles per fully-failed round from 100 ms up to 5 s. + long parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; + while (true) { + final int roundSize; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + roundSize = FLOCK_RELEASE_RETRY_QUEUE.size(); + if (roundSize == 0) { + flockReleaseRetryThread = null; + return; + } + } + boolean hasFailures = false; + boolean hasSuccesses = false; + for (int i = 0; i < roundSize; i++) { + final CursorSendEngine engine; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + engine = FLOCK_RELEASE_RETRY_QUEUE.pollFirst(); } + if (engine.retryFlockReleaseIfReady()) { + engine.flockReleaseRetryStarted.set(false); + hasSuccesses = true; + } else { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(engine); + } + hasFailures = true; + Runnable hook = afterFlockReleaseRetryFailureHook; + if (hook != null) { + hook.run(); + } + } + } + if (hasSuccesses) { + // Progress: the failure condition is clearing, so retry the + // remaining engines on the base cadence again. + parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; } + if (hasFailures) { + // Interruption must not abandon a retained flock, but clear + // the flag so subsequent parks still throttle retries. + Thread.interrupted(); + LongConsumer parkOverride = flockReleaseRetryParkOverride; + if (parkOverride != null) { + parkOverride.accept(parkNanos); + } else { + LockSupport.parkNanos(parkNanos); + } + parkNanos = Math.min(parkNanos * 2, FLOCK_RELEASE_RETRY_MAX_PARK_NANOS); + } + } + } + + private void startFlockReleaseRetry() { + if (!flockReleaseRetryStarted.compareAndSet(false, true)) { + return; + } + Throwable startFailure = null; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(this); + if (flockReleaseRetryThread != null) { + // The driver may be parked on a ramped backoff; wake it so a + // freshly failed release gets its first driver retry promptly + // instead of inheriting older engines' full backoff. + LockSupport.unpark(flockReleaseRetryThread); + } else { + try { + Thread retryThread = flockReleaseRetryThreadFactory.newThread( + CursorSendEngine::runFlockReleaseRetryDriver); + if (retryThread == null) { + throw new IllegalStateException("retry thread factory returned null"); + } + retryThread.setDaemon(true); + flockReleaseRetryThread = retryThread; + retryThread.start(); + } catch (Throwable t) { + startFailure = t; + flockReleaseRetryThread = null; + CursorSendEngine queued; + while ((queued = FLOCK_RELEASE_RETRY_QUEUE.pollFirst()) != null) { + queued.flockReleaseRetryStarted.set(false); + } + } + } + } + if (startFailure == null) { + LOG.error("SF slot flock release failed during engine close; keeping " + + "closeCompleted=false and retrying on the shared driver so " + + "retired capacity recovers after the transient failure [slot={}]", + sfDir == null ? "" : sfDir); + } else { + // A later explicit close() or a pool's retired-slot probe + // (ensureFlockReleaseRetryScheduled) can still retry without + // repeating the one-time ring/watermark cleanup. The failed queue + // is cleared so the driver does not retain engines it cannot + // service. + LOG.error("Could not start SF flock-release retry driver; a retried close() " + + "or pool re-probe re-arms the retry [slot={}, error={}]", + sfDir == null ? "" : sfDir, String.valueOf(startFailure)); + } + } + + /** + * Whether {@link #close()} completed all cleanup, including a + * confirmed release of the SF slot lock — the flip is published + * strictly after explicit unlock succeeds, so observing {@code true} + * guarantees the slot dir is acquirable by a replacement engine. A false + * value after close means manager-worker quiescence could not be + * confirmed (or the flock release itself failed) and the + * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager), + * which flips this to true the moment the worker's in-flight pass + * finishes, or leaked until a retried close() (shared manager). Owners + * must not reuse the slot while this is false; pools may re-probe it to + * recover a retired slot's capacity once it flips. + *

    + * Deliberately unsynchronized ({@code closeCompleted} is volatile): pools + * probe this under their own capacity lock, and the deferred cleanup can + * hold the engine monitor through munmap/unlink syscalls — a synchronized + * getter would stall the pool's hot borrow path behind them. + */ + public boolean isCloseCompleted() { + return closeCompleted; + } + + /** + * Registers a callback for confirmed SF slot-lock release. If release + * already completed, invokes the callback before returning. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && closeCompleted) { + listener.run(); + } + } + + /** + * Re-arms the shared flock-release retry for an engine whose terminal + * cleanup finished but whose confirmed flock release is still pending + * and no longer scheduled — the retry driver thread failed to start when + * the release first failed (e.g. OOM at thread creation). + * {@code Sender.close()} is one-shot by contract, so pool probes + * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a + * retired slot's capacity recoverable instead of lost until process + * exit. Cheap unless the engine is in that orphan state (volatile reads, + * then one failed CAS when the retry is already scheduled), so probes + * may call it under their capacity lock. + */ + public void ensureFlockReleaseRetryScheduled() { + if (closeCompleted || !terminalResourcesCleaned) { + return; } + startFlockReleaseRetry(); } /** @@ -660,36 +1257,138 @@ public long recoveredOrphanTipFsn() { } /** - * Unlinks every {@code .sfa} file under {@code dir}. Called only on - * clean shutdown when the ring confirms every published FSN has been - * acked — at that moment the slot has no recoverable work and the - * files are pure noise that would mislead the next sender's recovery. - * Best-effort: logs and continues on failures, since we're already on - * the close path. + * Ascending removal rank of a segment file name for + * {@link #unlinkAllSegmentFiles(String)}. {@code sf-initial.sfa} is + * always the fresh-start segment at baseSeq 0, so it ranks first. + * Spare files carry a monotonic generation ({@code sf-.sfa}) + * assigned in creation == rotation == baseSeq order, so the parsed + * generation ranks them. Anything else with the extension is not a + * live segment and ranks last. */ - private static void unlinkAllSegmentFiles(String dir) { - if (!io.questdb.client.std.Files.exists(dir)) return; - long find = io.questdb.client.std.Files.findFirst(dir); + private static long segmentCleanupRank(String name) { + if ("sf-initial.sfa".equals(name)) { + return Long.MIN_VALUE; + } + if (name.length() == 23 && name.startsWith("sf-")) { + try { + return Long.parseUnsignedLong(name.substring(3, 19), 16); + } catch (NumberFormatException ignored) { + // fall through to the unrecognized-name rank + } + } + return Long.MAX_VALUE; + } + + /** + * Unlinks every {@code .sfa} file under {@code dir}, then the SF + * manifest. Called only on clean shutdown when the ring confirms every + * published FSN has been acked — at that moment the slot has no + * recoverable work and the files are pure noise that would mislead the + * next sender's recovery. + *

    + * Crash safety comes from a single durable manifest update committed + * BEFORE the first unlink: collapsing the boundaries to + * {@code head == active} declares every frame below the active base + * trimmed, so any subset of surviving files recovers cleanly (stale + * files are discarded, a surviving active recovers as the whole chain, + * zero files with a leftover manifest is the recognized drain window + * and recovers as EMPTY). Removal still runs in ascending segment order + * and STOPS at the first failure; the retained ack watermark (== the + * final acked FSN) covers every surviving frame, so a successor replays + * nothing. Legacy slots without a manifest keep the pre-manifest + * argument: ascending-order removal preserves a contiguous top slice + * that passes FSN-contiguity. + * + * @return {@code true} only when enumeration succeeded and every + * {@code .sfa} file was confirmed removed — the caller keeps the ack + * watermark on {@code false} so residual acknowledged segments stay + * covered. + */ + private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir) { + if (!filesFacade.exists(dir)) return true; + long find = filesFacade.findFirst(dir); if (find < 0) { LOG.warn("close-time unlink could not enumerate {}; " + "any residual sf-*.sfa files will be picked up by the next recovery", dir); - return; + return false; } - if (find == 0) return; + if (find == 0) return true; + ArrayList names = new ArrayList<>(); + int rc = 1; try { - int rc = 1; while (rc > 0) { String name = io.questdb.client.std.Files.utf8ToString( - io.questdb.client.std.Files.findName(find)); - rc = io.questdb.client.std.Files.findNext(find); - if (name == null || !name.endsWith(".sfa")) continue; - String path = dir + "/" + name; - if (!io.questdb.client.std.Files.remove(path)) { - LOG.warn("Failed to unlink fully-acked segment {} on close", path); + filesFacade.findName(find)); + rc = filesFacade.findNext(find); + if (name != null && name.endsWith(".sfa")) { + names.add(name); } } } finally { - io.questdb.client.std.Files.findClose(find); + filesFacade.findClose(find); + } + if (rc < 0) { + // A partial listing must not drive any unlink: removing only the + // files we happened to see could delete the segment holding the + // highest frame while a lower one survives, leaving residual + // state the retained watermark can no longer vouch for. + LOG.warn("close-time unlink could not fully enumerate {}; " + + "leaving all segment files for the next recovery", dir); + return false; + } + names.sort((a, b) -> { + int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b)); + return byRank != 0 ? byRank : a.compareTo(b); + }); + // Manifest-aware drain protocol. Before removing anything, durably + // collapse the committed boundaries to head == active. From that + // moment EVERY crash state is a valid recovery input: + // - surviving data below the active base is "stale below head" + // (acked by definition here) and is discarded by recovery; + // - a surviving active-base segment recovers as the whole chain; + // - empty spares are validated extras; + // - zero segment files with the manifest still present is the + // recognized drain window and recovers as EMPTY. + // Without this barrier, deleting the file the manifest calls "head" + // would make the next startup fail on a slot that lost nothing. + // The manifest itself is removed last, after every segment unlink. + SfManifest manifest = null; + try { + try { + manifest = SfManifest.open(filesFacade, dir); + } catch (Throwable t) { + LOG.warn("close-time unlink could not read the SF manifest in {}; leaving " + + "all files for the next recovery", dir, t); + return false; + } + if (manifest != null && !names.isEmpty()) { + try { + long activeBase = manifest.activeBase(); + manifest.update(activeBase, activeBase); + } catch (Throwable t) { + LOG.warn("close-time unlink could not commit drain boundaries in {}; " + + "leaving all files for the next recovery", dir, t); + return false; + } + } + for (int i = 0, n = names.size(); i < n; i++) { + String path = dir + "/" + names.get(i); + if (!filesFacade.remove(path)) { + LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + + "residual files stay a watermark-covered, manifest-consistent range", path); + return false; + } + } + } finally { + if (manifest != null) { + manifest.close(); + } + } + if (!SfManifest.removeFile(filesFacade, dir)) { + // Safe to proceed: a manifest with zero segment files is the + // recognized drain window and recovers as EMPTY. + LOG.warn("could not remove SF manifest in {} after full segment cleanup", dir); } + return true; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..97b2d7d0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -88,6 +89,29 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * sub-second confirmation latency once the upload completes * server-side. {@code 0} or negative disables the keepalive entirely. */ + /** + * Bounded-await backstop for {@link #close()}: the maximum time close() + * waits for the I/O thread to stop (count down {@code shutdownLatch}) + * before it loud-fails and delegates final teardown to the I/O thread's + * exit path. In the common case the round-2 in-flight connect cancellation + * (see {@link ConnectCancellation}) makes the I/O thread unwind and count + * the latch down within milliseconds, so this budget is never reached. It + * only fires in the pathological, astronomically-rare TOCTOU window where + * {@code cancel()}'s {@code closeTraffic()} lands between the pre-connect + * guard and native fd creation, so it is a no-op and the connect blocks the + * full OS SYN-retry (~60-130s per endpoint, possibly across several). + *

    + * {@code 30_000} ms is comfortably UNDER the sidecar's 120 s shutdown + * deadline (~4x headroom) yet ~1000x a healthy close's millisecond latch + * countdown, so it never prematurely abandons a legitimately draining + * close (the I/O thread only has to finish its current — now traffic-broken + * — native send/receive and run {@code ioLoop}'s finally). This bounds the + * WAIT inside close(), NOT the connect itself (rejected Option A): a + * legitimately slow SUCCESSFUL connect is unaffected — it is not concurrent + * with a close, and when close() does fire it cancels the in-flight connect + * rather than waiting it out. + */ + public static final long DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS = 30_000L; public static final long DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS = 200L; public static final long DEFAULT_PARK_NANOS = 50_000L; // 50us idle backoff /** @@ -163,6 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // the durable watermark can lag behind the OK watermark. private final ArrayDeque pendingDurable = new ArrayDeque<>(); private final ArrayDeque pendingDurablePool = new ArrayDeque<>(); + // Race-safe cancellation handle for an in-flight connect attempt. Passed + // into reconnectFactory.reconnect(...) on the I/O thread so close() can + // break a connect blocked mid-attempt (a black-holed native connect that + // neither unpark nor interrupt cancels). See ConnectCancellation. + private final ConnectCancellation connectCancellation = new ConnectCancellation(); // Optional reconnect plumbing. When non-null, a wire failure triggers a // reconnect attempt instead of a terminal fail(). The factory produces a // fresh, connected+upgraded WebSocketClient. @@ -200,7 +229,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); - private WebSocketClient client; + private volatile WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's // handler. Null disables async delivery entirely; the producer-side @@ -225,6 +254,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // it is engine.ackedFsn() + 1, so the first replayed frame on the new // connection is wireSeq=0 and server-side cumulative ACKs still line up. private long fsnAtZero; + // Bounded-await backstop budget for close() (see + // DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via + // setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch + // deterministically without a multi-second real wait; production always + // uses the default. Read only on the owner thread inside close(). + private long shutdownAwaitTimeoutMillis = DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to @@ -244,11 +279,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; - // Failed-stop hand-off flag: set by delegateEngineClose() when an owner's - // close() could not stop the I/O thread and the engine close is therefore - // performed by the I/O thread's exit path. Write-once, owner thread only; - // read by the I/O thread strictly after its shutdown-latch countdown (see - // the handshake contract on delegateEngineClose). + // Failed-stop hand-off callback: set when an owner could not stop the I/O + // thread and must defer worker-reachable cleanup until the thread exits. + // Read strictly after shutdownLatch.countDown(); see delegateClose(). + private volatile Runnable delegatedClose; + // Engine-only hand-off retained for BackgroundDrainer, whose remaining + // resources are owned by its run method rather than by a sender. private volatile boolean engineCloseDelegated; // The latched terminal failure — THE exception every checkError() call // rethrows. Write-once for the loop's lifetime: the only writer is @@ -763,8 +799,74 @@ public synchronized void close() { // would block forever. isAlive()==false also covers the normal // post-exit case where the latch is already counted down. if (t.isAlive()) { + // Break a native send/receive before joining. Full client close + // must remain after the worker exit because it frees buffers the + // worker may still access. + WebSocketClient activeClient = client; + if (activeClient != null) { + try { + activeClient.closeTraffic(); + } catch (Throwable e) { + // A custom transport that cannot safely cancel traffic + // must not be destructively closed while the worker is + // live. Fail without joining indefinitely; the worker's + // exit path retains ownership of final cleanup. + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + } + // Cancel an in-flight connect attempt not yet installed as the + // `client` field: async initial connect leaves `client` null + // and a mid-flight reconnect leaves it pointing at the stale + // pre-drop client, so neither is reachable by the closeTraffic() + // above. The reconnect factory publishes the connecting client + // here before it blocks, so breaking its traffic unwinds a + // black-holed native connect (connect_timeout=0 => OS SYN-retry, + // ~60-130s) that would otherwise pin the untimed await below. + // Same loud-fail contract as the field client: a transport that + // cannot safely shut down traffic is not destructively closed; + // the worker's exit path retains ownership of final cleanup. try { - shutdownLatch.await(); + connectCancellation.cancel(); + } catch (Throwable e) { + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + try { + // Bounded backstop: never wait forever. The round-2 in-flight + // connect cancellation makes the I/O thread unwind and count + // the latch down in milliseconds, so this await returns true + // almost always. The timeout branch fires only in the + // pathological uninterruptible-connect TOCTOU window where + // cancel()'s closeTraffic() was a no-op and the connect blocks + // the full OS SYN-retry -- guaranteeing close() still returns + // (bounded, comfortably under the sidecar's 120s deadline) + // rather than hanging on an untimed await. + if (!shutdownLatch.await(shutdownAwaitTimeoutMillis, TimeUnit.MILLISECONDS)) { + // Latch still up after the budget: the I/O thread did not + // stop. Same failed-stop protocol as the interrupt branch + // below -- loud-fail without touching the client/engine + // (freeing native buffers under a still-live I/O thread + // risks a C5 SEGV; a quiet return would let the owner + // unmap the engine under it). The I/O thread's own exit + // path (ioLoop's finally) retains ownership of final + // cleanup; QwpWebSocketSender.close() keys its + // ioThreadStopped guard on this throw and BackgroundDrainer + // switches to delegateEngineClose(). ioThread stays set + // (not nulled below, since we throw) so a duplicate close() + // re-signals rather than silently succeeding. + throw new LineSenderException( + "cursor I/O thread did not stop: close() timed out after " + + shutdownAwaitTimeoutMillis + "ms awaiting shutdown; " + + "client/engine teardown is delegated to the I/O " + + "thread's exit path"); + } } catch (InterruptedException e) { // Re-assert the flag for the caller's stack, then decide. // If the I/O thread has genuinely not exited (latch still @@ -815,27 +917,20 @@ public synchronized void close() { } /** - * Failed-stop hand-off for the engine. Called by an owner whose - * {@link #close()} threw because the I/O thread would not stop: the owner - * must not free the engine (munmap/Unsafe.free of segment memory) while - * the thread may still touch it with raw {@code Unsafe} reads. Setting - * the delegation flag makes the I/O thread run {@code engine.close()} on - * its exit path, strictly after its last engine access and after the - * shutdown-latch countdown — releasing the slot lock as soon as the - * stuck wire call resolves (bounded by OS timeouts) instead of leaking - * the mapping and lock forever. - *

    - * Returns {@code true} when the I/O thread is still live and has adopted - * the engine close; {@code false} when the thread has already exited — - * the caller must close the engine itself. - *

    - * Memory model — the classic store/load handshake: this method writes the - * volatile flag, then reads the latch count; the exit path counts the - * latch down, then reads the flag. Under the sequential consistency of - * volatile (and AQS latch state) accesses, if this method observes the - * latch still up, the exit path is guaranteed to observe the flag — no - * missed close. If both sides act, {@link CursorSendEngine#close()} is - * synchronized and idempotent, so the double close is benign. + * Hands complete owner cleanup to the I/O thread when it could not be + * stopped. The callback runs after the thread's last client, buffer, and + * engine access. Returns false if the thread already exited, in which case + * the caller must run the callback. The callback itself must be idempotent: + * the latch/callback handshake guarantees execution but permits both sides + * to race after the countdown. + */ + public boolean delegateClose(Runnable closeCallback) { + delegatedClose = closeCallback; + return shutdownLatch.getCount() != 0L; + } + + /** + * Engine-only failed-stop hand-off used by BackgroundDrainer. */ public boolean delegateEngineClose() { engineCloseDelegated = true; @@ -983,6 +1078,17 @@ public void setProgressDispatcher(SenderProgressDispatcher dispatcher) { this.progressDispatcher = dispatcher; } + /** + * Test seam: shrink the {@link #close()} bounded-await backstop + * (default {@link #DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS}) so a test can + * exercise the timeout branch deterministically without a multi-second + * real wait. Production never calls this. Set before {@link #close()}. + */ + @TestOnly + public void setShutdownAwaitTimeoutMillis(long millis) { + this.shutdownAwaitTimeoutMillis = millis; + } + public synchronized void start() { if (ioThread != null) { throw new IllegalStateException("already started"); @@ -1114,7 +1220,10 @@ private void clearDurableAckTracking() { * before the first connect attempt — see {@link #failPaced}. */ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptMillis) { - if (reconnectFactory == null || !running) { + if (!running) { + return; + } + if (reconnectFactory == null) { recordFatal(initial); return; } @@ -1156,7 +1265,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { - WebSocketClient newClient = reconnectFactory.reconnect(); + WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { // close() ran while this connect attempt was in @@ -1670,15 +1779,14 @@ private void ioLoop() { } } shutdownLatch.countDown(); - // Failed-stop hand-off (see delegateEngineClose): the owner could - // not free the engine safely while this thread was alive, so the - // engine close — and with it the slot-lock release — happens - // here, strictly after this thread's last engine access. The flag - // is read only after the countDown: the store/load pairing with - // delegateEngineClose's flag-write-then-latch-read guarantees - // either this branch or the owner's fallback runs (or both — - // engine.close() is idempotent). - if (engineCloseDelegated) { + Runnable closeCallback = delegatedClose; + if (closeCallback != null) { + try { + closeCallback.run(); + } catch (Throwable ignored) { + // The owner callback logs individual cleanup failures. + } + } else if (engineCloseDelegated) { try { engine.close(); } catch (Throwable ignored) { @@ -2034,6 +2142,104 @@ private boolean tryRetireOrphanTail() { @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; + + /** + * Cancellable variant of {@link #reconnect()}. The loop passes a + * per-attempt {@link ConnectCancellation} so a transport that blocks + * inside a native connect can publish the in-flight client to the + * handle BEFORE it blocks; {@link #close()} then breaks that client's + * traffic to unwind a black-holed connect promptly, rather than falling + * back to the bounded {@code shutdownLatch.await(...)} backstop (which + * only bounds the WAIT, not the connect). + *

    + * Default: ignore the handle and delegate to {@link #reconnect()} -- + * a transport that cannot publish its in-flight client is simply not + * cancellable mid-connect, and {@code close()} falls back to its + * existing field-client {@code closeTraffic()} / loud-fail path. The + * default keeps this a {@link FunctionalInterface} and preserves + * source compatibility for existing lambda / method-reference + * implementors (e.g. {@link #connectWithRetry}). + */ + default WebSocketClient reconnect(ConnectCancellation cancellation) throws Exception { + return reconnect(); + } + } + + /** + * Race-safe cancellation handle for a single in-flight connect attempt. + * Owned per-loop and passed into {@link ReconnectFactory#reconnect(ConnectCancellation)} + * on the I/O thread. The transport's connect walk {@link #publish}es the + * {@link WebSocketClient} it is about to block on before the blocking + * {@code connect()}, and {@link #clear}s it once the attempt is installed + * or disposed. {@link #close()} (owner thread) calls {@link #cancel()} to + * break the in-flight client's traffic path so a black-holed native + * connect unwinds and the I/O thread counts down the shutdown latch. + *

    + * Java 8: two volatiles, no locks. Visibility/ordering: + *

    + * The publish (write inFlight, then read cancelled) / cancel (write + * cancelled, then read inFlight) handshake covers the close-vs-connect + * race in both directions: if the guard observes {@code cancelled==false} + * then, by the volatile total order, {@code cancel()} observed the + * published client and broke its traffic; if the guard observes + * {@code cancelled==true} it skips the blocking connect entirely. + */ + public static final class ConnectCancellation { + // The client the connect walk is currently about to block / blocked + // on. Written by the I/O thread only (publish/clear); read by the + // owner thread (cancel). volatile for cross-thread visibility. + private volatile WebSocketClient inFlight; + // Latched once close() requested cancellation. Written by the owner + // thread (cancel); read by the I/O thread's pre-connect guard. + private volatile boolean cancelled; + + public boolean isCancelled() { + return cancelled; + } + + /** + * I/O-thread hook: record the client the walk is about to block on, + * BEFORE the blocking {@code connect()}. Pairs with {@link #cancel()} + * so an attempt that publishes after cancellation is caught by the + * caller's {@link #isCancelled()} guard, and an attempt already + * blocked in {@code connect()} is broken by {@code cancel()}. + */ + public void publish(WebSocketClient client) { + inFlight = client; + } + + /** + * I/O-thread hook: the walk is done with the in-flight attempt + * (installed on success, disposed on failure). Drop the reference so a + * later {@link #cancel()} cannot touch a client the walk no longer + * owns. Single writer (I/O thread) for publish/clear, so no CAS is + * needed. + */ + public void clear() { + inFlight = null; + } + + /** + * Owner-thread hook from {@link #close()}: request cancellation, then + * break the traffic path of any in-flight connect so the I/O thread's + * connect walk unwinds. {@code cancelled} is written before reading + * {@code inFlight} to pair with {@link #publish(WebSocketClient)}. May + * throw when the transport cannot safely shut down traffic -- + * {@code close()} maps that to its existing loud-fail, exactly as it + * does for the field client's {@code closeTraffic()}. + */ + void cancel() { + cancelled = true; + WebSocketClient c = inFlight; + if (c != null) { + c.closeTraffic(); + } + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..ff7a00da 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -63,10 +63,12 @@ public final class MmapSegment implements QuietCloseable { public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian public static final int FRAME_HEADER_SIZE = 8; // u32 crc + u32 payloadLen public static final int HEADER_SIZE = 24; + public static final byte MANIFEST_REQUIRED_FLAG = 1; public static final byte VERSION = 1; private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); + private final FilesFacade filesFacade; private final String path; private final long sizeBytes; // memoryBacked: true when the segment buffer lives in malloc'd native @@ -95,6 +97,10 @@ public final class MmapSegment implements QuietCloseable { // because the consumer must see writes in publication order — once the // producer bumps publishedCursor, every byte before it is fully written. private volatile long publishedCursor; + // Monotonic in-memory link to the segment that immediately follows this + // one. SegmentRing publishes it before promoting the successor to active; + // close deliberately retains it so a cursor can advance after head trim. + private volatile MmapSegment successor; // Bytes between the last valid frame and the file end that look like an // attempted-but-invalid frame write (non-zero bytes at the bail-out // position). Zero for fresh segments and for cleanly partially-filled @@ -102,9 +108,10 @@ public final class MmapSegment implements QuietCloseable { // recovery callers for diagnostics. Final after construction. private final long tornTailBytes; - private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes, + private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes, long baseSeq, long initialCursor, long frameCount, boolean memoryBacked, long tornTailBytes) { + this.filesFacade = filesFacade; this.path = path; this.fd = fd; this.mmapAddress = mmapAddress; @@ -145,9 +152,13 @@ public static MmapSegment create(String path, long baseSeq, long sizeBytes) { * that filesystem only. */ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes) { + return create(ff, path, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes, boolean manifestRequired) { long pathPtr = ff.allocNativePath(path); try { - return create(ff, pathPtr, path, baseSeq, sizeBytes); + return create(ff, pathPtr, path, baseSeq, sizeBytes, manifestRequired); } finally { ff.freeNativePath(pathPtr); } @@ -162,13 +173,17 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long * per-call {@code byte[]} + native-malloc the way the String overload does. */ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) { + return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes, boolean manifestRequired) { if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) { throw new IllegalArgumentException( "sizeBytes too small for header + one minimal frame: " + sizeBytes); } - int fd = ff.openCleanRW(pathPtr); + int fd = ff.openRWExclusive(pathPtr); if (fd < 0) { - throw new MmapSegmentException("openCleanRW failed for " + displayPath); + throw new MmapSegmentException("exclusive create failed for " + displayPath); } // Reserve real disk blocks and advance EOF to sizeBytes in one // call. ENOSPC surfaces here, before the producer thread starts @@ -186,21 +201,21 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + displayPath); } // Header goes straight into the mapping — no separate write syscall. Unsafe.getUnsafe().putInt(addr, FILE_MAGIC); Unsafe.getUnsafe().putByte(addr + 4, VERSION); - Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags + Unsafe.getUnsafe().putByte(addr + 5, manifestRequired ? MANIFEST_REQUIRED_FLAG : (byte) 0); // flags Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); + return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // mmap (or header writes) failed after a successful allocate — @@ -234,7 +249,7 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { Unsafe.getUnsafe().putShort(addr + 6, (short) 0); Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); + return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); } catch (Throwable t) { Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT); throw t; @@ -280,7 +295,11 @@ public static MmapSegment openExisting(String path) { public static MmapSegment openExisting(FilesFacade ff, String path) { long fileSize = ff.length(path); if (fileSize < HEADER_SIZE) { - throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize); + // Corruption, not an operational error: the bytes themselves prove + // this cannot be a whole segment (a create() is never durable at a + // sub-header size — allocate() reserves the full extent up front). + throw new MmapSegmentCorruptionException( + "file shorter than header: " + path + " size=" + fileSize); } int fd = ff.openRW(path); if (fd < 0) { @@ -288,17 +307,22 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + path); } int magic = Unsafe.getUnsafe().getInt(addr); if (magic != FILE_MAGIC) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); } byte version = Unsafe.getUnsafe().getByte(addr + 4); if (version != VERSION) { + // Deliberately NOT the corruption subtype: an unsupported + // version is a well-formed segment written by a different + // client build (e.g. after a downgrade). Quarantine-renaming + // it would strand its frames for the writer that CAN read it; + // failing recovery keeps the slot intact for that writer. throw new MmapSegmentException("unsupported version in " + path + ": " + version); } long baseSeq = Unsafe.getUnsafe().getLong(addr + 8); @@ -310,7 +334,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // checks (which would place the segment last in baseSeq order // and trip the FSN-gap throw, taking the whole recovery down). if (baseSeq < 0L) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "bad baseSeq in " + path + ": " + baseSeq); } long lastGood = scanFrames(addr, fileSize); @@ -324,10 +348,10 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { + "Investigate disk health or unexpected writer crash.", path, tornTail, lastGood, fileSize, count); } - return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); + return new MmapSegment(ff, path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // The header reads above (magic/version/baseSeq) run before @@ -343,7 +367,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) { // covers the header block and any future reader placed ahead of the // scan. if (isMmapAccessFault(t)) { - throw new MmapSegmentException( + throw new MmapSegmentCorruptionException( "unreadable mapped header page in " + path + " (unbacked/sparse page 0): " + t.getMessage(), t); } @@ -375,16 +399,37 @@ public void close() { if (memoryBacked) { Unsafe.free(mmapAddress, sizeBytes, MemoryTag.NATIVE_DEFAULT); } else { - Files.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); + filesFacade.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); } mmapAddress = 0; } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); fd = -1; } } + boolean manifestRequired() { + return !memoryBacked && (Unsafe.getUnsafe().getByte(mmapAddress + 5) & MANIFEST_REQUIRED_FLAG) != 0; + } + + void markManifestRequired() { + if (memoryBacked || manifestRequired()) { + return; + } + Unsafe.getUnsafe().putByte(mmapAddress + 5, MANIFEST_REQUIRED_FLAG); + syncHeader(); + } + + void syncHeader() { + if (memoryBacked) { + return; + } + if (filesFacade.msync(mmapAddress, HEADER_SIZE, false) != 0 || filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment header " + path); + } + } + public boolean isFull() { return capacityRemaining() <= 0; } @@ -398,7 +443,7 @@ public void msync() { if (memoryBacked) return; // no on-disk pages to flush long pub = publishedCursor; if (pub > HEADER_SIZE) { - Files.msync(mmapAddress, pub, false); + filesFacade.msync(mmapAddress, pub, false); } } @@ -438,6 +483,21 @@ public long sizeBytes() { return sizeBytes; } + void linkSuccessor(MmapSegment next) { + if (next == null) { + throw new IllegalArgumentException("successor must not be null"); + } + MmapSegment existing = successor; + if (existing != null && existing != next) { + throw new IllegalStateException("segment successor already linked"); + } + successor = next; + } + + MmapSegment successor() { + return successor; + } + /** * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]} * starting at the current append cursor, then advances both cursors diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java new file mode 100644 index 00000000..34a89f38 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Positively-identified segment corruption: the file's own bytes prove it is + * not (or no longer) a readable SF segment — truncated below the fixed header, + * wrong magic, a negative {@code baseSeq}, or an unreadable (unbacked/torn) + * header page. Distinct from its parent {@link MmapSegmentException}, which + * recovery treats as an operational failure (open/mmap error on a file + * whose contents may be perfectly intact) and must therefore be fatal. + *

    + * Recovery quarantines corruption (rename to {@code .corrupt}) and + * relies on manifest boundaries / FSN contiguity to decide whether the + * quarantined file was load-bearing; operational failures always abort + * startup so a transient {@code EMFILE}/{@code ENOMEM} can never silently + * drop durable frames. + */ +public final class MmapSegmentCorruptionException extends MmapSegmentException { + + public MmapSegmentCorruptionException(String message) { + super(message); + } + + public MmapSegmentCorruptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index eec0c0d9..e54eecb5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -29,8 +29,14 @@ * too short for header, etc. Indicates the segment is unusable, not that * the disk is full (the latter surfaces as backpressure on the producer * via {@link io.questdb.client.cutlass.qwp.client.LineSenderException}). + *

    + * Recovery distinguishes two flavors: this base type marks operational + * failures (open/mmap/enumeration errors — the file's contents may be fine, + * so recovery must fail closed), while the {@link MmapSegmentCorruptionException} + * subtype marks positively-identified corruption in the file's own + * bytes, which recovery may quarantine instead of aborting. */ -public final class MmapSegmentException extends RuntimeException { +public class MmapSegmentException extends RuntimeException { public MmapSegmentException(String message) { super(message); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 3b7cff60..5319ee65 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -34,8 +34,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; /** * Background worker that keeps every registered {@link SegmentRing} supplied @@ -49,27 +52,32 @@ * in a JVM). Polls each ring on a configurable tick (default 1 ms) — short * enough that a producer rarely sees {@link SegmentRing#BACKPRESSURE_NO_SPARE} * in the steady state, long enough that an idle JVM doesn't burn CPU. - *

    - * baseSeq race window: the spare is created with - * {@code baseSeq = ring.nextSeqHint()} as observed by the manager. If the - * producer thread appends more frames before the rotation actually fires, - * the spare's baseSeq will be stale and {@link SegmentRing#appendOrFsn} will - * throw on the mismatch check. In practice this is benign — by the time - * {@link SegmentRing#needsHotSpare()} returns true the producer has very - * little room left in the active segment, and the manager polls fast enough - * to install before the producer fills the rest. Hardening to make the race - * impossible (lazy header write at rotation time) is a separate refinement - * deferred to PR2. */ public final class SegmentManager implements QuietCloseable { public static final long DEFAULT_POLL_NANOS = 1_000_000L; // 1 ms public static final long DISK_FULL_LOG_THROTTLE_NANOS = 30_000_000_000L; // 30 s public static final long UNLIMITED_TOTAL_BYTES = Long.MAX_VALUE; + private static final int ENTRY_DEREGISTERED = 3; + private static final int ENTRY_DEREGISTERED_IN_SERVICE = 2; + private static final int ENTRY_IN_SERVICE = 1; + private static final int ENTRY_REGISTERED = 0; + private static final AtomicReferenceFieldUpdater ENTRY_CLEANUP_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(RingEntry.class, Runnable.class, "quiescenceCleanup"); + private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state"); private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class); + private static final int MAX_TRIMS_PER_RING_PASS = 64; + private static final long TRIM_RETRY_INITIAL_NANOS = 4_000_000L; + private static final long TRIM_RETRY_MAX_NANOS = 1_024_000_000L; + private static final int TRIM_RETRY_NONE = 0; + private static final int TRIM_RETRY_POST_BARRIER = 3; + private static final int TRIM_RETRY_PRE_BARRIER = 1; + private static final int TRIM_RETRY_UNLINK = 2; private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; private final AtomicLong fileGeneration = new AtomicLong(); + private final FilesFacade filesFacade; private final Object lock = new Object(); private final long maxTotalBytes; // Reused by the manager worker thread to build spare-segment paths @@ -86,6 +94,22 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + private final LongSupplier ticks; + // Reused by the worker for one bounded trim quantum. Keeping the unlinked + // prefix here until the post-unlink directory barrier succeeds avoids both + // per-pass allocation and publishing logical removal before it is durable. + private final MmapSegment[] trimBatch = new MmapSegment[MAX_TRIMS_PER_RING_PASS]; + // Test seam: runs after a deferred ring-pass cleanup returns. Null in + // production; public sender/pool lifecycle tests use it to observe exact + // callback completion without sleeps or polling. + private volatile Runnable afterRingCleanupHook; + // Test seam: runs at the top of deferUntilWorkerExit, before the + // worker-liveness check. Null in production; registration-failure tests + // throw from it to simulate an allocation failure (OOM building the + // cleanup lambda or growing exitCleanups) while the worker is still + // live. Callers must treat such a throw as "worker state unknown", + // never as the exact false return meaning the worker loop has exited. + private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the install path's // synchronized(lock) entry (the one that performs installHotSpare + the // totalBytes += segmentSize commit). Null in production; tests use it to @@ -93,33 +117,74 @@ public final class SegmentManager implements QuietCloseable { // but before ownership/accounting commit. Callers may inject a deregister // or hold this stale worker snapshot while caller-side cleanup runs. private volatile Runnable beforeInstallSyncHook; + // Test seam: records entry into the per-ring quiescence wait. Null in + // production; owned-engine close tests use it to prove they take only the + // stronger whole-manager join path, not two sequential timeout budgets. + private volatile Runnable beforeRingQuiescenceAwaitHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically // inject a deregister(ring) call into the exact race window that the - // registered flag check inside the trim block closes for watermark writes - // and totalBytes accounting. + // entry-state check inside the trim block closes for watermark writes and + // totalBytes accounting. private volatile Runnable beforeTrimSyncHook; + // Test seam invoked exactly when a retry transition/recovery is logged. + // Null in production; persistent-failure tests use it to prove log bounds + // without binding to a particular SLF4J backend. + private volatile Runnable retryLogHook; + // Entry the worker is claiming or servicing, or null between passes. + // Volatile publication lets teardown find the entry after deregister() + // removes it from rings. RingEntry.state is the authoritative barrier: + // the worker publishes this reference before its REGISTERED->IN_SERVICE + // CAS and only touches ring resources after that CAS succeeds. + private volatile RingEntry inService; + // Cleanup actions handed to the worker's exit block by an owning engine + // whose close() found the worker still mid service pass after the bounded + // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock}; + // consumed exactly once by the worker-loop finally, which runs them + // OUTSIDE `lock`: they perform syscalls (munmap/unlink/flock release), + // and no caller-facing lock may ever be held while running third-party + // cleanup code. + private ObjList exitCleanups; + // A private manager belongs to exactly one CursorSendEngine. Its callback + // is preallocated by that engine and stored directly here, so the critical + // timed-out-close handoff never allocates an ObjList or grows a backing + // array. Guarded by lock and consumed on worker-loop exit outside lock. + private Runnable ownedEngineExitCleanup; private long lastDiskFullLogNs; private volatile boolean running; + // pathScratch free-exactly-once coordination between a timed-out close() + // and the worker's exit path. All three are guarded by {@link #lock}. + // When close() gives up on the join while the worker loop has not yet + // exited, it hands scratch ownership to the worker + // (scratchHandedToWorker=true) and the worker frees the buffer in its + // exit block; in every other case close() frees it. Without the handoff, + // a worker that outlives the bounded join leaks the native scratch + // buffer forever, because nobody retries manager cleanup after close() + // returns. + private boolean scratchFreed; + private boolean scratchHandedToWorker; + private boolean workerLoopExited; // Total bytes currently allocated across every segment owned by every // registered ring (active + sealed + hot-spare). Mutated by the manager // thread on provision/trim and by register/deregister callers under // {@link #lock}; the lock covers both paths so the counter stays // consistent across registration boundaries. private long totalBytes; - private long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; + // volatile: read by awaitRingQuiescence() from arbitrary caller threads + // while the @TestOnly setter may run on another. + private volatile long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; // volatile because wakeWorker() reads workerThread without holding the // monitor; the synchronized start()/close() pair handles the // start-vs-close ordering. private volatile Thread workerThread; public SegmentManager(long segmentSizeBytes) { - this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } public SegmentManager(long segmentSizeBytes, long pollNanos) { - this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } /** @@ -145,6 +210,22 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) { * hold an initial active plus one hot spare. */ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE, System::nanoTime); + } + + @TestOnly + public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, filesFacade, System::nanoTime); + } + + @TestOnly + public SegmentManager( + long segmentSizeBytes, + long pollNanos, + long maxTotalBytes, + FilesFacade filesFacade, + LongSupplier ticks + ) { // The pathScratch field initializer has already allocated its native // buffer by the time this body runs, so a validation throw must free // it or every failed construction leaks 256 bytes of native memory @@ -159,9 +240,15 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) "maxTotalBytes (" + maxTotalBytes + ") must allow at least one segment of " + segmentSizeBytes + " bytes"); } + this.filesFacade = filesFacade; this.segmentSizeBytes = segmentSizeBytes; this.pollNanos = pollNanos; this.maxTotalBytes = maxTotalBytes; + this.ticks = ticks; + } + + FilesFacade filesFacade() { + return filesFacade; } @Override @@ -194,18 +281,210 @@ public synchronized void close() { } } if (t.isAlive()) { - LOG.warn("SegmentManager worker did not stop before close wait completed; " - + "leaving worker-owned resources allocated"); - return; + synchronized (lock) { + if (!workerLoopExited) { + // Hand pathScratch ownership to the worker: its exit + // block frees the buffer under the same lock, so the + // native allocation is reclaimed even though this + // close() could not confirm termination. workerThread + // stays set so isWorkerReaped() reports the incomplete + // shutdown and a later close() can retry the join. + scratchHandedToWorker = true; + LOG.warn("SegmentManager worker did not stop before close wait completed; " + + "worker frees its native scratch buffer on exit"); + return; + } + } + // The worker loop has already run its exit block; the thread + // is at most a few instructions from terminating and can no + // longer touch manager state. Fall through and reap it. } workerThread = null; } // Free the rotation-path native scratch buffer only after worker - // termination has been observed. The worker is the only thread that - // touches the buffer, but close() uses a bounded join; if the worker is - // still alive, leaking this one scratch allocation is safer than freeing - // native memory it may still read or write. - pathScratch.close(); + // termination (or worker-loop exit) has been observed. The worker is + // the only thread that touches the buffer; the scratchFreed flag + // (shared with the worker's exit block) makes the free exactly-once + // no matter which side runs last. + synchronized (lock) { + if (!scratchFreed) { + scratchFreed = true; + pathScratch.close(); + } + } + } + + /** + * Hands the single owning engine's preallocated close callback to the + * worker exit block. Registration only assigns a reference under + * {@link #lock}; it cannot allocate in the timed-out teardown path. + * Returns {@code false} only after the worker loop has exited (or when it + * never started), so the caller may then clean up inline safely. + */ + public boolean deferOwnedEngineCloseUntilWorkerExit(Runnable cleanup) { + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (ownedEngineExitCleanup != null && ownedEngineExitCleanup != cleanup) { + throw new IllegalStateException("owned manager already has an engine-exit cleanup"); + } + ownedEngineExitCleanup = cleanup; + return true; + } + } + + /** + * Hands a cleanup action to the current service pass for {@code ring}. + * The worker runs it outside the manager lock immediately after that pass + * finishes. Returns {@code false} when no pass for the ring remains in + * flight, in which case the caller already owns a quiescent ring and must + * run the cleanup itself. + *

    + * Registration and pass completion coordinate through the entry's atomic + * state and callback fields, so there is no gap without an owner: either + * this method attaches the cleanup while the pass remains active, the + * worker claims an attached cleanup while completing, or this method + * observes completed state and rejects the handoff. A repeated registration + * for the same pass returns {@code true} without replacing its existing + * owner. + */ + public boolean deferUntilRingQuiescent(SegmentRing ring, Runnable cleanup) { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return false; + } + Runnable existing = ENTRY_CLEANUP_UPDATER.get(e); + if (existing == null && ENTRY_CLEANUP_UPDATER.compareAndSet(e, null, cleanup)) { + if (e.isInService()) { + return true; + } + // Completion changed the state before observing the callback. + // Remove our callback and clean inline, unless the worker already + // claimed it (in which case that worker remains the owner). + return !ENTRY_CLEANUP_UPDATER.compareAndSet(e, cleanup, null); + } + // A callback already attached to this pass is an owner. The sole + // production caller always supplies the engine's preallocated bound + // callback; duplicates intentionally do not replace it. + return true; + } + + /** + * Hands a cleanup action to the worker thread's exit block, to run + * strictly after the worker loop has finished its final service pass -- + * i.e. after the last point where the worker can create, write or unlink + * anything under a registered ring's slot directory. Returns {@code false} + * when the worker loop has already exited (or the worker never started): + * the caller must run the cleanup itself, which is equally safe for the + * same reason -- no further worker access to the slot is possible. + *

    + * This is the slot-ownership transfer used by an owning engine's close() + * when the bounded worker join timed out: instead of retiring the slot + * until process exit, ring/watermark/flock release moves to the worker, + * which is provably the last thread able to touch the slot directory. + * The registration here and the exit block's {@code workerLoopExited} + * flip share {@link #lock}, so the cleanup runs exactly once: either it + * is registered before the flip and the worker runs it, or the flip won + * and this method rejects the handoff. + *

    + * May throw on allocation failure (the lambda at the call site, the + * {@code ObjList}, or its growth). A throw carries NO liveness + * information: the worker was never observed, so the caller must treat + * it as "worker possibly still live" and retain resources — never as + * the exact {@code false} return above. + */ + public boolean deferUntilWorkerExit(Runnable cleanup) { + Runnable hook = beforeExitCleanupRegistrationHook; + if (hook != null) { + hook.run(); + } + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (exitCleanups == null) { + exitCleanups = new ObjList<>(); + } + exitCleanups.add(cleanup); + return true; + } + } + + /** + * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until + * the worker thread is provably no longer executing a service pass for + * {@code ring}, or the worker-join timeout elapses. After this returns + * {@code true}, the worker will never again touch the ring, its + * watermark, or path names under its slot directory: deregister has + * removed the entry from the registry, a stale snapshot entry that has + * not started its pass is skipped by the registration check at the top + * of {@link #serviceRing(RingEntry)}, and this method has observed the + * end of any in-flight pass. Only then may the caller release dependent + * resources (ring, watermark, segment files, slot lock). + *

    + * Returns {@code true} immediately when no worker is running or when + * called from the worker thread itself (test hooks inject deregister + * calls there; waiting would self-deadlock). Returns {@code false} when + * the in-flight pass did not finish within the timeout — the caller must + * treat the worker as still live and leak rather than release. + *

    + * A pending caller interrupt is preserved but does not abort the wait, + * mirroring {@link #close()}. + */ + public boolean awaitRingQuiescence(SegmentRing ring) { + Runnable hook = beforeRingQuiescenceAwaitHook; + if (hook != null) { + hook.run(); + } + Thread t = workerThread; + if (t == null || t == Thread.currentThread()) { + return true; + } + long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L; + boolean interrupted = Thread.interrupted(); + try { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return true; + } + synchronized (lock) { + e.quiescenceWaiters++; + try { + while (e.isInService()) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + try { + // Round up so a sub-millisecond remainder still waits + // instead of spinning through wait(0) == wait-forever. + lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } finally { + e.quiescenceWaiters--; + } + } + return true; + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * True when no manager worker thread can be running: either + * {@link #start()} was never called, or a {@link #close()} confirmed + * worker termination and reaped the thread. Owners use this as a + * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)} + * times out but a subsequent {@code close()} join succeeded. + */ + public synchronized boolean isWorkerReaped() { + return workerThread == null; } /** @@ -213,6 +492,13 @@ public synchronized void close() { * created after this returns, but already-installed spares stay with * the ring (the ring closes them on its own {@link SegmentRing#close}). * Idempotent; safe to call from any thread. + *

    + * Non-blocking: a worker service pass already in flight for this ring + * may still be running when this returns. Callers about to release + * resources the worker can reach (the ring itself, its watermark, its + * segment files, or the slot lock guarding its directory) MUST follow + * up with {@link #awaitRingQuiescence(SegmentRing)} and only release on + * a {@code true} result. */ public void deregister(SegmentRing ring) { synchronized (lock) { @@ -226,7 +512,7 @@ public void deregister(SegmentRing ring) { // single subtraction covers both the initial seed // and the net manager activity (provisions minus // trims) for this ring. - e.registered = false; + e.deregister(); totalBytes -= ring.totalSegmentBytes(); rings.remove(i); return; @@ -297,16 +583,36 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public void setAfterRingCleanupHook(Runnable hook) { + this.afterRingCleanupHook = hook; + } + + @TestOnly + public void setBeforeExitCleanupRegistrationHook(Runnable hook) { + this.beforeExitCleanupRegistrationHook = hook; + } + @TestOnly public void setBeforeInstallSyncHook(Runnable hook) { this.beforeInstallSyncHook = hook; } + @TestOnly + public void setBeforeRingQuiescenceAwaitHook(Runnable hook) { + this.beforeRingQuiescenceAwaitHook = hook; + } + @TestOnly public void setBeforeTrimSyncHook(Runnable hook) { this.beforeTrimSyncHook = hook; } + @TestOnly + public void setRetryLogHook(Runnable hook) { + this.retryLogHook = hook; + } + @TestOnly public void setWorkerJoinTimeoutMillis(long millis) { this.workerJoinTimeoutMillis = millis; @@ -341,21 +647,19 @@ public void wakeWorker() { * files in {@code dir}, or {@code -1} if none exist. Skips files that * don't match the pattern (e.g. the legacy {@code sf-initial.sfa}). */ - private static long scanMaxGeneration(String dir) { + private long scanMaxGeneration(String dir) { long max = -1L; - if (!Files.exists(dir)) return max; - long find = Files.findFirst(dir); + if (!filesFacade.exists(dir)) return max; + long find = filesFacade.findFirst(dir); if (find < 0) { - LOG.warn("scanMaxGeneration could not enumerate {}; " - + "next spare may collide with an existing on-disk segment", dir); - return max; + throw new IllegalStateException("could not enumerate SF segment directory " + dir); } if (find == 0) return max; try { int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - rc = Files.findNext(find); + String name = Files.utf8ToString(filesFacade.findName(find)); + rc = filesFacade.findNext(find); if (name == null || !name.startsWith("sf-") || !name.endsWith(".sfa")) { continue; } @@ -368,8 +672,11 @@ private static long scanMaxGeneration(String dir) { // sf-initial.sfa or non-hex — skip } } + if (rc < 0) { + throw new IllegalStateException("could not fully enumerate SF segment directory " + dir); + } } finally { - Files.findClose(find); + filesFacade.findClose(find); } return max; } @@ -411,7 +718,51 @@ private String nextSparePath(String dir) { return displayPath; } - private void serviceRing(RingEntry e) { + private boolean serviceRing(RingEntry e) { + // Publish before the CAS so deregister + await can always find the + // entry. The state CAS is the ownership decision: if deregister won, + // this stale snapshot pass skips the ring without touching it. + inService = e; + if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) { + inService = null; + return false; + } + boolean hasMoreTrimmable; + try { + hasMoreTrimmable = serviceRing0(e); + } finally { + e.finishService(); + Runnable cleanup = e.quiescenceCleanup == null + ? null + : ENTRY_CLEANUP_UPDATER.getAndSet(e, null); + inService = null; + // A normal pass performs only the two state CAS operations above. + // The manager monitor is entered here only for an actual close + // waiter; the recheck under lock prevents lost wakeups. + if (e.quiescenceWaiters > 0) { + synchronized (lock) { + if (e.quiescenceWaiters > 0) { + lock.notifyAll(); + } + } + } + if (cleanup != null) { + try { + cleanup.run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed after manager-worker ring pass", t); + } finally { + Runnable hook = afterRingCleanupHook; + if (hook != null) { + hook.run(); + } + } + } + } + return hasMoreTrimmable; + } + + private boolean serviceRing0(RingEntry e) { // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) @@ -455,14 +806,21 @@ private void serviceRing(RingEntry e) { // via its long-ptr overload, bypassing the byte[] + native // malloc that the String overload would incur on every // rotation. - spare = MmapSegment.create(FilesFacade.INSTANCE, + spare = MmapSegment.create(filesFacade, pathScratch.ptr(), path, - e.ring.nextSeqHint(), segmentSizeBytes); + e.ring.nextSeqHint(), segmentSizeBytes, true); } Runnable installHook = beforeInstallSyncHook; if (installHook != null) { installHook.run(); } + if (!memoryMode) { + spare.syncHeader(); + if (filesFacade.fsyncDir(e.dir) != 0) { + throw new MmapSegmentException( + "could not sync hot-spare directory " + e.dir); + } + } // Install + commit atomically under the manager lock. // If `e.ring` was deregistered between the snapshot // above and now, abandoning the spare here is the only @@ -476,7 +834,7 @@ private void serviceRing(RingEntry e) { // observe the spare in the ring (and subtract it) or // run before installation (so no install happens). synchronized (lock) { - if (e.registered) { + if (e.isRegistered()) { e.ring.installHotSpare(spare); totalBytes += segmentSizeBytes; installed = true; @@ -493,104 +851,302 @@ private void serviceRing(RingEntry e) { } catch (Throwable ignored) { } } - // Remove the file even when spare is null (i.e. when - // MmapSegment.create itself threw): MmapSegment.create's - // catch already best-effort removes, but if anything - // before mmap (e.g. an exception thrown by the JVM - // between openCleanRW and the try block) leaves a file - // on disk, this is the second-line defense. Repeated - // unlink on an already-removed path is a harmless no-op. - if (path != null) { - Files.remove(path); + // Only remove the file when the spare object exists, i.e. + // MmapSegment.create succeeded and ownership is ours but + // installation was rejected (ring deregistered/closed). + // When create() itself threw, its catch already removed + // anything it put on disk -- and with exclusive create + // (O_EXCL) a failure can also mean the path was ALREADY + // occupied by a file some other lifecycle owns, which a + // blanket unlink here would destroy. + if (path != null && spare != null) { + filesFacade.remove(path); } } } } - // 2. Trim any segments that the ring says are fully acked. For - // memory-mode rings, "trim" is just close() (Unsafe.free) — no - // file to unlink. - // - // The watermark write and totalBytes commit are registration-gated - // under `lock` so stale worker snapshots cannot touch the - // engine-owned watermark or mutate accounting after deregister() - // returns. drainTrimmable still runs for stale snapshots: it - // transfers ownership of fully-acked sealed segments to this - // worker, preserving the old close + unlink behavior. - // - // munmap + unlink stay outside the lock — they can be slow - // and shouldn't block register/deregister or sibling rings. - ObjList trim; + // 2. Trim any segments that the ring says are fully acked. Memory + // mode only frees native memory. Disk mode first makes the current + // cumulative ACK durable, then durably establishes the watermark's + // directory entry, unlinks one bounded batch, and finally commits + // those directory removals before ring/accounting state changes. + // No syscall runs under the manager lock. Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } - synchronized (lock) { - boolean registered = e.registered; - // Persist the current ackedFsn watermark BEFORE the trim runs. - // On host crash between the persist and the unlinks below, the - // segments survive and the watermark is correct. On crash AFTER - // the unlinks but before next tick, the segments are gone and - // the watermark is stale, but recovery clamps with - // max(lowestSurvivingBaseSeq - 1, watermark) so either ordering - // is safe. Memory-mode rings (and callers that didn't supply a - // watermark) skip the write. - // Persist only on advance to avoid pointless mmap stores when - // ackedFsn is steady. The store is a single 8-byte put against - // an already-mapped region -- no syscall, no allocation -- but - // the gate keeps the dirty-page footprint minimal under - // steady-state load with no new acks arriving. - if (registered && e.watermark != null) { - long currentAck = e.ring.ackedFsn(); - if (currentAck > e.lastPersistedAck) { - e.watermark.write(currentAck); - e.lastPersistedAck = currentAck; - } - } - trim = e.ring.drainTrimmable(); - if (registered && trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - totalBytes -= trim.get(i).sizeBytes(); + MmapSegment first = e.ring.firstTrimmable(); + if (first == null) { + // Preserve the cheap mmap-only watermark cadence for ACKs that do + // not yet cover a complete sealed segment. + synchronized (lock) { + if (e.isRegistered() && e.watermark != null) { + long currentAck = e.ring.ackedFsn(); + if (currentAck > e.lastPersistedAck) { + e.watermark.write(currentAck); + e.lastPersistedAck = currentAck; + } } } + return false; } - if (trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - MmapSegment s = trim.get(i); - String path = s.path(); + + if (memoryMode) { + int trimmed = 0; + while (trimmed < MAX_TRIMS_PER_RING_PASS) { + MmapSegment segment = e.ring.firstTrimmable(); + if (segment == null) { + break; + } try { - s.close(); - if (path != null && !Files.remove(path)) { - LOG.warn("Failed to unlink trimmed segment {}", path); + segment.close(); + synchronized (lock) { + if (e.ring.removeTrimmable(segment)) { + trimmed++; + if (e.isRegistered()) { + totalBytes -= segment.sizeBytes(); + } + } } } catch (Throwable t) { - LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); + LOG.warn("Failed to trim memory segment", t); + return false; + } + } + return trimmed == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + } + + // A deferred disk retry does no sync, unlink, or logging work. Signed + // subtraction is the standard wrap-safe deadline comparison because + // the bounded delay is many orders of magnitude below half the long + // range, even when the monotonic clock wraps. + long now = ticks.getAsLong(); + if (e.trimRetryDelayNanos != 0 && now - e.trimRetryAtNanos < 0) { + return false; + } + + // A disk segment cannot be safely unlinked without durable ACK cover. + // The registration check and mmap store are atomic with deregister. + // Once the store wins, deregistration may proceed, but its owner must + // await this in-service pass before closing the watermark or ring. + if (e.watermark == null) { + if (!e.missingWatermarkLogged) { + e.missingWatermarkLogged = true; + LOG.warn("Cannot durably trim acknowledged segments in {} without an ack watermark", e.dir); + } + return false; + } + long durableAck; + synchronized (lock) { + if (!e.isRegistered()) { + return false; + } + durableAck = e.ring.ackedFsn(); + if (durableAck > e.lastPersistedAck) { + e.watermark.write(durableAck); + e.lastPersistedAck = durableAck; + } + } + try { + e.watermark.sync(); + if (filesFacade.fsyncDir(e.dir) != 0) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, t); + return false; + } + + boolean trimFailed = false; + Throwable trimFailure = null; + int unlinked = 0; + MmapSegment segment = first; + long coveredAck = durableAck; + while (segment != null && unlinked < MAX_TRIMS_PER_RING_PASS) { + long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; + if (lastSeq > coveredAck) { + break; + } + MmapSegment next = e.ring.nextSealedAfter(segment); + String path = segment.path(); + try { + // Durably commit the head advance past this segment before + // unlinking it. The ring recomputes the successor under its + // own monitor -- computing it here from `next` would race + // with a concurrent rotation sealing the active, letting the + // head leapfrog a still-unacked sealed segment (whose file a + // later recovery would then discard as "stale below head"). + e.ring.advanceManifestHeadPast(segment); + segment.close(); + // A retry after a post-unlink directory-sync failure sees an + // already absent path. Treat absence as the prior successful + // unlink and retry the batch barrier rather than wedging. + if (!filesFacade.remove(path) && filesFacade.exists(path)) { + trimFailed = true; + break; + } + trimBatch[unlinked++] = segment; + segment = next; + } catch (Throwable t) { + trimFailed = true; + trimFailure = t; + break; + } + } + + if (unlinked > 0) { + try { + if (filesFacade.fsyncDir(e.dir) != 0) { + for (int i = 0; i < unlinked; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + for (int i = 0; i < unlinked; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); + return false; + } + synchronized (lock) { + for (int i = 0; i < unlinked; i++) { + MmapSegment removed = trimBatch[i]; + trimBatch[i] = null; + if (e.ring.removeTrimmable(removed) && e.isRegistered()) { + totalBytes -= removed.sizeBytes(); + } } } } + if (trimFailed) { + recordTrimFailure(e, TRIM_RETRY_UNLINK, now, trimFailure); + return false; + } + recordTrimSuccess(e); + return unlinked == MAX_TRIMS_PER_RING_PASS && e.ring.firstTrimmable() != null; + } + + private void recordTrimFailure(RingEntry e, int failureKind, long now, Throwable failure) { + long delay = e.trimRetryDelayNanos == 0 + ? TRIM_RETRY_INITIAL_NANOS + : Math.min(e.trimRetryDelayNanos << 1, TRIM_RETRY_MAX_NANOS); + e.trimRetryAtNanos = now + delay; + e.trimRetryDelayNanos = delay; + if (e.trimRetryFailureKind != failureKind) { + e.trimRetryFailureKind = failureKind; + if (failure == null) { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind)); + } else { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind), failure); + } + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private void recordTrimSuccess(RingEntry e) { + if (e.trimRetryDelayNanos != 0) { + e.trimRetryAtNanos = 0; + e.trimRetryDelayNanos = 0; + e.trimRetryFailureKind = TRIM_RETRY_NONE; + LOG.info("Durable segment trim recovered in {}", e.dir); + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private static String trimFailureName(int failureKind) { + switch (failureKind) { + case TRIM_RETRY_PRE_BARRIER: + return "covering barrier"; + case TRIM_RETRY_UNLINK: + return "segment unlink"; + default: + return "directory commit barrier"; + } } private void workerLoop() { - while (running) { - // Snapshot the rings under the lock so we don't hold it through the - // (potentially slow) syscalls during creation/unlink. ringSnapshot - // is a thread-confined field — no per-tick allocation. - ringSnapshot.clear(); + try { + while (running) { + // Snapshot the rings under the lock so we don't hold it through the + // (potentially slow) syscalls during creation/unlink. ringSnapshot + // is a thread-confined field — no per-tick allocation. + ringSnapshot.clear(); + synchronized (lock) { + for (int i = 0, n = rings.size(); i < n; i++) { + ringSnapshot.add(rings.getQuick(i)); + } + } + boolean hasMoreTrimmable = false; + for (int i = 0, n = ringSnapshot.size(); i < n; i++) { + if (!running) break; + if (serviceRing(ringSnapshot.getQuick(i))) { + hasMoreTrimmable = true; + } + } + // Drop strong refs so a deregistered ring becomes collectable + // before the next tick (otherwise the snapshot pins it for up + // to pollNanos after deregister). + ringSnapshot.clear(); + if (!running) break; + if (!hasMoreTrimmable) { + LockSupport.parkNanos(pollNanos); + } + } + } finally { + // If a timed-out close() abandoned the reap, it handed + // pathScratch ownership to this thread (see close()). Freeing it + // here reclaims the native buffer even when the worker outlives + // every close() attempt — nobody else retries manager cleanup. + ObjList cleanups; + Runnable ownedEngineCleanup; synchronized (lock) { - for (int i = 0, n = rings.size(); i < n; i++) { - ringSnapshot.add(rings.getQuick(i)); + workerLoopExited = true; + if (scratchHandedToWorker && !scratchFreed) { + scratchFreed = true; + pathScratch.close(); } + cleanups = exitCleanups; + exitCleanups = null; + ownedEngineCleanup = ownedEngineExitCleanup; + ownedEngineExitCleanup = null; } - for (int i = 0, n = ringSnapshot.size(); i < n; i++) { - if (!running) break; - serviceRing(ringSnapshot.getQuick(i)); + // Deferred engine cleanups run OUTSIDE + // `lock`: they perform syscalls (munmap, unlink, flock release) + // and must never execute under a lock that close()/register/ + // deregister callers contend on. Running them after the loop body + // is what makes the handoff safe: this thread can no longer touch + // any slot path. They must also never block on a caller-held + // monitor — a retried engine.close() joins this thread while + // holding the engine monitor, which is why the engine side uses a + // lock-free claim (CAS), not synchronization, for exactly-once. + if (ownedEngineCleanup != null) { + try { + ownedEngineCleanup.run(); + } catch (Throwable t) { + LOG.error("deferred owned-engine cleanup failed on manager-worker exit", t); + } + } + if (cleanups != null) { + for (int i = 0, n = cleanups.size(); i < n; i++) { + try { + cleanups.getQuick(i).run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed on manager-worker exit", t); + } + } } - // Drop strong refs so a deregistered ring becomes collectable - // before the next tick (otherwise the snapshot pins it for up - // to pollNanos after deregister). - ringSnapshot.clear(); - if (!running) break; - LockSupport.parkNanos(pollNanos); } } @@ -607,16 +1163,65 @@ private static final class RingEntry { // Survives across multiple serviceRing ticks and avoids a // write-storm when ackedFsn is steady. long lastPersistedAck = -1L; - // Guarded by SegmentManager.lock. A worker snapshot may retain this - // entry after deregister() removes it from rings; registered=false is - // the O(1) ownership check that prevents post-deregister writes through - // the engine-owned watermark, hot-spare installs, and accounting. - boolean registered = true; + // Prevents a legacy disk registration without a watermark from + // flooding the log on every manager tick. + boolean missingWatermarkLogged; + // Zero-allocation manager-thread-only retry state. The deadline uses + // the manager's monotonic clock and the delay doubles to a fixed cap. + long trimRetryAtNanos; + long trimRetryDelayNanos; + int trimRetryFailureKind; + // Updated lock-free by deferUntilRingQuiescent and pass completion. + // The field updater avoids one AtomicReference allocation per ring. + volatile Runnable quiescenceCleanup; + // Waiters mutate this under SegmentManager.lock; pass completion reads + // it before taking the otherwise-cold notification path. + volatile int quiescenceWaiters; + // REGISTERED -> IN_SERVICE -> REGISTERED on a normal pass. + // Deregistration changes either registered state to its corresponding + // deregistered state; completion then changes DEREGISTERED_IN_SERVICE + // to DEREGISTERED. The field updater avoids per-entry allocation. + volatile int state = ENTRY_REGISTERED; RingEntry(SegmentRing ring, String dir, AckWatermark watermark) { this.ring = ring; this.dir = dir; this.watermark = watermark; } + + void deregister() { + while (true) { + int current = state; + int next = current == ENTRY_IN_SERVICE + ? ENTRY_DEREGISTERED_IN_SERVICE + : ENTRY_DEREGISTERED; + if (current == ENTRY_DEREGISTERED || current == ENTRY_DEREGISTERED_IN_SERVICE + || ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + void finishService() { + while (true) { + int current = state; + int next = current == ENTRY_DEREGISTERED_IN_SERVICE + ? ENTRY_DEREGISTERED + : ENTRY_REGISTERED; + if (ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + boolean isInService() { + int current = state; + return current == ENTRY_IN_SERVICE || current == ENTRY_DEREGISTERED_IN_SERVICE; + } + + boolean isRegistered() { + int current = state; + return current == ENTRY_REGISTERED || current == ENTRY_IN_SERVICE; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c8516c4c..8b8e752c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; @@ -63,14 +64,22 @@ public final class SegmentRing implements QuietCloseable { /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */ public static final long PAYLOAD_TOO_LARGE = -2L; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); + // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only + // operation count for deterministic traversal-complexity assertions. + private static long nextSealedComparisons; // Tally of baseSeq comparisons performed by sortByBaseSeq across every // openExisting() recovery on this JVM. Used by SegmentRingTest to // assert the sort stays O(N log N) without relying on wall-clock time // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap - // in production: one volatile-free add per partition pass, dwarfed by - // the mmap I/O the recovery does on every segment. + // in production: one volatile-free add per partition pass (plus two per + // sift level in the rare heapsort fallback), dwarfed by the mmap I/O + // the recovery does on every segment. private static long sortComparisons; + // References copied while compacting the logical sealed-segment head. + // Test-only operation count for deterministic trim-complexity assertions. + private static long trimMovedReferences; private final long maxBytesPerSegment; + private final SfManifest manifest; // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time // (the segment manager). For now, both sides synchronize via the single- @@ -78,6 +87,9 @@ public final class SegmentRing implements QuietCloseable { // looks at sealedSegments after observing a higher ackedFsn, by which // point the producer thread's add to sealedSegments has retired. private final ObjList sealedSegments = new ObjList<>(); + // Logical head of sealedSegments. Head removal nulls one entry and advances + // this index; occasional compaction bounds unused prefix slots. + private int sealedHead; // High-water byte offset within the active segment at which we proactively // ask the segment manager to provision a spare (if one isn't already // installed). Computed once as 3/4 of segment capacity -- leaves the manager @@ -120,10 +132,15 @@ public final class SegmentRing implements QuietCloseable { * frameCount == 0); typically supplied by the segment manager at startup. */ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { + this(initialActive, maxBytesPerSegment, null); + } + + SegmentRing(MmapSegment initialActive, long maxBytesPerSegment, SfManifest manifest) { if (initialActive == null) { throw new IllegalArgumentException("initialActive must not be null"); } this.active = initialActive; + this.manifest = manifest; this.maxBytesPerSegment = maxBytesPerSegment; // 3/4 of capacity gives the manager a full quarter-segment of producer // runway before backpressure kicks in. Long math, no float, no alloc. @@ -133,190 +150,488 @@ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { } /** - * Recovers a ring from segments already on disk in {@code sfDir}. Used at - * sender startup when the user's previous session left durable but - * not-yet-acked frames behind. Walks every {@code *.sfa} file in the - * directory, opens each via {@link MmapSegment#openExisting}, and - * arranges them by baseSeq: - *

    - * Returns {@code null} if the directory is empty or contains no - * recognizable {@code .sfa} files -- the caller should then construct a - * fresh ring with {@link #SegmentRing(MmapSegment, long)} and a freshly - * created initial segment. - *

    - * Recovery is best-effort: a single bad-magic file is silently skipped - * (logged-then-ignored is the right call here; a stray unrelated file in - * the SF dir shouldn't take the whole sender down). A failure to open - * an otherwise-valid segment IS fatal -- the caller's data integrity - * depends on every segment being readable. + * Compatibility wrapper using the production facade. New startup code uses + * {@link #recover(FilesFacade, String, long)} so EMPTY is explicit. */ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) { - if (!Files.exists(sfDir)) { - return null; + return openExisting(FilesFacade.INSTANCE, sfDir, maxBytesPerSegment); + } + + /** + * Facade-aware variant of {@link #openExisting(String, long)}: every + * filesystem touch (enumeration, open, mmap, quarantine rename, manifest + * I/O) goes through {@code filesFacade} so recovery's fail-closed behavior + * can be fault-injected in tests. Returns {@code null} when the slot holds + * nothing recoverable; throws {@link MmapSegmentException} when the slot's + * state cannot be proven safe. + */ + public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + Recovery recovery = recover(filesFacade, sfDir, maxBytesPerSegment); + return recovery.status == RecoveryStatus.EMPTY ? null : recovery.ring; + } + + /** + * Exhaustively discovers and validates an SF slot without mutation. Only + * after enumeration, opens, CRC scans, contiguity and manifest boundaries + * all succeed does it migrate a legacy chain or discard validated spares. + */ + static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + if (!filesFacade.exists(sfDir)) { + return Recovery.empty(); } - ObjList opened = new ObjList<>(); - long find = Files.findFirst(sfDir); + ObjList names = new ObjList<>(); + long find = filesFacade.findFirst(sfDir); if (find < 0) { - LOG.warn("openExisting could not enumerate {} - treating as empty, " - + "but this may indicate a permission or transient error", sfDir); - return null; - } - if (find == 0) { - return null; + throw new MmapSegmentException("could not enumerate SF directory " + sfDir); } - // Outer try-catch: anything escaping the recovery body -- IOOBE from - // ObjList growth, OOM from native mmap during MmapSegment.openExisting, - // unforeseen RuntimeException from the contiguity check, etc. -- must - // not leave fds + mmaps owned by `opened` orphaned. Close every - // recovered segment and rethrow so the engine surfaces the failure. - try { + if (find > 0) { + int rc = 1; try { - int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); + String name = Files.utf8ToString(filesFacade.findName(find)); if (name != null && name.endsWith(".sfa")) { - String path = sfDir + "/" + name; - MmapSegment seg = null; - try { - seg = MmapSegment.openExisting(path); - // Filter out empty leftovers -- typically hot-spare - // segments the manager pre-allocated for a prior - // session that never got rotated into active. They - // carry the provisional baseSeq=0 and frameCount=0, - // which would otherwise collide with the real - // baseSeq=0 segment and trip the contiguity check - // below. No data to recover; close and unlink. - // Without the unlink the file persists across crash - // cycles and the disk leak compounds with every - // unclean shutdown. - // - // CAUTION: only unlink when the file is genuinely - // empty past the header. If frame[0] failed CRC - // (bit-rot, partial-page-write at crash, etc.) but - // valid frames followed, scanFrames returns - // lastGood=HEADER_SIZE and frameCount=0 -- yet - // tornTailBytes is non-zero. Treating that as - // "empty hot-spare" would silently destroy every - // surviving frame. Quarantine to .corrupt - // instead so a postmortem can recover what's left. - if (seg.frameCount() == 0) { - long torn = seg.tornTailBytes(); - seg.close(); - seg = null; - if (torn > 0) { - Files.rename(path, path + ".corrupt"); - } else { - Files.remove(path); - } - } else { - opened.add(seg); - seg = null; + names.add(name); + } + rc = filesFacade.findNext(find); + } + if (rc < 0) { + throw new MmapSegmentException("could not fully enumerate SF directory " + sfDir); + } + } finally { + filesFacade.findClose(find); + } + } + + ObjList all = new ObjList<>(); + // Files whose own bytes prove corruption (bad magic, sub-header size, + // negative baseSeq, unreadable header page). They are excluded from + // the chain and quarantined to .corrupt — but only AFTER the + // surviving chain validates (or resolves to EMPTY), so a failed + // recovery never mutates the slot. Whether a quarantined file was + // load-bearing is decided by the manifest-boundary / contiguity + // checks below, not by the skip itself. Operational open errors + // (EMFILE, EACCES, mmap rejection, unsupported version) are NOT in + // this bucket: they throw the plain MmapSegmentException type and + // abort recovery, because the underlying file may be perfectly + // intact and silently dropping it could lose durable frames. + ObjList corruptPaths = null; + SfManifest manifest = null; + try { + for (int i = 0, n = names.size(); i < n; i++) { + String path = sfDir + "/" + names.get(i); + try { + all.add(MmapSegment.openExisting(filesFacade, path)); + } catch (MmapSegmentCorruptionException e) { + LOG.warn("recovery: {} is not a readable SF segment; excluding it and " + + "deferring quarantine until the surviving chain validates -- {}", + path, e.toString()); + if (corruptPaths == null) { + corruptPaths = new ObjList<>(); + } + corruptPaths.add(path); + } catch (MmapSegmentException e) { + throw new MmapSegmentException("recovery failed for recognized segment " + path, e); + } + } + manifest = SfManifest.open(filesFacade, sfDir); + if (all.size() == 0) { + if (corruptPaths != null) { + // Nothing valid survived. With a manifest this is still a + // hole we can prove (boundaries reference segments that are + // now unreadable) -- fail without mutating. Without one, + // legacy semantics apply: quarantine and start fresh. + if (manifest != null) { + throw new MmapSegmentException("every SF segment in " + sfDir + + " is corrupt but " + SfManifest.FILE_NAME + + " references durable data"); + } + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); + } + if (manifest != null) { + // No .sfa files at all. Two legitimate protocols produce + // this: the close-time drain unlinks the last segment + // before it removes the manifest, and a fresh-start crash + // can leave a boundary-less (0,0) manifest behind. In + // both cases nothing recoverable exists, so accept EMPTY + // -- but shout, because a manual wipe of segment files + // looks identical and the operator should know. + LOG.warn("SF manifest exists in {} with no segment files " + + "(clean-drain or fresh-start crash window, or manual " + + "segment removal); discarding it and starting fresh", sfDir); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + } + return Recovery.empty(); + } + + ObjList data = new ObjList<>(); + boolean requiresManifest = false; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + requiresManifest |= segment.manifestRequired(); + if (segment.frameCount() > 0) { + data.add(segment); + } + } + sortByBaseSeq(data, 0, data.size()); + if (manifest == null && requiresManifest) { + throw new MmapSegmentException("new-format SF segment exists but " + + SfManifest.FILE_NAME + " is missing"); + } + + MmapSegment active; + ObjList chain = new ObjList<>(); + long headBase; + long activeBase; + if (manifest == null) { + if (data.size() > 0) { + validateContiguous(data); + for (int i = 0, n = data.size(); i < n; i++) { + chain.add(data.get(i)); + } + active = chain.get(chain.size() - 1); + headBase = chain.get(0).baseSeq(); + activeBase = active.baseSeq(); + } else { + active = chooseEmptyInitial(all, sfDir); + if (active == null) { + // Legacy slot holding only empty leftovers, every one + // of them torn (a clean empty would have been chosen). + // Nothing recoverable: quarantine the torn evidence, + // drop the clean debris, start fresh -- exactly the + // pre-manifest behavior. + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove empty SF leftover {}", path); } - } catch (MmapSegmentException t) { - // Per-file data error (bad magic, bad header, - // unsupported version, mmap rejection on this one - // file). Don't take down recovery for one corrupt - // .sfa -- log and skip so siblings still recover. - // Resource exhaustion (OOM) and programmer errors - // (IOOBE) deliberately propagate to the outer - // catch, which closes every already-recovered - // segment and rethrows: continuing the loop after - // an OOM would just fail again on the next file - // while silently leaking the segments we managed - // to recover before it. - LOG.warn("openExisting: skipping {} -- {}", path, t.toString()); - } finally { - // Close any seg whose ownership wasn't transferred - // (either to opened, or via the empty-branch close - // above). Fires on a propagating throw between - // open and transfer -- most importantly an OOM - // from ObjList.add growing its backing array - // after the mmap+fd were already acquired. - if (seg != null) { - try { - seg.close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing in-flight segment {}", - path, closeErr); - } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); + } + chain.add(active); + headBase = active.baseSeq(); + activeBase = active.baseSeq(); + } + manifest = SfManifest.create(filesFacade, sfDir, headBase, activeBase); + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); + } + } else { + headBase = manifest.headBase(); + activeBase = manifest.activeBase(); + for (int i = 0, n = data.size(); i < n; i++) { + MmapSegment segment = data.get(i); + long end = segment.baseSeq() + segment.frameCount(); + if (segment.baseSeq() < headBase) { + if (end > headBase) { + throw new MmapSegmentException("segment overlaps committed SF head boundary"); + } + continue; // acknowledged stale file after manifest-before-unlink crash + } + if (segment.baseSeq() > activeBase) { + throw new MmapSegmentException("segment exists beyond committed SF active boundary"); + } + chain.add(segment); + } + if (chain.size() > 0) { + validateContiguous(chain); + if (chain.get(0).baseSeq() != headBase) { + throw new MmapSegmentException("missing expected SF head segment at base " + headBase); + } + } + active = findActive(all, activeBase); + if (active == null) { + if (chain.size() == 0 && headBase == activeBase && corruptPaths == null) { + // Clean-drain crash window: the close-time drain first + // durably collapses the boundaries to head == active + // (declaring every frame acked), then unlinks segments + // in ascending order -- so dying between the active's + // unlink and the spare's/manifest's leaves exactly + // this state: no data frame anywhere, no file at the + // committed active base, only empty spares and/or + // acked stale files. Nothing recoverable exists; + // accept EMPTY and clear the debris. Guarded on + // corruptPaths because an unreadable .sfa of unknown + // identity could be the real active -- in that case + // fail closed instead of guessing. + LOG.warn("SF manifest in {} has collapsed boundaries ({}) with no " + + "segment at the active base and no recovered frames; " + + "accepting the clean-drain crash window as empty", + sfDir, activeBase); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove drained SF leftover {}", path); } } + all.clear(); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + return Recovery.empty(); } - rc = Files.findNext(find); + throw new MmapSegmentException("missing expected SF active segment at base " + activeBase); } - } finally { - Files.findClose(find); - } - if (opened.size() == 0) { - return null; - } - // Sort by baseSeq ascending. Worst-case segment count is - // sf_max_total_bytes / sf_max_bytes -- at the documented ceiling - // (1 TiB / 64 MiB) that is ~16K entries, where an O(N²) sort spends - // multiple seconds in compares + shifts before the I/O thread can - // start. In-place quicksort with median-of-three pivot keeps the - // no-allocation discipline of the surrounding code; median-of-three - // is required because readdir on many filesystems returns entries - // in lexicographic (== baseSeq-hex) order and a naive first-element - // pivot would degrade back to O(N²) on exactly that common case. - sortByBaseSeq(opened, 0, opened.size()); - // Sanity: the recovered segments must form a contiguous FSN range. - // Detect gaps so they don't silently produce duplicate or missing - // FSNs after recovery. A gap means a segment went missing (a - // manual deletion) or a sealed segment under-recovered -- its tail - // was cut short by a sparse/unbacked page or a mid-file media error - // (bad sector), the same class of fault scanFrames tolerates on the - // active segment but which corrupts the range on a sealed one. - for (int i = 1, n = opened.size(); i < n; i++) { - MmapSegment prev = opened.get(i - 1); - MmapSegment curr = opened.get(i); - long expected = prev.baseSeq() + prev.frameCount(); - if (curr.baseSeq() != expected) { - throw new MmapSegmentException( - "FSN gap in recovered segments: prev baseSeq=" + prev.baseSeq() - + " frameCount=" + prev.frameCount() - + " expected next baseSeq=" + expected - + " but got " + curr.baseSeq() - + " -- a segment was deleted, or a sealed segment's tail was" - + " truncated (sparse/unbacked page or disk media error);" - + " check disk health"); + if (chain.size() == 0) { + if (headBase != activeBase || active.frameCount() != 0 || corruptPaths != null) { + // corruptPaths guard: with an unreadable .sfa in the + // slot, the innocent-looking empty at the active base + // could be a leftover spare coincidentally carrying + // the same provisional baseSeq as a corrupted real + // active -- accepting it would quarantine unacked + // frames and re-issue their FSNs. Fail closed. + throw new MmapSegmentException( + "missing SF chain between committed boundaries" + + (corruptPaths != null + ? " (a corrupt segment prevents proving the empty state)" : "")); + } + chain.add(active); + } else if (chain.get(chain.size() - 1) != active) { + MmapSegment tail = chain.get(chain.size() - 1); + long chainEnd = tail.baseSeq() + tail.frameCount(); + if (corruptPaths == null && active.frameCount() == 0 && active.baseSeq() == chainEnd) { + // Rotation committed (manifest fsync'd, promoted spare's + // header synced) but the process/OS died before a single + // frame of the new active reached disk: the sealed chain + // ends exactly where the empty active begins. Legal + // crash state -- resume appending into it. Refused when + // corrupt segments exist (same stand-in hazard as the + // empty-chain acceptance above). + chain.add(active); + } else { + throw new MmapSegmentException( + "missing expected SF active/tail segment at base " + activeBase); + } + } + for (int i = 0, n = chain.size() - 1; i < n; i++) { + if (chain.get(i).tornTailBytes() > 0) { + throw new MmapSegmentException("corrupt torn tail in sealed SF segment " + chain.get(i).path()); + } + } + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); } } - // The newest segment becomes the active. Even if it's full, that's OK: - // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager - // installs a hot spare, the producer rotates. Same fast path as a - // mid-life ring. - int last = opened.size() - 1; - MmapSegment active = opened.get(last); - opened.remove(last); - SegmentRing ring = new SegmentRing(active, maxBytesPerSegment); - // Older segments become sealed in baseSeq order. - for (int i = 0, n = opened.size(); i < n; i++) { - ring.sealedSegments.add(opened.get(i)); + + for (int i = 1, n = chain.size(); i < n; i++) { + chain.get(i - 1).linkSuccessor(chain.get(i)); } - return ring; + SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest); + manifest = null; + for (int i = 0, n = chain.size() - 1; i < n; i++) { + ring.sealedSegments.add(chain.get(i)); + } + // Ownership of the chain transferred. Clean up only validated + // extras; recovery is already successful, so cleanup failure + // must never turn startup into a partially-mutating error or + // orphan the constructed ring -- swallow and let the next + // startup re-examine the leftovers. Extras with a torn tail + // carry evidence of attempted writes -- keep the bytes under a + // .corrupt name instead of unlinking them. + try { + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (!containsIdentity(chain, segment)) { + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove validated stale/empty SF segment {}", path); + } + } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + } catch (Throwable cleanupError) { + LOG.warn("post-recovery cleanup failed; leftover files will be " + + "re-examined on the next startup", cleanupError); + } + return Recovery.recovered(ring); } catch (Throwable t) { - // Close every recovered MmapSegment that's still in `opened`. - // After the success path, `opened` no longer contains the active - // segment (removed above), but the sealed segments transferred to - // ring.sealedSegments are still owned by the ring once it's - // returned -- so this catch only fires before the return statement. - for (int i = 0, n = opened.size(); i < n; i++) { + for (int i = 0, n = all.size(); i < n; i++) { try { - opened.get(i).close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing recovered segment during cleanup", - closeErr); + all.get(i).close(); + } catch (Throwable closeError) { + LOG.warn("error closing SF segment after recovery failure", closeError); } } + if (manifest != null) { + manifest.close(); + } throw t; } } + /** + * Durably advances the manifest head past {@code trimming} (the sealed + * segment the manager is about to unlink). The successor and the current + * active are both read under the ring monitor, so a concurrent rotation + * (which also mutates the manifest under this monitor) can never make the + * head leapfrog a still-live sealed segment: if rotation sealed the old + * active after the caller's snapshot, {@code trimming.successor()} now + * points at that sealed segment, not at the new active. + */ + synchronized void advanceManifestHeadPast(MmapSegment trimming) { + if (manifest == null) { + return; + } + MmapSegment successor = trimming.successor(); + long newHeadBase = (successor == null || successor == active) + ? active.baseSeq() + : successor.baseSeq(); + manifest.update(newHeadBase, active.baseSeq()); + } + + /** + * Picks the clean (untorn) empty segment to reuse as a legacy slot's + * initial active, preferring {@code sf-initial.sfa}. Returns {@code null} + * when no clean empty exists; torn empties are never reused here because + * their bytes are quarantine evidence, not blank space. + */ + private static MmapSegment chooseEmptyInitial(ObjList all, String sfDir) { + String initialPath = sfDir + "/sf-initial.sfa"; + MmapSegment selected = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.frameCount() != 0 || segment.tornTailBytes() > 0) { + continue; + } + if (selected == null || initialPath.equals(segment.path())) { + selected = segment; + } + } + return selected; + } + + /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */ + private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) { + if (corruptPaths == null) { + return; + } + for (int i = 0, n = corruptPaths.size(); i < n; i++) { + quarantineFile(filesFacade, corruptPaths.get(i)); + } + } + + private static void quarantineFile(FilesFacade filesFacade, String path) { + if (filesFacade.rename(path, path + ".corrupt") != 0) { + LOG.warn("could not quarantine {} to {}.corrupt; it will be re-examined " + + "on the next recovery", path, path); + } + } + + private static boolean containsIdentity(ObjList list, MmapSegment value) { + for (int i = 0, n = list.size(); i < n; i++) { + if (list.get(i) == value) return true; + } + return false; + } + + /** + * Locates the segment the manifest's {@code activeBase} refers to. + * Preference order among same-base candidates: + *

      + *
    1. a segment with recovered frames (the durable chain tail);
      2. + *
    2. an empty segment with a torn tail (the promoted active whose + * first frame write was cut short — an attempted write marks it as + * the one rotation actually exposed);
      3. + *
    3. a clean empty segment.
      4. + *
    + * Multiple equivalent empties at the same base are NOT an error: a fresh + * start or a rotation crash routinely leaves both the initial/promoted + * segment and a provisioned hot spare carrying the same provisional + * baseSeq. They are interchangeable blanks — pick one deterministically + * and let the extras cleanup discard the rest. Bricking startup on this + * state would turn every "kill -9 shortly after start" into a manual + * repair. + */ + private static MmapSegment findActive(ObjList all, long activeBase) { + MmapSegment tornEmpty = null; + MmapSegment cleanEmpty = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.baseSeq() != activeBase) { + continue; + } + if (segment.frameCount() > 0) { + return segment; + } + if (segment.tornTailBytes() > 0) { + if (tornEmpty == null) { + tornEmpty = segment; + } + } else if (cleanEmpty == null) { + cleanEmpty = segment; + } + } + return tornEmpty != null ? tornEmpty : cleanEmpty; + } + + private static void validateContiguous(ObjList segments) { + for (int i = 1, n = segments.size(); i < n; i++) { + MmapSegment previous = segments.get(i - 1); + MmapSegment current = segments.get(i); + long expected = previous.baseSeq() + previous.frameCount(); + if (current.baseSeq() != expected) { + throw new MmapSegmentException("FSN gap in recovered segments: expected " + + expected + " but got " + current.baseSeq()); + } + } + } + + static final class Recovery { + private final SegmentRing ring; + private final RecoveryStatus status; + + private Recovery(RecoveryStatus status, SegmentRing ring) { + this.status = status; + this.ring = ring; + } + + static Recovery empty() { + return new Recovery(RecoveryStatus.EMPTY, null); + } + + SegmentRing ring() { + return ring; + } + + static Recovery recovered(SegmentRing ring) { + return new Recovery(RecoveryStatus.RECOVERED, ring); + } + + RecoveryStatus status() { + return status; + } + } + + enum RecoveryStatus { + EMPTY, + RECOVERED + } + /** * Highest FSN that the server has ACK'd. Read by the segment manager to * decide which sealed segments are safe to munmap + unlink. @@ -379,15 +694,52 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { // full, so its frameCount is stable, and (b) the spare hasn't been // appended to yet (rebaseSeq enforces that). The segment manager's // earlier guess at baseSeq is irrelevant. - long actualBase = active.baseSeq() + active.frameCount(); + MmapSegment previous = active; + long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); - // Mutate sealedSegments under the same monitor used by - // snapshotSealedSegments -- the I/O thread reads through that - // path and must not see a half-resized ObjList. + if (manifest != null) { + // Make the spare's rebased identity durable BEFORE the manifest + // references it. Without this barrier an OS crash could leave a + // durable manifest pointing at baseSeq=actualBase while the + // spare's on-disk header still carries the manager's + // provisional guess -- recovery would then find no segment at + // the committed active boundary and fail a startup that lost + // nothing. One msync per rotation, amortized over a whole + // segment of appends; runs outside the monitor because the + // spare is not yet visible to any other thread. + // + // Deliberately NOT msync'd here: the sealed predecessor's + // data pages. A power loss can therefore tear the sealed + // tail after the boundary is committed, and recovery will + // fail closed on chainEnd != activeBase. That is the + // intended semantics -- page-level durability of frame data + // follows the sender's opt-in msync cadence, and recovery + // must refuse to guess when the two disagree. + spare.syncHeader(); + } + // Publish the successor before the volatile active promotion. The + // same monitor protects the sealed list and nextSealedAfter's trim + // fallback, while the volatile link also remains readable from a + // current segment after the manager removes and closes it. synchronized (this) { - sealedSegments.add(active); + if (manifest != null) { + // Inside the monitor: serialized with the trim path's + // advanceManifestHeadPast so neither writer publishes a + // boundary computed from a state the other has already + // moved past (SfManifest additionally clamps monotonic). + // BEFORE any ring mutation: if the manifest fsync throws, + // the rotation never happened -- previous stays active, + // the spare stays installed, and the producer's retry + // re-runs this block from a consistent state. + long headBase = sealedHead < sealedSegments.size() + ? sealedSegments.get(sealedHead).baseSeq() + : previous.baseSeq(); + manifest.update(headBase, actualBase); + } + previous.linkSuccessor(spare); + sealedSegments.add(previous); + active = spare; } - active = spare; hotSpare = null; // Fresh active just consumed the spare → ask the manager to start // making the next one immediately, before this segment fills. @@ -439,13 +791,14 @@ public synchronized void close() { hotSpare.close(); hotSpare = null; } - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) { - s.close(); - } + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + sealedSegments.get(i).close(); } sealedSegments.clear(); + sealedHead = 0; + if (manifest != null) { + manifest.close(); + } } /** @@ -461,10 +814,8 @@ public synchronized ObjList drainTrimmable() { ObjList out = null; // Sealed segments are in baseSeq order, oldest first; once we hit one // that isn't fully acked, none of the later ones can be either. - // Synchronized so the I/O thread's snapshotSealedSegments() can't - // race against the remove(0) shuffling slots underneath it. - while (sealedSegments.size() > 0) { - MmapSegment s = sealedSegments.get(0); + while (sealedHead < sealedSegments.size()) { + MmapSegment s = sealedSegments.get(sealedHead); long lastSeq = s.baseSeq() + s.frameCount() - 1; if (lastSeq > acked) { break; @@ -473,7 +824,7 @@ public synchronized ObjList drainTrimmable() { out = new ObjList<>(); } out.add(s); - sealedSegments.remove(0); + removeSealedHead(); } return out; } @@ -490,7 +841,7 @@ public synchronized ObjList drainTrimmable() { * scan cost doesn't matter. */ public synchronized MmapSegment findSegmentContaining(long fsn) { - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { MmapSegment s = sealedSegments.get(i); long base = s.baseSeq(); if (fsn >= base && fsn < base + s.frameCount()) { @@ -513,7 +864,22 @@ public synchronized MmapSegment findSegmentContaining(long fsn) { * fallback -- see {@link #nextSealedAfter(MmapSegment)}. */ public synchronized MmapSegment firstSealed() { - return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; + return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; + } + + /** + * Returns the oldest fully acknowledged sealed segment without removing + * it. The segment manager keeps it owned by the ring until close + unlink + * succeeds, so a failed unlink cannot make the path disappear from live + * bookkeeping or allow its identifier to be reused. + */ + public synchronized MmapSegment firstTrimmable() { + if (sealedHead == sealedSegments.size()) { + return null; + } + MmapSegment segment = sealedSegments.get(sealedHead); + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + return lastSeq <= ackedFsn ? segment : null; } /** Active segment -- exposed for the I/O thread's "send next batch" path. */ @@ -536,7 +902,7 @@ public synchronized MmapSegment firstSealed() { */ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) { long best = -1L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen); if (fsn > best) { best = fsn; @@ -556,7 +922,8 @@ public MmapSegment getActive() { * concurrent rotation. Cross-thread readers (typically the I/O loop) * should use {@link #snapshotSealedSegments(MmapSegment[])} instead. */ - public ObjList getSealedSegments() { + public synchronized ObjList getSealedSegments() { + compactSealedSegments(); return sealedSegments; } @@ -600,21 +967,28 @@ public boolean needsHotSpare() { * outpaces the I/O thread and sealed segments accumulate well beyond * any reasonable snapshot-array size. *

    - * Identity match is intentionally avoided: we compare {@code baseSeq} - * so the loop is robust against the case where {@code current} was - * trimmed out from under us (already ACK'd before the I/O thread - * advanced) -- we still return the next segment in baseSeq order rather - * than failing. Synchronized against rotation. + * Each segment publishes its successor once, before rotation exposes that + * successor as active. A constant-time head check detects when trimming + * removed the immediate successor and falls forward to the oldest live + * sealed segment. Synchronized against rotation and head removal. */ public synchronized MmapSegment nextSealedAfter(MmapSegment current) { - long currentBase = current.baseSeq(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s.baseSeq() > currentBase) { - return s; - } + nextSealedComparisons++; + MmapSegment successor = current.successor(); + if (successor == null) { + return null; } - return null; + if (successor == active) { + return null; + } + MmapSegment first = sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; + if (first != null && successor.baseSeq() >= first.baseSeq()) { + return successor; + } + // Head trimming may have removed the immediate successor while the + // I/O cursor still held an older segment. Trims only remove a prefix, + // so the current head is the first live segment after that prefix. + return first; } /** @@ -634,6 +1008,22 @@ public long publishedFsn() { return publishedFsn; } + /** + * Commits removal of the segment returned by {@link #firstTrimmable()}. + * Returns false if concurrent lifecycle activity changed the head. + */ + public synchronized boolean removeTrimmable(MmapSegment segment) { + if (sealedHead == sealedSegments.size() || sealedSegments.get(sealedHead) != segment) { + return false; + } + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + if (lastSeq > ackedFsn) { + return false; + } + removeSealedHead(); + return true; + } + /** * Registers a wakeup callback that the producer thread will invoke when * a hot spare is needed -- either right after a rotation has consumed the @@ -663,17 +1053,12 @@ public void setManagerWakeup(Runnable wakeup) { * the I/O loop is about to do. */ public synchronized int snapshotSealedSegments(MmapSegment[] target) { - int n = sealedSegments.size(); - if (n > target.length) { - for (int i = 0; i < target.length; i++) { - target[i] = sealedSegments.get(i); - } - return -1; - } - for (int i = 0; i < n; i++) { - target[i] = sealedSegments.get(i); + int n = sealedSegments.size() - sealedHead; + int copyCount = Math.min(n, target.length); + for (int i = 0; i < copyCount; i++) { + target[i] = sealedSegments.get(sealedHead + i); } - return n; + return n > target.length ? -1 : n; } /** @@ -689,21 +1074,47 @@ public synchronized long totalSegmentBytes() { if (a != null) total += a.sizeBytes(); MmapSegment hs = hotSpare; if (hs != null) total += hs.sizeBytes(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) total += s.sizeBytes(); + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + total += sealedSegments.get(i).sizeBytes(); } return total; } + private void compactSealedSegments() { + if (sealedHead > 0) { + int liveCount = sealedSegments.size() - sealedHead; + trimMovedReferences += liveCount; + sealedSegments.remove(0, sealedHead - 1); + sealedHead = 0; + } + } + + private void removeSealedHead() { + sealedSegments.setQuick(sealedHead++, null); + int size = sealedSegments.size(); + if (sealedHead == size) { + sealedSegments.clear(); + sealedHead = 0; + } else if (sealedHead >= 64 && sealedHead >= size - sealedHead) { + compactSealedSegments(); + } + } + + /** Returns the sealed-list operation count used by traversal tests. */ + @TestOnly + public static long getNextSealedComparisons() { + return nextSealedComparisons; + } + /** * Returns the cumulative count of baseSeq comparisons performed by * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} * (or process start). The count is incremented once per partition pass * for the median-of-three pivot pick plus once per element compared - * against the pivot, so a clean run on N segments adds roughly - * {@code 3 + (hi - lo - 1)} per recursive frame, summing to O(N log N). - * Exposed for {@code SegmentRingTest} to detect O(N²) regressions + * against the pivot ({@code 3 + (hi - lo - 1)} per pass), and by two per + * sift-down level when a range falls back to heapsort, so it strictly + * upper-bounds the true compare count and sums to O(N log N) on every + * input. Exposed for {@code SegmentRingTest} to detect O(N²) regressions * deterministically. */ @TestOnly @@ -711,22 +1122,71 @@ public static long getSortComparisons() { return sortComparisons; } + /** Returns the references moved by sealed-list compaction. */ + @TestOnly + public static long getTrimMovedReferences() { + return trimMovedReferences; + } + + /** Zeroes the counter exposed via {@link #getNextSealedComparisons()}. */ + @TestOnly + public static void resetNextSealedComparisons() { + nextSealedComparisons = 0; + } + /** Zeroes the counter exposed via {@link #getSortComparisons()}. */ @TestOnly public static void resetSortComparisons() { sortComparisons = 0; } + /** Zeroes the counter exposed via {@link #getTrimMovedReferences()}. */ + @TestOnly + public static void resetTrimMovedReferences() { + trimMovedReferences = 0; + } + + /** + * Drives the recovery-time baseSeq sort over the whole list. Exposed so + * {@code SegmentRingTest} can feed adversarial orders (organ-pipe, mass + * duplicates, median-of-three killer, unsigned-boundary keys) straight + * into the sort and assert comparison bounds without staging thousands + * of segment files on disk. + */ + @TestOnly + public static void sortByBaseSeqForTest(ObjList list) { + sortByBaseSeq(list, 0, list.size()); + } + /** - * In-place quicksort over {@code list[lo, hi)} keyed by ascending - * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) - * on already-sorted input that lexicographic readdir produces (our - * filenames are zero-padded hex of {@code baseSeq}). Recursion depth is - * bounded by ~2 log₂(N) -- for the documented 16K-segment ceiling, well - * under the JVM default stack. + * In-place introsort over {@code list[lo, hi)} keyed by ascending + * unsigned {@code baseSeq}. Median-of-three quicksort handles the readdir + * orders a healthy slot produces (lexicographic enumeration of the + * generation-numbered filenames yields already-sorted baseSeqs; hashed + * directory order is effectively random), and a partition-pass budget of + * 2·⌊log₂(N)⌋ demotes any range that keeps splitting badly to in-place + * heapsort. Without that budget, Lomuto with a median-of-three pivot is + * O(N²) on organ-pipe, duplicate-heavy and median-of-three-killer orders + * -- reachable only through corrupted-yet-parseable or operator-copied + * headers, but at the documented 16K-segment ceiling that is 10⁷..10⁸ + * comparisons of startup stall before recovery validation gets to + * reject the slot, so the fallback makes O(N log N) unconditional. + * Recursion depth stays under log₂(N) (recurse on the smaller side, + * loop on the larger), well within the JVM default stack. */ private static void sortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + if (n > 1) { + sortByBaseSeq(list, lo, hi, 2 * (31 - Integer.numberOfLeadingZeros(n))); + } + } + + private static void sortByBaseSeq(ObjList list, int lo, int hi, int budget) { while (hi - lo > 1) { + if (budget-- == 0) { + heapSortByBaseSeq(list, lo, hi); + return; + } int mid = (lo + hi) >>> 1; long a = list.get(lo).baseSeq(); long b = list.get(mid).baseSeq(); @@ -756,17 +1216,61 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) { } swap(list, store, hi - 1); // Recurse on the smaller partition; loop on the larger to keep - // recursion depth bounded by log₂(N). + // recursion depth bounded by log₂(N). Children inherit the + // remaining pass budget: it counts passes along a root-to-leaf + // path, so a chain of bad splits exhausts it after ~2 log₂(N) + // levels no matter how the work is divided. if (store - lo < hi - store - 1) { - sortByBaseSeq(list, lo, store); + sortByBaseSeq(list, lo, store, budget); lo = store + 1; } else { - sortByBaseSeq(list, store + 1, hi); + sortByBaseSeq(list, store + 1, hi, budget); hi = store; } } } + /** + * In-place heapsort over {@code list[lo, hi)} keyed by ascending unsigned + * {@code baseSeq}: the introsort fallback for ranges whose partition-pass + * budget ran out. Guaranteed O(N log N) for any key distribution and any + * initial order; no allocation. + */ + private static void heapSortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + for (int root = (n >>> 1) - 1; root >= 0; root--) { + siftDownByBaseSeq(list, lo, root, n); + } + for (int end = n - 1; end > 0; end--) { + swap(list, lo, lo + end); + siftDownByBaseSeq(list, lo, 0, end); + } + } + + private static void siftDownByBaseSeq(ObjList list, int lo, int root, int heapSize) { + while (true) { + int child = (root << 1) + 1; + if (child >= heapSize) { + return; + } + // At most two unsigned compares per level (sibling pick + parent + // test); bump the counter by the constant 2 up front -- same + // cheap-upper-bound convention as the partition pass. + sortComparisons += 2; + if (child + 1 < heapSize + && Long.compareUnsigned(list.get(lo + child).baseSeq(), + list.get(lo + child + 1).baseSeq()) < 0) { + child++; + } + if (Long.compareUnsigned(list.get(lo + root).baseSeq(), + list.get(lo + child).baseSeq()) >= 0) { + return; + } + swap(list, lo + root, lo + child); + root = child; + } + } + private static void swap(ObjList list, int i, int j) { if (i == j) return; MmapSegment tmp = list.get(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java index 2dec7668..541f7dcf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,6 +147,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Non-blocking enqueue. Always admits the new event unless the dispatcher * is closed or {@code event} is null. Returns {@code true} when the new diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java index aaa0e4f9..fb796712 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderError; import io.questdb.client.SenderErrorHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -209,6 +210,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * True if at least one error has been delivered to a user-installed * (non-default) handler since this dispatcher started. Used by diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java index 2c0e90a1..9e01a630 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java @@ -26,6 +26,7 @@ import io.questdb.client.SenderProgressHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,6 +173,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Replace the user-supplied handler. Effective immediately for any * subsequent delivery. Pass {@code null} to install the no-op default. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java new file mode 100644 index 00000000..f09ce94a --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -0,0 +1,282 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Crash-safe boundary record for an SF segment chain. Two fixed-size, + * independently CRC-protected records alternate on update. Recovery selects + * the valid record with the greatest generation, so a torn update cannot + * erase the previous committed head/active boundary. + */ +final class SfManifest implements QuietCloseable { + static final String FILE_NAME = "sf-manifest.bin"; + private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class); + private static final int CRC_OFFSET = 60; + private static final long FILE_SIZE = 128; + private static final int MAGIC = 0x314d4653; // SFM1 little-endian + private static final int RECORD_SIZE = 64; + private static final int VERSION = 1; + private final int fd; + private final FilesFacade filesFacade; + private final String path; + private long activeBase; + private boolean closed; + private long generation; + private long headBase; + + private SfManifest(FilesFacade filesFacade, String path, int fd, + long generation, long headBase, long activeBase) { + this.filesFacade = filesFacade; + this.path = path; + this.fd = fd; + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + + static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { + String path = dir + "/" + FILE_NAME; + int fd = filesFacade.openRWExclusive(path); + if (fd < 0) { + throw new MmapSegmentException("exclusive create failed for SF manifest " + path); + } + boolean success = false; + try { + if (!filesFacade.allocate(fd, FILE_SIZE)) { + throw new MmapSegmentException("could not allocate SF manifest " + path); + } + SfManifest manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1); + manifest.update(headBase, activeBase); + if (filesFacade.fsyncDir(dir) != 0) { + throw new MmapSegmentException("could not sync SF manifest directory " + dir); + } + success = true; + return manifest; + } finally { + if (!success) { + filesFacade.close(fd); + filesFacade.remove(path); + } + } + } + + static SfManifest open(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + if (!filesFacade.exists(path)) { + return null; + } + if (filesFacade.length(path) != FILE_SIZE) { + // A wrong-sized manifest is creation debris: create() reaches the + // full FILE_SIZE via allocate() before writing the first record, + // so a mis-sized file proves the crash happened before any + // boundary was ever committed — nothing can depend on it yet + // (segment flags are stamped only after create() returns). Treat + // as absent so startup self-heals; genuine post-creation loss is + // still caught by the manifest-required flag check. + quarantineDebris(filesFacade, path, "wrong size " + filesFacade.length(path)); + return null; + } + int fd = filesFacade.openRW(path); + if (fd < 0) { + throw new MmapSegmentException("could not open SF manifest " + path); + } + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Record first = readRecord(filesFacade, fd, buffer, 0); + Record second = readRecord(filesFacade, fd, buffer, RECORD_SIZE); + Record selected; + if (first == null) { + selected = second; + } else if (second == null || first.generation > second.generation) { + selected = first; + } else { + selected = second; + } + if (selected == null) { + // No valid record in either slot. create() makes the first + // record durable (write + fsync) before returning, and every + // later update() rewrites only ONE slot — so a torn update + // leaves the sibling record intact. Zero valid records + // therefore proves a creation crash, not boundary loss. + // Self-heal by treating the file as absent. If durable state + // DID depend on a manifest (flags stamped, i.e. double-slot + // bit rot), recovery still fails closed on the + // manifest-required flag check. + filesFacade.close(fd); + quarantineDebris(filesFacade, path, "no valid CRC-protected record"); + return null; + } + return new SfManifest(filesFacade, path, fd, selected.generation, + selected.headBase, selected.activeBase); + } catch (Throwable t) { + filesFacade.close(fd); + throw t; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + long activeBase() { + return activeBase; + } + + @Override + public void close() { + if (!closed) { + closed = true; + filesFacade.close(fd); + } + } + + long headBase() { + return headBase; + } + + /** + * Unlinks {@code dir}'s manifest file. Used when a slot is being reset to + * the "nothing durable" state (fresh-start cleanup, close-time drain, or + * recovery accepting a segment-less slot as empty). Returns {@code true} + * when the file is confirmed gone (removed, or never existed). + */ + static boolean removeFile(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + return filesFacade.remove(path) || !filesFacade.exists(path); + } + + synchronized void update(long newHeadBase, long newActiveBase) { + if (closed) { + throw new IllegalStateException("SF manifest is closed"); + } + // Committed boundaries only ever move forward: head advances on trim, + // active advances on rotation. Clamp instead of throwing because the + // two writers (producer rotation, manager trim) are serialized on the + // ring monitor but may each compute their argument from a snapshot + // the other has already moved past — e.g. rotation reads the sealed + // list while a trimmed-but-not-yet-removed head segment still sits in + // it. Regressing a durable boundary would let a later crash-recovery + // demand a segment file the trim path already unlinked (startup would + // fail on "missing head segment") or, worse, re-expose stale files + // below an already-committed head. + if (generation > 0) { + if (newHeadBase < headBase) { + newHeadBase = headBase; + } + if (newActiveBase < activeBase) { + newActiveBase = activeBase; + } + } + if (newHeadBase < 0 || newActiveBase < newHeadBase) { + throw new IllegalArgumentException("invalid SF manifest boundaries"); + } + if (generation > 0 && headBase == newHeadBase && activeBase == newActiveBase) { + return; + } + long nextGeneration = generation + 1; + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(buffer, MAGIC); + Unsafe.getUnsafe().putInt(buffer + 4, VERSION); + Unsafe.getUnsafe().putLong(buffer + 8, nextGeneration); + Unsafe.getUnsafe().putLong(buffer + 16, newHeadBase); + Unsafe.getUnsafe().putLong(buffer + 24, newActiveBase); + int crc = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + Unsafe.getUnsafe().putInt(buffer + CRC_OFFSET, crc); + long offset = (nextGeneration & 1L) * RECORD_SIZE; + if (filesFacade.write(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + throw new MmapSegmentException("short write updating SF manifest " + path); + } + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync SF manifest " + path); + } + generation = nextGeneration; + headBase = newHeadBase; + activeBase = newActiveBase; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + /** + * Moves creation-crash debris aside so a subsequent exclusive create can + * succeed. Prefers rename (keeps the bytes for postmortem); falls back to + * remove; throws when neither works — leaving the debris in place would + * wedge every subsequent {@link #create}. + */ + private static void quarantineDebris(FilesFacade filesFacade, String path, String reason) { + LOG.warn("SF manifest {} is creation-crash debris ({}); quarantining and starting " + + "from the segment files", path, reason); + if (filesFacade.rename(path, path + ".corrupt") == 0) { + return; + } + if (filesFacade.remove(path)) { + return; + } + throw new MmapSegmentException("could not quarantine invalid SF manifest " + path); + } + + private static Record readRecord(FilesFacade filesFacade, int fd, long buffer, long offset) { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + if (filesFacade.read(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + return null; + } + if (Unsafe.getUnsafe().getInt(buffer) != MAGIC + || Unsafe.getUnsafe().getInt(buffer + 4) != VERSION) { + return null; + } + int expected = Unsafe.getUnsafe().getInt(buffer + CRC_OFFSET); + int actual = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + if (expected != actual) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(buffer + 8); + long headBase = Unsafe.getUnsafe().getLong(buffer + 16); + long activeBase = Unsafe.getUnsafe().getLong(buffer + 24); + if (generation <= 0 || headBase < 0 || activeBase < headBase) { + return null; + } + return new Record(generation, headBase, activeBase); + } + + private static final class Record { + private final long activeBase; + private final long generation; + private final long headBase; + + private Record(long generation, long headBase, long activeBase) { + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0b8379de..2bbcc109 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -36,9 +36,9 @@ * Advisory exclusive lock for a single SF slot directory. *

    * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx} - * for the entire lifetime of the engine that owns the slot. The lock is - * automatically released when the fd is closed — including on hard process - * exit, since the kernel cleans up file locks for terminated processes. + * for the entire lifetime of the engine that owns the slot. Normal teardown + * explicitly unlocks it before closing the fd; hard process exit remains a + * backstop because the kernel cleans up file locks for terminated processes. *

    * The holder's PID is written to a sibling {@code .lock.pid} file at * acquisition time. A failed acquisition reads it back so the error message @@ -116,15 +116,44 @@ public String slotDir() { return slotDir; } - @Override - public void close() { - // Closing the fd releases the lock. We do NOT remove the .lock - // file or the .lock.pid sidecar — a stale PID is harmless (next - // acquirer overwrites .lock.pid on success). - if (fd >= 0) { - Files.close(fd); + /** + * Explicitly releases the flock and reports whether the release was + * confirmed. After a successful unlock the native primitive closes + * the descriptor once, best-effort, and this object forgets its numeric + * value. It never retries that close: POSIX leaves descriptor state + * unspecified after some close failures (notably {@code EINTR}), so a + * retry could close an unrelated descriptor that reused the same number. + * We do NOT remove the {@code .lock} file or {@code .lock.pid} sidecar; a + * stale PID is harmless because the next acquirer overwrites it. + *

    + * When the explicit unlock itself fails, the fd is retained so a later + * attempt can safely retry the non-consuming unlock operation. Idempotent + * once the unlock has succeeded. + *

    + * Owners that gate a "slot dir is reusable" signal on the release + * (e.g. {@code CursorSendEngine.finishClose} publishing + * {@code closeCompleted}) must call this and check the result rather + * than {@link #close()}, which is best-effort by contract. + * + * @return {@code true} if the lock was explicitly released (or was already + * released), {@code false} if the OS reported an unlock failure + */ + public synchronized boolean release() { + if (fd < 0) { + return true; + } + if (release0(fd) == 0) { fd = -1; + return true; } + return false; + } + + @Override + public void close() { + // QuietCloseable contract: best-effort, no signal. Callers that + // must confirm the release use release() and check the result. + release(); } private static String readHolder(String pidPath) { @@ -152,6 +181,8 @@ private static String readHolder(String pidPath) { } } + private static native int release0(int fd); + private static void writePid(String pidPath) { long pid; try { diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index e3da539b..3545d3fa 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -181,8 +181,20 @@ public Sender borrowSender() { return senderPool.borrow(); } + // synchronized so concurrent close() callers serialize THROUGH shutdown + // completion, not merely through the `closed` flip. `closed` is set before + // the teardown chain runs, so a plain volatile guard (or a bare CAS) would + // let a second caller observe closed==true and return while the first is + // still inside closeQuietly(senderPool) releasing the flock/mmap/I/O-thread + // resources -- a premature return that breaks the AutoCloseable contract + // that shutdown has completed once close() returns. The monitor makes the + // losing caller block until the winner finishes, then it enters, sees + // `closed` and returns a no-op. No deadlock: the teardown steps + // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close()) + // never call back into QuestDBImpl.close() on another thread, so nothing + // contends for this monitor from within the critical section. @Override - public void close() { + public synchronized void close() { if (closed) { return; } diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1cc214b4..1bd60f3f 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -119,6 +119,10 @@ public final class SenderPool implements AutoCloseable { // RuntimeException Throwable (e.g. an -ea AssertionError) mid-prewarm, // exercising the Error-safe delegate cleanup loop. private final IntFunction senderFactory; + // Test seam: runs immediately after a delegate factory returns, before + // listener registration and SenderSlot construction. Null in production; + // error-safety tests inject a preallocated throwable at this ownership gap. + private final Runnable postFactoryHook; // Factory for startup-recovery delegates. Distinct from senderFactory so a // recoverer can force a non-blocking initial connect (initial_connect_mode= // OFF) regardless of user config: a recovery build runs on the @@ -148,6 +152,16 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Test seam: runs immediately before a capacity-starved borrow enters its + // condition wait, while it still holds the pool lock. Null in production; + // concurrency tests use a latch here to prove that several borrowers have + // all reached the wait path before recovering retired capacity. + private volatile Runnable beforeBorrowWaitHook; + // Test seam: runs after a capacity-starved borrow's condition wait has + // exhausted its positive timeout, before the loop's terminal pass. Null in + // production; regression tests release a retired slot here to prove that + // the terminal pass re-probes returned capacity before throwing. + private volatile Runnable borrowWaitExpiredHook; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the // flock drops, so the cap check and the slot allocator stay consistent @@ -171,11 +185,31 @@ public final class SenderPool implements AutoCloseable { // down on another thread. Guarded by lock. private int pendingLeaseTeardowns; // Slots whose delegate close() returned with the SF flock still held - // (the I/O thread refused to stop). Permanently consumed: the index is + // because an I/O or manager worker did not stop. Consumed while retired: // never freed and never reused, so no borrow ever hands out a still- // locked slot dir. Counted in the cap check so the lost capacity is - // accounted for. Guarded by lock; only ever ticks for SF slots. + // accounted for. NOT necessarily permanent: engine cleanup may be pending + // on a worker/I/O-thread exit path, so reprobeRetiredSlots() re-checks + // retiredSlots and returns any index whose flock has since dropped. + // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; + // Deterministic white-box complexity counter. Counts delegate release + // probes performed by direct callbacks and fallback scans. Guarded by lock. + private long retiredSlotProbeCount; + // The retired slots behind the leakedSlots count: runtime reclaim paths + // (discardBroken/reapIdle via reclaimSlot) and the in-range startup- + // recovery pass (recoverOneSlotStep, which retains the recoverer slot for + // exactly this purpose). Re-probed by reprobeRetiredSlots() so a late + // flock release (deferred engine cleanup on a worker exit path) restores + // the pool's capacity instead of ratcheting it down until process exit. + // Out-of-range startup recoverers are NEVER added: they carry no + // leakedSlots tick and their index has no slotInUse entry to free. + // Pre-sized to maxSize (every entry keeps a distinct in-range slot index + // reserved, so size can never exceed maxSize): add() never grows the + // backing array, so a retire (leakedSlots++ then add, under lock) cannot + // fail on allocation and strand a counted-but-untracked slot that + // reprobeRetiredSlots() could never recover. Guarded by lock. + private final ArrayList retiredSlots; // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a @@ -193,8 +227,9 @@ public final class SenderPool implements AutoCloseable { // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) // for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily // built pass-2 work list (same-base slots at index >= maxSize) and its - // cursor; recoveryComplete latches true when the whole scan finishes or is - // aborted, making runStartupRecoveryStep()/...ToCompletion() idempotent. + // cursor; recoveryComplete latches true only when the whole scan finishes. + // A transient build failure or drain timeout leaves the current candidate + // pending so a later tick or explicit drive can retry it on the same pool. private int recoveryInRangeNext; private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; @@ -256,6 +291,25 @@ public SenderPool( deferStartupRecovery, null, null, null); } + // Test-only constructor adding a deterministic fault hook for the ownership + // gap after a delegate factory returns. + @TestOnly + public SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + Runnable postFactoryHook + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, null, null, null, postFactoryHook); + } + // Full constructor adding the user-supplied ingest callbacks (error // handler, connection listener and background-drainer listener), applied // to every Sender the pool builds (see buildManagedSlotSender). The public @@ -273,6 +327,26 @@ public SenderPool( SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, errorHandler, connectionListener, + drainerListener, null); + } + + private SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + SenderErrorHandler errorHandler, + SenderConnectionListener connectionListener, + BackgroundDrainerListener drainerListener, + Runnable postFactoryHook ) { if (minSize < 0 || maxSize < 1 || minSize > maxSize) { throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize); @@ -291,8 +365,10 @@ public SenderPool( this.acquireTimeoutMillis = acquireTimeoutMillis; this.idleTimeoutMillis = idleTimeoutMillis; this.maxLifetimeMillis = maxLifetimeMillis; + this.postFactoryHook = postFactoryHook; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.retiredSlots = new ArrayList<>(maxSize); this.slotReleased = lock.newCondition(); // Probe the config once, up front: this validates it eagerly (so a // bad config fails at construction even when minSize == 0) and tells @@ -399,8 +475,9 @@ void runStartupRecoveryToCompletion() { * No-op (returns {@code false}) when SF is off, the pool is shutting down, or * recovery has already finished. * - * @return {@code true} if recovery has more work (call again), {@code false} - * when recovery is complete or the pool is shutting down + * @return {@code true} if recovery has more work immediately; {@code false} + * when recovery is complete, the pool is shutting down, or a transient + * failure deferred the current candidate until a later tick */ boolean runStartupRecoveryStep() { if (!storeAndForward || closed || recoveryComplete) { @@ -441,9 +518,10 @@ boolean runStartupRecoveryStep() { * {@code slotInUse} entry and are never allocated by borrow(). *

    * Best-effort throughout: a build/close Error or a slow drain is logged and - * never propagates, since the data stays durable on disk for a later attempt; - * the first build failure or drain timeout latches {@code recoveryComplete} - * (the failure will very likely repeat for every remaining slot). + * never propagates, since the data stays durable on disk for a later attempt. + * A build failure or drain timeout stops the current drive but leaves its + * candidate pending so a later housekeeper tick can retry after a transient + * condition clears; it does not poison recovery for the life of the pool. *

    * Boundedness / residual window. Recovery is driven on the * PoolHousekeeper thread, and {@code close()} relies on a step finishing @@ -470,7 +548,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryComplete = true; return false; } - final boolean[] flockHeld = new boolean[1]; + final SenderSlot[] retained = new SenderSlot[1]; // Pass 1: in-range managed slots [0, maxSize). Skip live and empty slots // cheaply; spend the step on the first slot that actually holds data. @@ -513,26 +591,31 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryInRangeNext++; continue; } - // A real candidate -> spend the step on it. Advance the cursor first - // so a resume never reprocesses this index. - recoveryInRangeNext++; - boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, flockHeld); + // A real candidate -> spend the step on it. Advance the cursor only + // after success so a transient build/drain failure remains retryable. + boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { // Release the recovery reservation accounting; from here either // leakedSlots (retire) or the freed index carries the cap math. recoveringSlots--; - if (flockHeld[0]) { - // close() bailed early with the I/O thread still running and - // the flock still held. Retire the slot permanently (mirror + if (retained[0] != null) { + // close() retained the flock because an I/O or manager + // worker did not stop. Retire the slot (mirror // discardBroken/reapIdle): keep slotInUse[i] set and count it // in leakedSlots so the borrow() cap math accounts for the // lost capacity and no later borrow ever reuses the - // still-locked dir. + // still-locked dir. Keep the recoverer in retiredSlots so + // reprobeRetiredSlots() restores the capacity once the + // deferred engine cleanup releases the flock — without it + // the retirement would be permanent even after the release + // (fatal at maxSize=1: every later borrow would time out). leakedSlots++; - LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with " - + "the flock still held (I/O thread refused to stop); pool capacity reduced by 1, " - + "now {} of {} usable [leakedSlots={}]", + addRetiredSlot(retained[0]); + LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with " + + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " + + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered " + + "if the worker releases the flock later", i, maxSize - leakedSlots, maxSize, leakedSlots); } else { slotInUse[i] = false; @@ -545,12 +628,12 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { lock.unlock(); } if (stopScan) { - // A build failure or drain timeout that will very likely repeat - // for every remaining slot -- abort the scan; the data stays - // durable on disk for a later attempt. Do not start pass 2. - recoveryComplete = true; + // Stop this drive without advancing the cursor. The same live + // pool retries this candidate after the transient condition is + // removed instead of requiring pool recreation. return false; } + recoveryInRangeNext++; return true; } @@ -571,27 +654,34 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (closed) { return false; } - int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext++); + int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext); String slotPath = sfDir + "/" + slotBaseId + "-" + idx; if (!OrphanScanner.isCandidateOrphan(slotPath)) { + recoveryOutOfRangeNext++; continue; } - boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, flockHeld); - if (flockHeld[0]) { + boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); + if (retained[0] != null) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this - // dir, so a still-held flock only leaks this recoverer's I/O - // thread (a best-effort teardown loss, logged). Crucially we do + // dir, so a still-held flock only leaks this recoverer's + // worker-reachable resources (a best-effort teardown loss, + // logged). Crucially we do // NOT touch leakedSlots -- that would wrongly shrink the - // in-range pool capacity. + // in-range pool capacity -- and we do NOT add to retiredSlots: + // there is no capacity to recover, and freeSlotIndex(idx) + // would index past the slotInUse array (sized maxSize). LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held " - + "(I/O thread refused to stop); its data is durable on disk for a later attempt", + + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); } if (stopScan) { - recoveryComplete = true; + // Keep the out-of-range cursor on this candidate. In particular, + // a transient flock/build collision must be retried after the + // primary sender returns; no capacity bookkeeping is involved. return false; } + recoveryOutOfRangeNext++; return true; } @@ -605,21 +695,24 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { * (whose {@link #defaultSender} derives the dir {@code -slotIndex}), * drains its unacked data, and closes the delegate. Shared by both recovery * passes -- the in-range pass and the out-of-range pass -- which differ only - * in their slot bookkeeping, handled by the caller via {@code flockHeld}. + * in their slot bookkeeping, handled by the caller via {@code retainedOut}. * - * @param flockHeld single-element out-param set to {@code true} iff a - * recoverer was built and its {@code close()} returned with - * the flock still held (the I/O thread refused to stop) + * @param retainedOut single-element out-param set to the recoverer iff one + * was built and its {@code close()} returned with the + * flock still held because a worker did not stop; the + * in-range caller keeps it in {@link #retiredSlots} so a + * late flock release can be re-probed. {@code null} when + * the flock was released (or no recoverer was built). * @return {@code true} if a build/drain failure occurred that will very * likely repeat for every remaining slot, so the caller should stop scanning */ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, - long remainingMillis, boolean[] flockHeld) { - flockHeld[0] = false; + long remainingMillis, SenderSlot[] retainedOut) { + retainedOut[0] = null; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and - // delegate().close() can early-return with the I/O thread still running - // (flock still held). + // delegate().close() can retain it when an I/O or manager worker does + // not stop. SenderSlot recoverer = null; boolean stopScan = false; try { @@ -638,7 +731,7 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // likely repeat for every remaining slot, so stop here rather // than pay a connect timeout per slot. LOG.warn("startup SF recovery: could not open slot {} ({}); " - + "skipping remaining slots", slotPath, buildErr.toString()); + + "deferring this and remaining slots", slotPath, buildErr.toString()); return true; } try { @@ -648,13 +741,14 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // same reasoning as the build-failure case above. if (!recoverer.delegate().drain(remainingMillis)) { LOG.warn("startup SF recovery: drain did not ack slot {} " - + "within {}ms; skipping remaining slots", + + "within {}ms; deferring this and remaining slots", slotPath, remainingMillis); stopScan = true; } } catch (Throwable drainErr) { - LOG.warn("startup SF recovery: drain failed for slot {} ({})", + LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it", slotPath, drainErr.toString()); + stopScan = true; } finally { try { recoverer.delegate().close(); @@ -664,11 +758,12 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, } } } catch (Throwable scanErr) { - LOG.warn("startup SF recovery: scan failed for slot {} ({})", + LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it", slotPath, scanErr.toString()); + stopScan = true; } - if (recoverer != null) { - flockHeld[0] = !flockReleased(recoverer); + if (recoverer != null && !flockReleased(recoverer)) { + retainedOut[0] = recoverer; } return stopScan; } @@ -768,12 +863,36 @@ public PooledSender borrow() { created.bumpGeneration(); return new PooledSender(created, created.generation()); } + // Capacity-starved: re-probe retired slots BEFORE the terminal + // timeout check — a deferred engine cleanup may have released a + // flock since the retire, and the freed index can admit a + // creation right now. The delegate normally signals this pool + // after deferred release, while this probe also covers delegates + // that do not expose that notification and release/listener races. + // Ordering matters twice over: a + // zero-timeout (try-once) borrow must get its one probe before + // throwing, and a borrower whose awaitNanos budget just expired + // must get a final probe on its wake-up pass instead of timing + // out on capacity that has already come back. + if (reprobeRetiredSlots()) { + continue; + } if (remainingNanos <= 0) { throw new LineSenderException( "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } try { + Runnable beforeWaitHook = beforeBorrowWaitHook; + if (beforeWaitHook != null) { + beforeWaitHook.run(); + } remainingNanos = slotReleased.awaitNanos(remainingNanos); + if (remainingNanos <= 0) { + Runnable hook = borrowWaitExpiredHook; + if (hook != null) { + hook.run(); + } + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new LineSenderException("interrupted while waiting for a Sender from the pool"); @@ -786,6 +905,16 @@ public PooledSender borrow() { } } + @TestOnly + public void setBeforeBorrowWaitHook(Runnable hook) { + this.beforeBorrowWaitHook = hook; + } + + @TestOnly + public void setBorrowWaitExpiredHook(Runnable hook) { + this.borrowWaitExpiredHook = hook; + } + /** * Raises the shutdown signal early -- without tearing down live delegates -- * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper} @@ -1002,7 +1131,9 @@ private void retireLease(PooledSender ps, String context) { pendingLeaseTeardowns--; if (reserved) { // Free the index only when the flock was released; a slot - // left locked is retired permanently. + // left locked is retired into retiredSlots, recoverable + // by reprobeRetiredSlots() if the deferred cleanup drops + // the flock later. reclaimSlot(s, context); } slotReleased.signalAll(); @@ -1028,6 +1159,11 @@ public void reapIdle() { if (closed) { return; } + // Housekeeper tick doubles as the retired-slot recovery driver: + // a slot retired because its worker did not stop is re-probed + // here and returns to the free set once the deferred cleanup + // finally released its flock. + reprobeRetiredSlots(); Iterator it = available.iterator(); while (it.hasNext() && all.size() > minSize) { SenderSlot s = it.next(); @@ -1067,7 +1203,7 @@ public void reapIdle() { } // Return reserved SF slot indices to the free set -- but only for // slots whose delegate confirmed the flock was released. A slot - // left locked (I/O thread refused to stop) is retired permanently. + // left locked because a worker did not stop is retired permanently. if (storeAndForward) { lock.lock(); try { @@ -1106,12 +1242,15 @@ public int totalSize() { } /** - * Snapshot of the number of SF slots permanently retired because a - * delegate {@code close()} returned with the slot flock still held (the - * I/O thread refused to stop). Each leaked slot permanently lowers the - * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A - * non-zero, growing value explains a pool that has started timing out - * every {@code borrow()}. For metrics and tests. + * Snapshot of the number of SF slots currently retired because a + * delegate {@code close()} returned with the slot flock still held after + * an I/O or manager worker did not stop. Each leaked slot lowers the + * pool's effective capacity ({@code maxSize - leakedSlotCount()}) while + * retired. Retired slots are re-probed (housekeeper tick and + * capacity-starved borrows) and recovered once the delegate's deferred + * cleanup releases the flock, so the count can go back down; a non-zero, + * persistent value means a worker is still wedged and explains a pool + * that has started timing out {@code borrow()}. For metrics and tests. */ public int leakedSlotCount() { lock.lock(); @@ -1122,8 +1261,39 @@ public int leakedSlotCount() { } } + private SenderSlot createSlot(IntFunction factory, int slotIndex) { + Sender delegate = factory.apply(slotIndex); + try { + if (postFactoryHook != null) { + postFactoryHook.run(); + } + SenderSlot slot = new SenderSlot(delegate, this, slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener( + () -> recoverReleasedSlot(slot)); + } + return slot; + } catch (Throwable failure) { + if (delegate instanceof QwpWebSocketSender) { + try { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(null); + } catch (Throwable deregistrationFailure) { + addSuppressed(failure, deregistrationFailure); + } + } + if (delegate != null) { + try { + delegate.close(); + } catch (Throwable closeFailure) { + addSuppressed(failure, closeFailure); + } + } + throw failure; + } + } + private SenderSlot createUnlocked(int slotIndex) { - return new SenderSlot(senderFactory.apply(slotIndex), this, slotIndex); + return createSlot(senderFactory, slotIndex); } /** @@ -1134,7 +1304,18 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { - return new SenderSlot(recoverySenderFactory.apply(slotIndex), this, slotIndex); + return createSlot(recoverySenderFactory, slotIndex); + } + + private static void addSuppressed(Throwable failure, Throwable cleanupFailure) { + if (failure != cleanupFailure) { + try { + failure.addSuppressed(cleanupFailure); + } catch (Throwable ignored) { + // Preserve the original construction failure even if recording + // the secondary cleanup failure cannot allocate. + } + } } private Sender defaultSender(int slotIndex) { @@ -1271,7 +1452,7 @@ private void freeSlotIndex(int idx) { * non-QWP delegate never holds an SF flock, so it is always treated as * released. A {@link QwpWebSocketSender} reports it via * {@link QwpWebSocketSender#isSlotLockReleased()} -- false means close() - * bailed early with the I/O thread still running and the flock still held. + * retained the flock because an I/O or manager worker did not stop. */ private static boolean flockReleased(SenderSlot s) { Sender d = s.delegate(); @@ -1281,11 +1462,14 @@ private static boolean flockReleased(SenderSlot s) { /** * Reclaims one SF slot after its delegate's {@code close()} has been * attempted. When the flock was released the index returns to the free - * set; when {@code close()} returned with the flock still held (the I/O - * thread refused to stop) the slot is retired permanently -- - * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap - * math accounts for the lost capacity and no later borrow ever reuses the - * still-locked dir. Either way {@code closingSlots} is decremented. + * set; when {@code close()} returned with the flock still held because an + * I/O or manager worker did not stop, the slot is retired -- + * {@code leakedSlots++}, {@code slotInUse[idx]} stays set, and the sender + * joins {@code retiredSlots} -- so the cap math accounts for the lost + * capacity and no borrow reuses the still-locked dir unless + * {@link #reprobeRetiredSlots} later observes the deferred cleanup's + * release and recovers the index. Either way {@code closingSlots} is + * decremented. *

    * Caller must hold {@code lock} and is responsible for signalling waiters * (only the free path admits a new creation). Shared by @@ -1303,9 +1487,87 @@ private boolean reclaimSlot(SenderSlot s, String context) { return true; } leakedSlots++; - LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " + - "(I/O thread refused to stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", + addRetiredSlot(s); + LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " + + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " + + "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later", s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots); return false; } + + private boolean recoverReleasedSlot(SenderSlot s) { + lock.lock(); + try { + int retiredIndex = s.retiredIndex(); + if (retiredIndex < 0 + || retiredIndex >= retiredSlots.size() + || retiredSlots.get(retiredIndex) != s) { + // The callback raced retirement, or is stale/duplicate. The + // retirement path probes before insertion, and periodic scans + // remain as a fallback, so no capacity can be lost here. + return false; + } + retiredSlotProbeCount++; + if (!flockReleased(s)) { + return false; + } + recoverRetiredSlotAt(retiredIndex); + slotReleased.signalAll(); + return true; + } finally { + lock.unlock(); + } + } + + /** + * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to + * the free set any whose delegate now reports the flock released — the + * deferred engine cleanup (manager-worker or I/O-thread exit path) has + * run since the retire. Restores {@code leakedSlots} capacity and signals + * waiters so a parked borrow can admit a creation immediately. + *

    + * Caller must hold {@code lock}. The probe is cheap on the delegate side + * ({@code isSlotLockReleased()} reads volatiles, and only re-arms the + * shared flock-release retry in the rare orphaned-retry state), so + * holding the pool lock across it cannot stall behind delegate teardown. + * + * @return {@code true} if at least one slot's capacity was recovered + */ + private boolean reprobeRetiredSlots() { + boolean recovered = false; + for (int i = retiredSlots.size() - 1; i >= 0; i--) { + SenderSlot s = retiredSlots.get(i); + retiredSlotProbeCount++; + if (flockReleased(s)) { + recoverRetiredSlotAt(i); + recovered = true; + } + } + if (recovered) { + slotReleased.signalAll(); + } + return recovered; + } + + private void addRetiredSlot(SenderSlot s) { + s.retiredIndex(retiredSlots.size()); + retiredSlots.add(s); + } + + private void recoverRetiredSlotAt(int retiredIndex) { + SenderSlot s = retiredSlots.get(retiredIndex); + int last = retiredSlots.size() - 1; + if (retiredIndex < last) { + SenderSlot moved = retiredSlots.get(last); + retiredSlots.set(retiredIndex, moved); + moved.retiredIndex(retiredIndex); + } + retiredSlots.remove(last); + s.retiredIndex(-1); + leakedSlots--; + freeSlotIndex(s.slotIndex()); + LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + + "pool capacity restored, now {} of {} usable [leakedSlots={}]", + s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index 19c93671..5d9ed6ff 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -48,6 +48,11 @@ final class SenderSlot { private final Sender delegate; private final SenderPool pool; private final int slotIndex; + // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. + // Guarded by the pool lock. The pool maintains this index across swap + // removals so the delegate's release callback can recover this exact slot + // without scanning every other retired slot. + private int retiredIndex = -1; // Monotonic lease id. Mutated only under the SenderPool lock (bumped in // borrow() when the slot is handed out and in giveBack()/discardBroken() // when it is returned). A PooledSender wrapper captures it live for its @@ -112,6 +117,14 @@ SenderPool pool() { return pool; } + int retiredIndex() { + return retiredIndex; + } + + void retiredIndex(int retiredIndex) { + this.retiredIndex = retiredIndex; + } + int slotIndex() { return slotIndex; } diff --git a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java index c1b1eec7..b5e43a35 100644 --- a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java +++ b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java @@ -149,6 +149,14 @@ public void close() { } } + @Override + public void closeTraffic() { + // A concurrent TLS send may still use the SSLEngine and direct buffers. + // Shut down only the delegate traffic path here; close() releases the + // retained fd and frees TLS state after the owning worker has stopped. + delegate.closeTraffic(); + } + @Override public int getFd() { return delegate.getFd(); diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index f649d330..b9e669d0 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -151,6 +151,8 @@ public static void init() { public static native int setTcpNoDelay(int fd, boolean noDelay); + public static native int shutdown(int fd); + public static long sockaddr(int ipv4address, int port) { SOCK_ADDR_COUNTER.incrementAndGet(); return sockaddr0(ipv4address, port); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java index d23824a5..4c66f0df 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java @@ -78,6 +78,15 @@ public interface NetworkFacade { int setTcpNoDelay(int fd, boolean noDelay); + /** + * Shuts down traffic on a descriptor owned by this facade without releasing + * the descriptor. Custom facades must override this method rather than let + * native code operate on a synthetic or remapped descriptor. + */ + default int shutdown(int fd) { + throw new UnsupportedOperationException("traffic shutdown is not supported by this network facade"); + } + long sockaddr(int address, int port); int socketTcp(boolean blocking); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java index 64ea0dc7..1d1d7e20 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java @@ -132,6 +132,11 @@ public int setTcpNoDelay(int fd, boolean noDelay) { return Net.setTcpNoDelay(fd, noDelay); } + @Override + public int shutdown(int fd) { + return Net.shutdown(fd); + } + @Override public long sockaddr(int address, int port) { return Net.sockaddr(address, port); diff --git a/core/src/main/java/io/questdb/client/network/PlainSocket.java b/core/src/main/java/io/questdb/client/network/PlainSocket.java index 555affd2..cc175f51 100644 --- a/core/src/main/java/io/questdb/client/network/PlainSocket.java +++ b/core/src/main/java/io/questdb/client/network/PlainSocket.java @@ -29,7 +29,7 @@ public class PlainSocket implements Socket { private final Logger log; private final NetworkFacade nf; - private int fd = -1; + private volatile int fd = -1; public PlainSocket(NetworkFacade nf, Logger log) { this.nf = nf; @@ -37,13 +37,22 @@ public PlainSocket(NetworkFacade nf, Logger log) { } @Override - public void close() { + public synchronized void close() { if (fd != -1) { nf.close(fd, log); fd = -1; } } + @Override + public synchronized void closeTraffic() { + if (fd != -1 && nf.shutdown(fd) != 0) { + throw new IllegalStateException( + "could not shut down socket traffic [fd=" + fd + ", errno=" + nf.errno() + ']' + ); + } + } + @Override public int getFd() { return fd; diff --git a/core/src/main/java/io/questdb/client/network/Socket.java b/core/src/main/java/io/questdb/client/network/Socket.java index 0cdce517..6fceb30b 100644 --- a/core/src/main/java/io/questdb/client/network/Socket.java +++ b/core/src/main/java/io/questdb/client/network/Socket.java @@ -37,6 +37,17 @@ public interface Socket extends QuietCloseable { int WRITE_FLAG = 1; + /** + * Closes only the network traffic path so a concurrent blocking send or + * receive returns. Implementations must retain any I/O buffers until the + * owning worker has stopped and {@link #close()} performs full cleanup. + * The compatibility default fails without touching implementation-owned + * resources; custom sockets must override this method to opt in. + */ + default void closeTraffic() { + throw new UnsupportedOperationException("traffic shutdown is not supported by this socket"); + } + /** * @return file descriptor associated with the socket. */ diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java index dca93e84..194d0972 100644 --- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java @@ -86,6 +86,11 @@ public int fsync(int fd) { return Files.fsync(fd); } + @Override + public int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + @Override public long length(int fd) { return Files.length(fd); @@ -106,6 +111,21 @@ public int mkdir(String path, int mode) { return Files.mkdir(path, mode); } + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + + @Override + public int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + + @Override + public void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { return Files.openCleanRW(path); @@ -126,6 +146,16 @@ public int openRW(long pathPtr) { return Files.openRW(pathPtr); } + @Override + public int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + @Override public long length(long pathPtr) { return Files.length(pathPtr); diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 02756c37..b0fdfb2e 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -154,6 +154,24 @@ public static int openRW(long pathPtr) { return openRW0(pathPtr); } + /** + * Atomically creates {@code path} for read-write access. Fails with -1 + * when the path already exists; existing bytes are never truncated. + */ + public static int openRWExclusive(String path) { + long ptr = pathPtr(path); + try { + return openRWExclusive0(ptr); + } finally { + freePathPtr(ptr); + } + } + + /** Native-path variant of {@link #openRWExclusive(String)}. */ + public static int openRWExclusive(long pathPtr) { + return openRWExclusive0(pathPtr); + } + /** * Opens {@code path} for append-only writes, creating it (mode 0644) if * absent. Every {@link #append(int, long, long)} writes at end-of-file @@ -408,6 +426,20 @@ public static String utf8ToString(long nameZ) { */ public static native int fsync(int fd); + /** + * Forces directory-entry updates under {@code dir} to durable storage. + * Returns 0 on success and non-zero on failure. Callers use this after + * unlinking files whose absence must survive a host crash. + */ + public static int fsyncDir(String dir) { + long ptr = pathPtr(dir); + try { + return fsyncDir0(ptr); + } finally { + freePathPtr(ptr); + } + } + /** * Truncates the file to exactly {@code size} bytes via {@code ftruncate}. * Returns {@code true} on success. Does NOT reserve disk space — the @@ -555,10 +587,14 @@ public static void munmap(long address, long len, int memoryTag) { static native int close0(int fd); + static native int fsyncDir0(long lpszName); + static native int openRO0(long lpszName); static native int openRW0(long lpszName); + static native int openRWExclusive0(long lpszName); + static native int openAppend0(long lpszName); static native int openCleanRW0(long lpszName); diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 1b408cf4..082e680e 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -85,6 +85,10 @@ public interface FilesFacade { int fsync(int fd); + default int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + long length(int fd); /** @@ -107,6 +111,18 @@ public interface FilesFacade { int mkdir(String path, int mode); + default long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + + default int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + + default void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + int openCleanRW(String path); /** @@ -122,6 +138,14 @@ public interface FilesFacade { /** Variant of {@link #openRW(String)} taking a pre-encoded native UTF-8 path pointer. */ int openRW(long pathPtr); + default int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + default int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + /** * Variant of {@code length(String)} taking a pre-encoded native UTF-8 path * pointer; same allocation-elision rationale as {@link #openRW(long)}. diff --git a/core/src/main/java/io/questdb/client/std/ObjList.java b/core/src/main/java/io/questdb/client/std/ObjList.java index 9e0e7b29..34d989ae 100644 --- a/core/src/main/java/io/questdb/client/std/ObjList.java +++ b/core/src/main/java/io/questdb/client/std/ObjList.java @@ -149,7 +149,7 @@ public void remove(int from, int to) { System.arraycopy(buffer, to + 1, buffer, from, move); } pos = Math.max(0, pos - (to - from + 1)); - Arrays.fill(buffer, pos, buffer.length - 1, null); + Arrays.fill(buffer, pos, buffer.length, null); } public void remove(int index) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java index 9f1500f2..e4fb86c5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java @@ -35,8 +35,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** * Regression tests for the close() drain semantics. @@ -49,6 +51,83 @@ */ public class CloseDrainTest { + @Test(timeout = 30_000L) + public void testCloseBlocksAcrossAllReplicaWindowUntilPromotion() throws Exception { + DelayingAckHandler handler = new DelayingAckHandler(0); + try (TestWebSocketServer server = new TestWebSocketServer(handler, false, "PRIMARY")) { + server.setRejectWithRole("REPLICA"); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + server.getPort() + + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=100" + + ";close_flush_timeout_millis=10000;"; + QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg); + CountDownLatch closeDrainWaiting = new CountDownLatch(1); + CountDownLatch releaseCloseDrain = new CountDownLatch(1); + AtomicReference closeFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseDrainWaitingHook(() -> { + closeDrainWaiting.countDown(); + try { + if (!releaseCloseDrain.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("promotion did not release the close-drain witness"); + } + } catch (Throwable t) { + hookFailure.set(t); + } + }); + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "all-replica-close-drain"); + try { + closer.start(); + Assert.assertTrue("server never produced the all-replica role rejection", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + Assert.assertTrue("close never observed its real unacknowledged drain target", + closeDrainWaiting.await(5, TimeUnit.SECONDS)); + + Assert.assertTrue("pre-promotion witness must include a role rejection", + server.roleRejectCount() >= 1); + Assert.assertEquals("pre-promotion data must remain unacknowledged", + -1L, sender.getAckedFsn()); + Assert.assertTrue("close must remain pending for the whole all-replica window", + closer.isAlive()); + Assert.assertEquals("promotion must not have delivered or replayed the frame yet", + 0L, handler.nextSeq.get()); + + // The close-drain barrier has held continuously since it observed + // targetFsn > ackedFsn. Promote first, then release the barrier: + // completion therefore cannot precede the deterministic recovery event. + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + + Assert.assertFalse("close did not complete after promotion", closer.isAlive()); + Assert.assertNull("close-drain witness failed", hookFailure.get()); + Assert.assertNull("close failed after promotion", closeFailure.get()); + Assert.assertEquals("the unacknowledged frame must be delivered exactly once", + 1L, handler.nextSeq.get()); + } finally { + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + sender.close(); + } + } + } + @Test public void testCloseBlocksUntilAckArrives() throws Exception { // Server delays every ACK by 800ms. With the default @@ -77,6 +156,56 @@ public void testCloseBlocksUntilAckArrives() throws Exception { } } + @Test + public void testCloseStartedHookRunsAfterClosedStateTransition() throws Exception { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference closerFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseStartedHook(() -> { + try { + try { + sender.table("must_reject_after_close_started"); + Assert.fail("close-started hook ran before closed=true was published"); + } catch (LineSenderException expected) { + // Required lifecycle boundary: public operations already reject. + } + entered.countDown(); + if (!release.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("close-started hook release timed out"); + } + } catch (Throwable t) { + hookFailure.set(t); + entered.countDown(); + } + }); + + Assert.assertEquals("installing the hook must not emit a pre-invocation witness", + 1L, entered.getCount()); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closerFailure.set(t); + } + }, "close-started-hook-test"); + try { + closer.start(); + Assert.assertTrue("close() never reached its internal lifecycle witness", + entered.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close() completed while its internal witness was held", + closer.isAlive()); + } finally { + release.countDown(); + closer.join(10_000L); + sender.close(); + } + Assert.assertFalse("close thread did not finish", closer.isAlive()); + Assert.assertNull("close-started hook failed", hookFailure.get()); + Assert.assertNull("close() failed", closerFailure.get()); + } + @Test public void testCloseFastWhenTimeoutIsZero() throws Exception { // Same delayed-ACK server, but with close_flush_timeout_millis=0 diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java new file mode 100644 index 00000000..5bfb0863 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpWebSocketSenderCursorEngineAttachmentTest { + + private static final long SEGMENT_SIZE = 64 * 1024L; + + @Rule + public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testNullDoesNotDetachAnAttachedEngine() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, true); + assertSecondAttachmentRejected(sender, null, false); + sender.close(); + Assert.assertTrue("sender must retain ownership after rejected detach", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + @Test + public void testOwnedEngineCannotBeReplacedAndItsSlotIsReleased() throws Exception { + assertMemoryLeak(() -> { + File slotDir = new File(sfDir.getRoot(), "owned-slot"); + CursorSendEngine first = null; + CursorSendEngine replacement = null; + CursorSendEngine reacquired = null; + QwpWebSocketSender sender = null; + boolean replacementRejected = false; + try { + first = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + replacement = new CursorSendEngine(null, SEGMENT_SIZE); + sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(first, true); + try { + sender.setCursorEngine(replacement, true); + } catch (LineSenderException e) { + replacementRejected = true; + Assert.assertTrue(e.getMessage().contains("already attached")); + } + + sender.close(); + sender = null; + try { + reacquired = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + } catch (IllegalStateException e) { + throw new AssertionError("sender close did not release the first owned " + + "engine's slot after a second attachment attempt", e); + } + Assert.assertTrue("second attachment must be rejected", replacementRejected); + } finally { + if (sender != null) { + sender.close(); + } + if (reacquired != null) { + reacquired.close(); + } + if (replacement != null) { + replacement.close(); + } + if (first != null) { + first.close(); + } + } + }); + } + + @Test + public void testSameSharedEngineCannotTransferOwnership() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, false); + assertSecondAttachmentRejected(sender, engine, true); + sender.close(); + Assert.assertFalse("rejected attachment must not transfer ownership", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + private static void assertSecondAttachmentRejected( + QwpWebSocketSender sender, + CursorSendEngine engine, + boolean takeOwnership + ) { + try { + sender.setCursorEngine(engine, takeOwnership); + Assert.fail("expected second cursor engine attachment to be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("already attached")); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 85522e9d..63553ebc 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -24,9 +24,22 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.SenderError; +import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -36,8 +49,12 @@ import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; /** @@ -46,9 +63,13 @@ *

      *
    • Happy path — a clean {@code close()} that winds the I/O thread * down reports {@code isSlotLockReleased() == true}.
      • - *
    • Leak path — a {@code close()} that early-returns because the I/O - * thread refused to stop ({@code ioThreadStopped == false}) reports + *
    • Leak path — a {@code close()} that retains the lock because an + * I/O or manager worker did not stop reports * {@code isSlotLockReleased() == false}.
      • + *
    • Recovery path — the getter is monotonic but not frozen: it + * re-probes the retained engine, so once the deferred cleanup (worker + * exit path or a retried close) releases the flock, the getter flips to + * {@code true} and a pool that retired the slot can recover it.
      • *
    * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held, * retire the slot permanently". Before this test that contract was driven only @@ -178,6 +199,380 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception }); } + /** + * Manager-worker handoff path: engine close retains the slot while a shared + * manager is still inside a service pass. The sender must expose that + * retained flock to SenderPool. Repeated sender close calls remain no-ops; + * the manager pass owns deferred cleanup and releases the flock when it + * finishes, so the sender can expose recovery without a direct close retry. + */ + @Test(timeout = 30_000L) + public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-manager-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + CursorSendEngine engine = null; + QwpWebSocketSender wss = null; + boolean managerClosed = false; + try { + manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + manager.setWorkerJoinTimeoutMillis(50L); + wss.close(); + + Assert.assertFalse("sender reported a retained flock as released", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must remain incomplete while worker is in service", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while manager worker was still in service"); + } catch (Exception expected) { + // good — the incomplete close retained the flock. + } + + // Sender.close() is idempotent: a second call must not retry + // or change ownership while the manager worker is still live. + wss.close(); + Assert.assertFalse("repeated close changed retained-flock reporting", + wss.isSlotLockReleased()); + Assert.assertFalse("repeated close unexpectedly retried engine cleanup", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + Assert.assertTrue("shared-manager pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("deferred cleanup did not complete after the ring pass", + engine.isCloseCompleted()); + // Recovery contract: the sender re-probes its retained engine, + // so the completed cleanup (and released flock) MUST become + // visible — this is what lets SenderPool recover a slot it + // had already retired as leaked. + Assert.assertTrue("sender must expose the late flock release to the pool", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after deferred cleanup", probe); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (wss != null) { + try { + wss.close(); + } catch (Throwable ignored) { + } + } + if (!managerClosed) { + manager.close(); + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + + /** + * Delegated-I/O-close recovery path: when {@code close()} bails early + * because the I/O thread refuses to stop, the engine close is delegated + * to that thread's exit path ({@code delegateEngineClose()} returned + * true) and the sender must retain the engine for re-probing — + * {@code isSlotLockReleased()} stays false while the thread lives, then + * flips true the moment the thread's exit path completes the engine + * close and releases the flock. Pre-fix, this branch discarded the + * engine reference, so the late release was permanently invisible to a + * pool that had retired the slot. The existing forged I/O-refusal test + * can never reach this branch: its sabotaged loop throws from + * {@code delegateEngineClose()} itself (nulled latch), before the + * retained-engine assignment. + *

    + * The wedge is the same deterministic one CursorWebSocketSendLoop's C5 + * test uses: the I/O thread sits in a blocking "connect" (a + * ReconnectFactory immune to unpark, never interrupted by loop.close()), + * and the closer thread's pre-set interrupt flag makes + * {@code shutdownLatch.await()} throw immediately — the real production + * path to {@code ioThreadStopped = false}. + */ + @Test(timeout = 30_000L) + public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-delegated-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + final CountDownLatch enteredConnect = new CountDownLatch(1); + final CountDownLatch releaseConnect = new CountDownLatch(1); + final AtomicReference ioThreadRef = new AtomicReference<>(); + final StubWebSocketClient stubClient = new StubWebSocketClient(); + // Healthy owned manager: once the wedged I/O thread is released, + // its exit-path engine.close() completes normally and releases + // the flock — the flip this test pins. + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + QwpWebSocketSender wss = null; + try { + // Stand-in for a blocking native connect(2): entered by the + // loop's I/O thread, immune to unpark, never interrupted by + // loop.close(). + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return stubClient; + }; + loop = new CursorWebSocketSendLoop( + null /* async-initial-connect: the I/O thread drives the connect */, + engine, 0L, 1_000L, + stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + setField(wss, "cursorSendLoop", loop); + + // Drive the real early-bail close() on a thread whose pending + // interrupt lands in loop.close()'s shutdownLatch.await(). + AtomicReference closeFailure = new AtomicReference<>(); + QwpWebSocketSender wssRef = wss; + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + wssRef.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "delegated-close-closer"); + closer.setDaemon(true); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + // The I/O thread is still wedged: the flock is retained and + // must be reported retained. + Assert.assertFalse( + "isSlotLockReleased() must be false while the delegated engine " + + "close is pending on the wedged I/O thread", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must not have run yet", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while the delegated engine close " + + "was still pending"); + } catch (Exception expected) { + // good — the flock is genuinely held. + } + + // Un-wedge the connect. The I/O thread exits; its exit path + // runs the delegated engine.close(), which releases the flock. + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after the connect returned", + ioThread.isAlive()); + + // The recovery contract under test: the getter re-probes the + // retained engine, so the late release MUST become visible — + // this is what lets SenderPool recover the retired slot. + Assert.assertTrue("delegated engine close must have completed on the " + + "I/O thread's exit path", + engine.isCloseCompleted()); + Assert.assertTrue( + "isSlotLockReleased() must flip true once the delegated engine " + + "close released the flock — otherwise the pool retires the " + + "slot's capacity until process exit", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the delegated close", probe); + } + } finally { + releaseConnect.countDown(); + // Reap the loop's bookkeeping now that the I/O thread is gone + // (close() threw mid-teardown, so ioThread was left set). + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + if (engine != null && !engine.isCloseCompleted()) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + // The early-return close() deliberately leaked the resources + // the (then-running) I/O thread might touch; free the same set + // the post-guard tail would have freed. + if (wss != null) { + freeFieldQuietly(wss, "buffer0"); + freeFieldQuietly(wss, "buffer1"); + freeFieldQuietly(wss, "client"); + freeFieldQuietly(wss, "errorDispatcher"); + freeFieldQuietly(wss, "progressDispatcher"); + freeFieldQuietly(wss, "connectionDispatcher"); + } + stubClient.close(); + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + + @Test(timeout = 30_000L) + public void testFailedIoStopReclaimsSenderResourcesAfterWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-full-cleanup-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + CountDownLatch enteredConnect = new CountDownLatch(1); + CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicReference ioThreadRef = new AtomicReference<>(); + StubWebSocketClient loopClient = new StubWebSocketClient(); + StubWebSocketClient senderClient = new StubWebSocketClient(); + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + try { + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return loopClient; + }; + loop = new CursorWebSocketSendLoop( + null, engine, 0L, 1_000L, stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setConnectionListener(event -> { + }); + sender.setErrorHandler(error -> { + }); + sender.setProgressHandler(ackedFsn -> { + }); + sender.setClientForTesting(senderClient); + sender.setCursorEngine(engine, true); + sender.setCursorSendLoopForTesting(loop); + + SenderConnectionDispatcher connectionDispatcher = sender.getConnectionDispatcherForTesting(); + SenderErrorDispatcher errorDispatcher = sender.getErrorDispatcherForTesting(); + SenderProgressDispatcher progressDispatcher = sender.getProgressDispatcherForTesting(); + Assert.assertTrue(connectionDispatcher.offer(new SenderConnectionEvent( + SenderConnectionEvent.Kind.DISCONNECTED, + null, SenderConnectionEvent.NO_PORT, + null, SenderConnectionEvent.NO_PORT, + SenderConnectionEvent.NO_ATTEMPT_NUMBER, + SenderConnectionEvent.NO_ROUND_NUMBER, + null, 0L))); + Assert.assertTrue(errorDispatcher.offer(new SenderError( + SenderError.Category.UNKNOWN, SenderError.Policy.RETRIABLE, + SenderError.NO_STATUS_BYTE, null, SenderError.NO_MESSAGE_SEQUENCE, + -1L, -1L, null, 0L))); + Assert.assertTrue(progressDispatcher.offer(0L)); + Thread connectionDispatcherThread = connectionDispatcher.getWorkerThreadForTesting(); + Thread errorDispatcherThread = errorDispatcher.getWorkerThreadForTesting(); + Thread progressDispatcherThread = progressDispatcher.getWorkerThreadForTesting(); + Assert.assertNotNull(connectionDispatcherThread); + Assert.assertNotNull(errorDispatcherThread); + Assert.assertNotNull(progressDispatcherThread); + + AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "full-cleanup-closer"); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after release", ioThread.isAlive()); + Assert.assertTrue("complete sender cleanup callback did not finish", + sender.isCloseCleanupComplete()); + Assert.assertTrue("loop-owned WebSocket client was not closed", loopClient.isClosed()); + Assert.assertTrue("sender-owned WebSocket client was not closed", senderClient.isClosed()); + Assert.assertFalse("connection dispatcher worker was not reclaimed", + connectionDispatcherThread.isAlive()); + Assert.assertFalse("error dispatcher worker was not reclaimed", + errorDispatcherThread.isAlive()); + Assert.assertFalse("progress dispatcher worker was not reclaimed", + progressDispatcherThread.isAlive()); + Assert.assertTrue("engine cleanup did not complete", engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull(probe); + } + } finally { + releaseConnect.countDown(); + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -192,6 +587,28 @@ private static void freeFieldQuietly(Object target, String name) { } } + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + private static T readField(Object target, String name, Class type) throws Exception { Class cls = target.getClass(); while (cls != null) { @@ -221,6 +638,39 @@ private static void setField(Object target, String name, Object value) throws Ex throw new NoSuchFieldException(name); } + /** + * Minimal concrete {@link WebSocketClient} — never performs I/O. Handed + * to the loop by the stuck-connect factory; the loop's exit path closes + * it (close is idempotent via the superclass). + */ + private static final class StubWebSocketClient extends WebSocketClient { + private final AtomicBoolean isClosed = new AtomicBoolean(); + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + isClosed.set(true); + super.close(); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + boolean isClosed() { + return isClosed.get(); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + /** ACKs every binary frame with a running sequence so flush/close drain cleanly. */ private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler { private final AtomicLong nextSeq = new AtomicLong(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java new file mode 100644 index 00000000..a9df2b67 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java @@ -0,0 +1,209 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Regression for the close-time segment cleanup on a fully-drained slot. + *

    + * {@code CursorSendEngine.finishClose} unlinks the acknowledged {@code .sfa} + * files and then removes the ack watermark. When the unlink fails (transient + * I/O error, permission problem), the residual segment files hold rows the + * server already acknowledged. If the watermark does not cover them, a + * successor engine on the same slot seeds recovery from {@code lowestBase - 1} + * and replays every acknowledged row — duplicates on a non-DEDUP table. + *

    + * The test injects the unlink failure by dropping write permission on the + * slot directory (POSIX: unlink requires a writable parent directory), so it + * is skipped on Windows and when permissions are not enforced (root). + * The shared {@link SegmentManager} is deliberately never started: no worker + * thread exists, so no manager tick can persist the watermark behind the + * test's back, and the close-path quiescence barrier is trivially satisfied — + * fully deterministic, no timing coordination needed. + */ +public class CursorSendEngineCloseUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test(timeout = 20_000L) + public void testFailedCloseTimeUnlinkMustNotExposeAckedFramesToSuccessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String slot = tmpDir + "/slot"; + Path slotPath = Paths.get(slot); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60)); + CursorSendEngine pred = null; + CursorSendEngine succ = null; + boolean slotDirReadOnly = false; + try { + fill(payload, 32, (byte) 0x33); + pred = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals(0L, pred.appendBlocking(payload, 32)); + Assert.assertEquals(0L, pred.publishedFsn()); + // The server durably acknowledged FSN 0 in this session. + Assert.assertTrue(pred.acknowledge(0L)); + Assert.assertEquals(0L, pred.ackedFsn()); + + // Inject a close-time unlink failure: drop write permission on + // the slot dir. Prove the injection works with a probe file -- + // root (and some filesystems) ignore directory permissions. + String probePath = slot + "/probe"; + Assert.assertTrue(java.nio.file.Files.exists( + java.nio.file.Files.createFile(Paths.get(probePath)))); + try { + setPermissions(slotPath, "r-xr-xr-x"); + } catch (UnsupportedOperationException e) { + Assume.assumeNoException("POSIX permissions unavailable on this platform", e); + } + slotDirReadOnly = true; + boolean probeRemoved = Files.remove(probePath); + if (probeRemoved) { + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + } + Assume.assumeFalse("directory permissions not enforced (running as root?)", + probeRemoved); + + // Fully-drained close: tries to unlink the acknowledged + // segment files and fails. + pred.close(); + Assert.assertTrue("flock release needs no dir write; close must complete", + pred.isCloseCompleted()); + pred = null; + Assert.assertTrue("injected unlink failure must leave the acknowledged segment", + Files.exists(slot + "/sf-initial.sfa")); + + // The transient failure clears before the successor arrives. + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + Files.remove(probePath); + + succ = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(succ.wasRecoveredFromDisk()); + Assert.assertEquals(0L, succ.publishedFsn()); + // THE regression: FSN 0 was acknowledged by the server during + // the predecessor's session. The successor must not see it as + // replayable, or a non-DEDUP table receives duplicate rows. + Assert.assertTrue("successor exposes already-acknowledged frames for replay " + + "[ackedFsn=" + succ.ackedFsn() + + ", publishedFsn=" + succ.publishedFsn() + "]", + succ.ackedFsn() >= succ.publishedFsn()); + + // The successor's own fully-drained close retries the cleanup + // now that the failure has cleared: segments and watermark gone. + succ.close(); + Assert.assertTrue(succ.isCloseCompleted()); + succ = null; + Assert.assertFalse("successor close did not retry the segment unlink", + Files.exists(slot + "/sf-initial.sfa")); + Assert.assertFalse("watermark must be removed once no segment file remains", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } finally { + if (slotDirReadOnly) { + try { + setPermissions(slotPath, "rwxr-xr-x"); + } catch (Throwable ignored) { + } + } + if (pred != null) { + pred.close(); + } + if (succ != null) { + succ.close(); + } + manager.close(); + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static void setPermissions(Path path, String posix) throws Exception { + Set perms = PosixFilePermissions.fromString(posix); + java.nio.file.Files.setPosixFilePermissions(path, perms); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java new file mode 100644 index 00000000..d6c3f665 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java @@ -0,0 +1,310 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class CursorSendEngineCrashConsistencyTest { + + @Test + public void testCloseDurabilityOrderAndSyncFailurePropagation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{-1, 0, 1, 2, 4}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine engine = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, failAt); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + engine = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, engine.appendBlocking(payload, 32)); + Assert.assertTrue(engine.acknowledge(0L)); + ff.beginClose(); + try { + engine.close(); + if (failAt >= 0 && failAt != 3) { + Assert.fail("sync failure was swallowed at boundary " + failAt); + } + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, + failAt >= 0 && failAt != 3); + } + engine = null; + + if (failAt >= 0 && failAt <= 2) { + Assert.assertFalse("segment deletion started after watermark barrier failure", + ff.events.contains("segment-remove")); + Assert.assertTrue("watermark was removed after its durability barrier failed", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } else { + Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, + ff.durableSegments && !ff.durableWatermark); + } + if (failAt == -1) { + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", + "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"), + ff.events); + } + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, engine); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) { + if (failure == null) { + return cleanupFailure; + } + if (failure != cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + return failure; + } + + private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) { + if (engine != null) { + try { + engine.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable closeManager(Throwable failure, SegmentManager manager) { + if (manager != null) { + try { + manager.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable freePayload(Throwable failure, long payload) { + if (payload != 0) { + try { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static Throwable removeRoot(Throwable failure, String root) { + try { + removeRecursive(root); + Assert.assertFalse("test directory was not removed: " + root, Files.exists(root)); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + return failure; + } + + private static void rethrow(Throwable failure) { + if (failure != null) { + CursorSendEngineCrashConsistencyTest.throwUnchecked(failure); + } + } + + @SuppressWarnings("unchecked") + private static void throwUnchecked(Throwable failure) throws T { + throw (T) failure; + } + + private static final class CrashImageFilesFacade implements FilesFacade { + private final List events = new ArrayList<>(); + private final int failAt; + private final String slot; + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private int watermarkFd = -1; + + private CrashImageFilesFacade(String slot, int failAt) { + this.slot = slot; + this.failAt = failAt; + } + + private void beginClose() { + active = true; + events.clear(); + eventIndex = 0; + } + + private boolean fail(String event) { + events.add(event); + return eventIndex++ == failAt; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override + public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override + public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override + public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { + return active && fd == watermarkFd && fail("watermark-fsync") ? -1 : INSTANCE.fsync(fd); + } + @Override + public int fsyncDir(String dir) { + if (active && slot.equals(dir)) { + if (fail("dir-fsync")) return -1; + if (events.contains("segment-remove")) { + durableSegments = false; + } else { + durableWatermark = true; + } + } + return INSTANCE.fsyncDir(dir); + } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + return active && fail("watermark-msync") ? -1 : INSTANCE.msync(addr, len, async); + } + @Override + public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override + public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + if (fail("segment-remove")) return false; + } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) { + if (fail("watermark-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java new file mode 100644 index 00000000..2c5f76c3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -0,0 +1,865 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Engine-level regression for the shutdown hazard where + * {@link CursorSendEngine#close()} released the slot lock, closed the ring + * and watermark, and unlinked segment files while the shared + * {@link SegmentManager} worker was still mid service pass for the engine's + * ring. A replacement engine could acquire the same slot the moment the + * lock was released, after which the stale worker's abandon/trim path could + * unlink a segment path the replacement was actively writing through — + * store-and-forward data loss after restart. + *

    + * The fix makes {@code close()} run a quiescence barrier + * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and + * refuse to release any worker-reachable resource (ring, watermark, segment + * files, slot lock) until the barrier confirms the worker cannot touch the + * slot again. On barrier timeout an owned-manager engine hands cleanup + * ownership to the worker's exit path (see + * {@code SegmentManager.deferUntilWorkerExit}); a shared-manager engine + * deliberately leaks and a later {@code close()} retries the cleanup. + */ +public class CursorSendEngineSlotReacquisitionTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-slot-reacq-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + + /** + * The structural guarantee: while the manager worker is provably still + * inside a service pass for the engine's ring, repeated direct + * {@code close()} calls must NOT hand the slot to anyone else. With the + * duplicate-cleanup-owner branch reverted, the second close runs cleanup + * inline and the mid-test {@code SlotLock.acquire} probe succeeds. Once + * the pass finishes, its deferred cleanup must run exactly once and + * release the slot without another close retry. + */ + @Test(timeout = 30_000L) + public void testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/slot"; + // 60 s poll: the worker only acts when explicitly woken, so the + // single pass we park below is the only pass in flight. + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger cleanupCount = new AtomicInteger(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + CursorSendEngine engine = null; + try { + manager.setAfterRingCleanupHook(() -> { + cleanupCount.incrementAndGet(); + cleanupFinished.countDown(); + }); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + + // Shared manager: ownsManager=false, so engine close() cannot + // fall back on manager.close()'s join — the per-ring barrier + // is the only protection, which is exactly what we pin here. + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached the install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Barrier must time out fast: the worker is parked inside the + // service pass for this engine's ring. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("incomplete close must remain observable to the owner", + engine.isCloseCompleted()); + + // Exercise CursorSendEngine.close() directly again while the + // same pass already owns deferredClose. Sender.close() cannot + // reach this branch because its second call is a no-op. + engine.close(); + Assert.assertFalse("repeated close must not steal deferred cleanup ownership", + engine.isCloseCompleted()); + Assert.assertEquals("cleanup ran while the worker pass was blocked", + 0, cleanupCount.get()); + + // The slot must still be locked: a replacement engine (or raw + // SlotLock) acquiring it now would race the stale worker. + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while the manager " + + "worker was still mid service pass for its ring — a " + + "replacement engine could acquire the slot and have its " + + "segment files unlinked by the stale worker"); + } catch (Exception expected) { + // good — slot retained. + } + + // Let the worker finish its pass (it abandons the spare: the + // ring was deregistered by the close attempt above). + releaseWorker.countDown(); + Assert.assertTrue("ring pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("ring-pass cleanup must run exactly once", + 1, cleanupCount.get()); + Assert.assertTrue("ring-pass cleanup must report complete cleanup", + engine.isCloseCompleted()); + engine = null; + + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("ring-pass cleanup did not release the slot lock", e); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + manager.close(); + } + } + }); + } + + /** + * Owned-manager twin of {@link #testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass}: + * the ONLY construction shape production uses (Sender.build, BackgroundDrainer, + * QwpWebSocketSender.connect all own their manager). The owned close path does + * not run the per-ring barrier at all — it relies on {@code manager.close()}'s + * bounded join and the {@code isWorkerReaped()} check. If that check regressed + * to report quiescence unconditionally (or {@code isWorkerReaped()} itself + * returned true while the worker is alive), close() would release the slot + * lock mid service pass and the shared-manager tests would stay green — this + * test is the red gate for the production path. + *

    + * Determinism: the owned manager starts inside the engine ctor (1 ms poll), + * so its first spare-install pass races test setup. We first wait until the + * initial hot spare is installed — after that the worker cannot enter another + * install pass until a rotation consumes the spare, so the park hook installed + * afterwards can neither be missed nor fire early. Two appends then fill the + * active segment and rotate onto the spare; the worker's next poll tick + * re-enters the install pass and parks in the hook. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-parked-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: let the worker finish the initial spare install so + // the hook below can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: one frame fills the active segment exactly; the + // second forces rotation onto the spare. needsHotSpare() is + // true again, so the worker's next tick parks in the hook. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid service + // pass. manager.close()'s 50 ms join times out, the worker is + // not reaped, and close() must retain every worker-reachable + // resource — above all the slot flock. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("incomplete owned close must remain observable to the owner", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("owned engine.close() released the slot lock while its manager " + + "worker was still mid service pass — a replacement engine could " + + "acquire the slot and have its segment files unlinked by the " + + "stale worker (the production SF-data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained. + } + + // Phase 4: release the worker (it abandons the spare: the ring + // was deregistered by the close attempt) and retry. The join + // now reaps the worker and the full cleanup must complete. + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried owned close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("retried owned close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * The ownership handoff (owned manager): when close() cannot confirm + * worker quiescence within the bounded join, the terminal cleanup (ring, + * watermark, flock release) transfers to the worker's exit path — the + * worker is provably the last thread able to touch the slot directory. + * Once the parked pass is released the worker must run the cleanup + * itself, WITHOUT any retried {@code close()}: {@code isCloseCompleted()} + * flips true and the slot becomes acquirable again. This is what lets a + * pool recover a retired slot instead of losing its capacity until + * process exit. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-handoff-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid-pass. The + // bounded join times out; cleanup ownership is handed to the + // worker's exit path. Every worker-reachable resource — above + // all the slot flock — must still be retained at this point. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while its manager worker " + + "was still mid service pass — the handoff must not weaken the " + + "quiescence gate"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4 — the contract under test: release the worker and do + // NOT retry close(). The worker finishes its pass, exits, and + // runs the deferred cleanup itself, flipping isCloseCompleted + // and releasing the flock with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred cleanup never ran on manager-worker exit — the slot " + + "would stay retired until process exit"); + } + Thread.sleep(1); + } + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } catch (Exception e) { + throw new AssertionError("worker-exit cleanup did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Registration-failure twin of + * {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: when + * {@code deferUntilWorkerExit} itself throws (allocation failure while + * building the handoff), close() must NOT mistake the swallowed throw + * for "worker already exited" and run the terminal cleanup inline — the + * worker is provably still mid service pass, so releasing the ring, + * watermark or slot flock here is the original stale-worker UAF/data-loss + * hazard. Every worker-reachable resource must be retained and the close + * must stay incomplete; a retried close() after the worker exits + * converges and releases the slot. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-regfail-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: make the handoff registration throw — simulating + // an OutOfMemoryError allocating the cleanup lambda/list — + // with the worker provably still mid service pass. close() + // must retain everything: no inline finishClose, no flock + // release, closeCompleted stays false. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated allocation failure registering exit cleanup"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete when handoff registration fails", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock after a failed handoff " + + "registration while the manager worker was still mid service " + + "pass — the swallowed throw was mistaken for proof the worker " + + "exited (stale-worker UAF/data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4: clear the fault, release the worker (its loop was + // already stopped by the close attempt, so it exits), and + // retry close(). The retry must converge via isWorkerReaped() + // and release the slot. + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the retried close", probe); + } catch (Exception e) { + throw new AssertionError("retried close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Exactly-once contention on the terminal-cleanup claim + * ({@code terminalCleanupClaimed} CAS): after a timed-out owned close + * handed cleanup to the worker's exit path, a retried {@code close()} + * that races the worker MID-{@code finishClose} must neither re-run the + * terminal cleanup (double munmap / double flock release) nor block on + * the worker, nor publish completion on the worker's behalf. The race + * window is made deterministic by parking the worker inside + * {@code finishClose} (via {@code beforeFlockReleaseHook}) while the + * retried close() converges through {@code isWorkerReaped()} and loses + * the CAS. + */ + @Test(timeout = 30_000L) + public void testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/cas-contention-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch inFinishClose = new CountDownLatch(1); + CountDownLatch releaseFinishClose = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger finishCloseRuns = new AtomicInteger(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Counts terminal-cleanup executions and parks the FIRST one + // (the worker's deferred cleanup) mid-finishClose, before the + // flock release — the exact window a retried close() races. + engine.setBeforeFlockReleaseHook(() -> { + if (finishCloseRuns.incrementAndGet() == 1) { + inFinishClose.countDown(); + try { + if (!releaseFinishClose.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release finishClose")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + } + }); + + // Phase 2: rotate onto the spare so the worker parks in the + // next install pass. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out close — cleanup ownership transfers to + // the worker's exit path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + + // Phase 4: release the pass. The worker exits its loop, wins + // the cleanup CAS, enters finishClose and parks in the hook — + // mid-cleanup, flock still held, completion unpublished. + releaseWorker.countDown(); + Assert.assertTrue("worker never entered the deferred finishClose", + inFinishClose.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, finishCloseRuns.get()); + Assert.assertFalse("completion must not be observable mid-finishClose", + engine.isCloseCompleted()); + + // Phase 5 — the contention under test: a retried close() while + // the worker is parked INSIDE finishClose. The worker loop has + // already exited (workerLoopExited=true precedes the deferred + // cleanups), so the short bounded join reaps the manager state + // and close() converges to the CAS — which it must LOSE, + // returning promptly without touching ring/watermark/flock. + engine.close(); + Assert.assertEquals( + "retried close() re-ran the terminal cleanup while the worker's " + + "deferred cleanup was mid-flight — ring/watermark/flock would " + + "be double-released", + 1, finishCloseRuns.get()); + Assert.assertFalse("retried close() must not publish completion on the worker's behalf", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot lock observable as released while the worker was still " + + "parked before its flock release"); + } catch (Exception expected) { + // good — flock still held by the parked cleanup. + } + + // Phase 6: let the worker finish. Completion publishes, the + // slot becomes acquirable, and the cleanup count stays at 1. + releaseFinishClose.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError("deferred cleanup never completed after release"); + } + Thread.sleep(1); + } + Assert.assertEquals(1, finishCloseRuns.get()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } + // A final close() takes the fast no-op path. + engine.close(); + Assert.assertEquals("post-completion close() must be a no-op", + 1, finishCloseRuns.get()); + if (hookErr.get() != null) { + throw new AssertionError("hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + engine.setBeforeFlockReleaseHook(null); + releaseWorker.countDown(); + releaseFinishClose.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Memory-mode twin of {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: + * {@code sfDir == null}, so there is no slot lock, no watermark and no + * segment files — but the ring's malloc'd native segments are still + * worker-reachable, so the timed-out close must take the same handoff + * path with every SF-only resource null. Pins that (a) the handoff + * branch tolerates null slotLock/watermark/sfDir without NPE, (b) the + * close stays incomplete while the worker can still touch the ring, and + * (c) the worker-exit cleanup completes the close and frees the ring's + * native memory (assertMemoryLeak is the leak oracle here). + */ + @Test(timeout = 30_000L) + public void testMemoryModeOwnedCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Memory mode: null sfDir, private owned manager — the exact + // shape non-SF async ingest uses. + CursorSendEngine engine = new CursorSendEngine(null, segSize); + SegmentManager manager = readManager(engine); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = readRing(engine); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: rotate onto the spare; the worker's next tick + // re-enters the (in-memory) install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out memory-mode close. The worker can still + // touch the ring's native memory, so the close must hand off + // and stay incomplete — releasing the ring here would be a + // use-after-free on the worker's install path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse( + "memory-mode close must stay incomplete while the worker is mid service pass", + engine.isCloseCompleted()); + + // Phase 4: release the worker; its exit path must run the + // deferred cleanup (null slotLock/watermark/sfDir) and flip + // completion with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred memory-mode cleanup never ran on manager-worker exit — " + + "the ring's native segments would leak for the process lifetime"); + } + Thread.sleep(1); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * An engine that owns its manager must use the whole-manager stop/join as + * its only quiescence barrier. Calling the per-ring barrier first would + * give a stuck worker two independent timeout budgets. + */ + @Test(timeout = 30_000L) + public void testOwnedManagerCloseSkipsPerRingQuiescenceWait() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/owned-slot"; + CursorSendEngine engine = new CursorSendEngine(slot, 4L * 1024 * 1024); + SegmentManager manager = readManager(engine); + AtomicBoolean perRingAwaited = new AtomicBoolean(); + try { + manager.setBeforeRingQuiescenceAwaitHook(() -> perRingAwaited.set(true)); + engine.close(); + Assert.assertTrue("owned engine close did not complete", engine.isCloseCompleted()); + Assert.assertFalse("owned engine close spent a separate per-ring wait budget", + perRingAwaited.get()); + } finally { + manager.setBeforeRingQuiescenceAwaitHook(null); + engine.close(); + } + }); + } + + /** + * Plain-positive path: after a normal close (worker quiesces promptly), + * a second engine must be able to acquire and use the same slot. + */ + @Test(timeout = 30_000L) + public void testSameSlotReacquirableAfterNormalClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/slot"; + CursorSendEngine first = new CursorSendEngine(slot, 4L * 1024 * 1024); + first.close(); + CursorSendEngine second = new CursorSendEngine(slot, 4L * 1024 * 1024); + try { + Assert.assertFalse("fully-drained close must leave no segments to recover", + second.wasRecoveredFromDisk()); + } finally { + second.close(); + } + }); + } + + private static SegmentManager readManager(CursorSendEngine engine) throws Exception { + Field field = CursorSendEngine.class.getDeclaredField("manager"); + field.setAccessible(true); + return (SegmentManager) field.get(engine); + } + + private static SegmentRing readRing(CursorSendEngine engine) throws Exception { + Field field = CursorSendEngine.class.getDeclaredField("ring"); + field.setAccessible(true); + return (SegmentRing) field.get(engine); + } + + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index f4de1ff2..c11fe1fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -197,6 +197,25 @@ public void testCloseIsIdempotent() throws Exception { }); } + @Test + public void testCallbackCreationFailurePrecedesOwnedManagerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> { + throw new OutOfMemoryError("simulated bound callback allocation failure"); + }); + try { + try { + new CursorSendEngine(tmpDir, 4096); + fail("expected callback allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("simulated bound callback allocation failure", expected.getMessage()); + } + } finally { + CursorSendEngine.setBeforeDeferredCloseCreationHook(null); + } + }); + } + @Test public void testConstructorFailureAfterOwnedManagerStartCleansResources() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java new file mode 100644 index 00000000..0a53e28e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java @@ -0,0 +1,256 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSendLoopBlockedSendCloseTest { + + @Test(timeout = 30_000L) + public void testCloseBreaksBlockedSendBeforeJoiningWorker() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(true); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "blocked-send-closer"); + closer.start(); + + Assert.assertTrue("close did not break the traffic path before joining", + client.trafficClosed.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("traffic path must close exactly once", 1, client.trafficCloseCount.get()); + Assert.assertEquals("the closer must break traffic, not the I/O worker", + closer, client.trafficCloseThread.get()); + Assert.assertTrue("blocked send did not observe traffic-path closure", + client.sendExited.await(5, TimeUnit.SECONDS)); + + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("close did not join the I/O worker", closer.isAlive()); + Assert.assertNull("close failed", closeFailure.get()); + Assert.assertNull("ordinary close must not manufacture a terminal error", loop.getTerminalError()); + + Thread ioThread = client.sendThread.get(); + Assert.assertNotNull(ioThread); + Assert.assertFalse("I/O worker must be dead when close returns", ioThread.isAlive()); + Assert.assertEquals("full client cleanup must run exactly once", 1, client.cleanupCount.get()); + Assert.assertEquals("the I/O worker must own cleanup before publishing exit", + ioThread, client.cleanupThread.get()); + Assert.assertFalse("close must not manufacture caller interruption", + closer.isInterrupted()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test(timeout = 30_000L) + public void testUnsupportedCustomTransportFailsWithoutDestroyingWorkerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(false); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + CountDownLatch closeEntered = new CountDownLatch(1); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + closeEntered.countDown(); + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "unsupported-transport-closer"); + closer.start(); + + Assert.assertTrue("close did not start", closeEntered.await(5, TimeUnit.SECONDS)); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("unsupported transport made close join indefinitely", closer.isAlive()); + Assert.assertTrue(closeFailure.get() instanceof LineSenderException); + Assert.assertTrue(closeFailure.get().getCause() instanceof UnsupportedOperationException); + Assert.assertEquals("unsupported cancellation must not release the blocked send", + 1L, client.sendExited.getCount()); + Assert.assertEquals("unsupported cancellation must not perform full cleanup", 0, client.cleanupCount.get()); + Assert.assertTrue("worker must retain resource ownership", client.sendThread.get().isAlive()); + + client.releaseSend.countDown(); + Assert.assertTrue("released send did not exit", client.sendExited.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("worker did not complete delegated cleanup", client.cleanupDone.await(5, TimeUnit.SECONDS)); + client.sendThread.get().join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker lingered after the custom transport was released", client.sendThread.get().isAlive()); + Assert.assertEquals(1, client.cleanupCount.get()); + Assert.assertNull("ordinary worker exit must remain non-terminal", loop.getTerminalError()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + Thread ioThread = client.sendThread.get(); + if (ioThread != null) { + ioThread.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static final class BlockingSendClient extends WebSocketClient { + private final AtomicBoolean cleanupClaimed = new AtomicBoolean(); + private final AtomicInteger cleanupCount = new AtomicInteger(); + private final CountDownLatch cleanupDone = new CountDownLatch(1); + private final AtomicReference cleanupThread = new AtomicReference<>(); + private final boolean trafficShutdownSupported; + private final CountDownLatch releaseSend = new CountDownLatch(1); + private final CountDownLatch sendEntered = new CountDownLatch(1); + private final CountDownLatch sendExited = new CountDownLatch(1); + private final AtomicReference sendThread = new AtomicReference<>(); + private final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final AtomicReference trafficCloseThread = new AtomicReference<>(); + private final CountDownLatch trafficClosed = new CountDownLatch(1); + + private BlockingSendClient(boolean trafficShutdownSupported) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.trafficShutdownSupported = trafficShutdownSupported; + } + + @Override + public void close() { + if (cleanupClaimed.compareAndSet(false, true)) { + cleanupCount.incrementAndGet(); + cleanupThread.set(Thread.currentThread()); + cleanupDone.countDown(); + } + super.close(); + } + + @Override + public void closeTraffic() { + if (!trafficShutdownSupported) { + throw new UnsupportedOperationException("custom transport has no safe cancellation capability"); + } + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficClosed.countDown(); + releaseSend.countDown(); + } + + @Override + public void sendBinary(long dataPtr, int length, int timeout) { + sendThread.set(Thread.currentThread()); + sendEntered.countDown(); + try { + if (!releaseSend.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("traffic path was not closed"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("I/O worker was interrupted instead of traffic being closed", e); + } finally { + sendExited.countDown(); + } + throw new LineSenderException("traffic path closed"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java new file mode 100644 index 00000000..0225d071 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java @@ -0,0 +1,469 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * TRUE-CANCELLATION proof (P1): {@code close()} must cancel a connect attempt + * that is blocked inside {@code reconnectFactory.reconnect(...)} (a black-holed + * native connect that neither unpark nor interrupt can cancel). The in-flight + * {@link WebSocketClient} is walk-local -- it is NOT the loop's {@code client} + * field (that field is {@code null} on the async-initial connect and points at + * the stale pre-drop client on a mid-flight reconnect) -- so before the fix + * {@code close()}'s field-client {@code closeTraffic()} could not reach it and + * {@code close()} blocked on the untimed {@code shutdownLatch.await()} for the + * whole OS SYN-retry window (~60-130s per endpoint). + *

    + * The fix publishes the in-flight client to a race-safe + * {@link CursorWebSocketSendLoop.ConnectCancellation} handle before the + * blocking connect, and {@code close()} breaks that client's traffic. These + * tests assert TRUE cancellation, not just return: the fake in-flight client's + * {@code closeTraffic()} is the ONLY thing that unblocks the parked + * {@code reconnect()}, and the tests witness that {@code closeTraffic()} was + * invoked (exactly once, by the closer thread) AND that {@code close()} + * returned. Deterministic latches only; {@code @Test(timeout=...)} fails a + * regression fast. + */ +public class CursorWebSocketSendLoopConnectPhaseCloseTest { + + /** + * Generous budget: with the fix, close() breaks the in-flight connect and + * returns well within this. Without the fix, close() blocks on the untimed + * shutdown latch for the entire (simulated) connect and this budget lapses. + */ + private static final long CLOSE_BUDGET_MILLIS = 5_000L; + + /** + * Small, injectable bounded-await backstop for the TOCTOU test. Shrunk from + * the production {@code DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS} (30 s) via + * {@link CursorWebSocketSendLoop#setShutdownAwaitTimeoutMillis(long)} so the + * timeout branch fires fast and deterministically -- no multi-second real + * wait -- while still leaving CLOSE_BUDGET_MILLIS of slack for the closer + * thread to return. + */ + private static final long BACKSTOP_MILLIS = 500L; + + /** + * Async-initial-connect path: the loop is built with a {@code null} client + * and a reconnect factory, so the I/O thread drives the very first connect + * through connectLoop while the {@code client} field stays {@code null}. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsAsyncInitialConnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(false); + } + + /** + * Mid-flight-reconnect path: an initial client is installed, its first + * receive fails, so connectLoop reconnects and blocks. The {@code client} + * field then points at the stale pre-drop client, never the in-flight one. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsMidFlightReconnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(true); + } + + /** + * BACKSTOP (bounded-await) proof: models the pathological, uninterruptible + * TOCTOU case where {@code cancel()}'s {@code closeTraffic()} CANNOT break + * the in-flight connect (as if cancellation landed after the pre-connect + * guard but before native fd creation, making closeTraffic() a no-op). The + * connect stays parked, so round-2 cancellation does NOT release the latch. + * Asserts {@code close()} STILL returns within the bounded backstop (does + * not hang) and surfaces the failed-stop contract: a loud + * {@link LineSenderException} whose message names the stalled I/O thread and + * the timeout, with client/engine teardown delegated to the I/O thread's + * own exit path (no destructive close under the still-live worker). + */ + @Test(timeout = 30_000L) + public void testCloseReturnsBoundedWhenCancellationCannotBreakConnect() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + // The ONLY thing that releases the parked connect. closeTraffic() + // deliberately does NOT touch it, modelling a cancel that cannot + // break the connect; the test itself counts it down, AFTER the + // backstop assertions, so the worker can unwind for the leak check. + final CountDownLatch release = new CountDownLatch(1); + + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + UninterruptibleInFlightClient c = new UninterruptibleInFlightClient(release); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Uninterruptible AND uncancellable: closeTraffic() does not + // release this park; only the test's release latch does. + c.awaitRelease(); + c.close(); + throw new LineSenderException("connect ended after release"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + // Shrink the bounded-await backstop so the timeout branch fires fast + // (no multi-second real wait); production uses the 30s default. + loop.setShutdownAwaitTimeoutMillis(BACKSTOP_MILLIS); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "backstop-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final UninterruptibleInFlightClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published", inFlight); + // cancel() DID attempt to break the connect -- but closeTraffic() + // is a no-op here, so the connect is still parked. + Assert.assertTrue("close() must have attempted to cancel the in-flight connect", + inFlight.trafficCloseCount.get() >= 1); + Assert.assertEquals("the parked connect must NOT have been released by closeTraffic()", + 1L, release.getCount()); + + // Decisive: close() returns within the bounded backstop even + // though cancellation could not break the connect. + Assert.assertTrue( + "close() must return within the bounded backstop (" + CLOSE_BUDGET_MILLIS + + "ms) even when cancellation cannot break the connect; instead it hung", + closedInTime); + + // Failed-stop contract: loud LineSenderException naming the + // stalled thread and the timeout. + final Throwable failure = closeFailure.get(); + Assert.assertNotNull("close() must loud-fail when the backstop times out", failure); + Assert.assertTrue("failed stop must be a LineSenderException, was " + failure, + failure instanceof LineSenderException); + Assert.assertTrue("message must name the stalled I/O thread: " + failure.getMessage(), + failure.getMessage().contains("cursor I/O thread did not stop")); + Assert.assertTrue("message must attribute the failed stop to the backstop timeout: " + + failure.getMessage(), + failure.getMessage().contains("timed out")); + // Teardown is delegated to the I/O thread's exit path, not done + // destructively under the live worker: no terminal manufactured. + Assert.assertNull("backstop timeout must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + // Now let the parked connect unwind so the worker exits and the + // leak check sees a clean teardown. Restore a generous backstop + // so the reconciling close() below awaits the worker's exit. + loop.setShutdownAwaitTimeoutMillis(CLOSE_BUDGET_MILLIS); + release.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + } + }); + } + + private void runConnectCancelledByCloseTraffic(boolean withInitialClient) throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Allocate the initial client INSIDE the leak-checked block so its + // native buffers are inside the baseline, not counted as over-free. + final WebSocketClient initialClient = withInitialClient ? new FailingInitialClient() : null; + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + + // A reconnect factory that models the real connect walk: it creates + // the client it is about to block on, PUBLISHES it to the loop's + // cancellation handle before the blocking connect, then parks in a + // way that ONLY closeTraffic() (i.e. cancellation) can release -- + // exactly a black-holed native connect that unpark/interrupt cannot + // cancel. + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + InFlightConnectClient c = new InFlightConnectClient(); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Black-holed connect: returns only once closeTraffic() breaks it. + c.awaitTrafficBreak(); + // Cancelled: dispose the half-built client (frees native + // buffers) and surface a transport error, exactly as the + // real connect walk's catch does on a broken connect. + c.close(); + throw new LineSenderException("connect cancelled by closeTraffic()"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "connect-phase-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final InFlightConnectClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published to the cancellation handle", + inFlight); + + // Let the worker finish unwinding regardless of outcome so the + // memory-leak check sees a clean teardown. + inFlight.trafficBroken.countDown(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + + Assert.assertNull("close() must not fail", closeFailure.get()); + Assert.assertTrue( + "close() must cancel the connect blocked inside reconnect() and return " + + "within " + CLOSE_BUDGET_MILLIS + "ms; instead it blocked on the " + + "untimed shutdown latch for the whole connect", + closedInTime); + // Decisive: closeTraffic() on the IN-FLIGHT client is what + // unblocked the parked connect -- and it was the closer thread, + // not the I/O worker, that broke it. + Assert.assertEquals( + "close() must break the in-flight connect's traffic exactly once", + 1, inFlight.trafficCloseCount.get()); + Assert.assertEquals( + "the closer thread (not the I/O worker) must cancel the in-flight connect", + closer, inFlight.trafficCloseThread.get()); + Assert.assertNull("ordinary connect-phase close must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + InFlightConnectClient inFlight = published.get(); + if (inFlight != null) { + inFlight.trafficBroken.countDown(); + } + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + if (initialClient != null) { + initialClient.close(); + } + } + }); + } + + /** + * The fake in-flight client. It never really connects: the factory parks in + * {@link #awaitTrafficBreak()} until {@link #closeTraffic()} releases it, + * which is the ONLY path out -- neither unpark nor interrupt can cancel it. + */ + private static final class InFlightConnectClient extends WebSocketClient { + final CountDownLatch trafficBroken = new CountDownLatch(1); + final AtomicInteger trafficCloseCount = new AtomicInteger(); + final AtomicReference trafficCloseThread = new AtomicReference<>(); + + private InFlightConnectClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void closeTraffic() { + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficBroken.countDown(); + } + + void awaitTrafficBreak() { + // Uninterruptible: only closeTraffic()'s countDown may release us, + // faithfully modelling a native connect that unpark/interrupt cannot + // cancel. + boolean interrupted = false; + while (trafficBroken.getCount() != 0L) { + try { + trafficBroken.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * The fake in-flight client for the BACKSTOP test. Its {@link #closeTraffic()} + * is a no-op w.r.t. releasing the park -- it only records that cancellation + * was attempted -- so a {@code close()}->{@code cancel()} CANNOT unblock the + * parked connect. This models the pathological uninterruptible/TOCTOU case; + * the ONLY release is the test-owned {@code release} latch, counted down + * AFTER the backstop assertions. + */ + private static final class UninterruptibleInFlightClient extends WebSocketClient { + final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final CountDownLatch release; + + private UninterruptibleInFlightClient(CountDownLatch release) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.release = release; + } + + @Override + public void closeTraffic() { + // Records the cancel attempt but does NOT release the parked + // connect: exactly a closeTraffic() that lands before the native fd + // exists (a no-op), so the connect blocks on regardless. + trafficCloseCount.incrementAndGet(); + } + + void awaitRelease() { + // Uninterruptible: neither unpark, interrupt, nor closeTraffic() + // releases us -- only the test's release latch. + boolean interrupted = false; + while (release.getCount() != 0L) { + try { + release.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * A pre-installed client whose first receive fails, driving the I/O loop + * into a reconnect so the in-flight client is published while the + * {@code client} field still points here (the mid-flight-reconnect path). + */ + private static final class FailingInitialClient extends WebSocketClient { + + private FailingInitialClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + throw new LineSenderException("initial wire dropped"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java new file mode 100644 index 00000000..c142dd6a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -0,0 +1,242 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Regression test for the completion/release ordering in + * {@code CursorSendEngine.finishClose()}: {@code closeCompleted} must be + * published strictly AFTER the slot flock release is confirmed, never + * before. + * + *

    The bug being pinned: {@code closeCompleted = true} used to be written + * before {@code slotLock} was closed. A pool thread could observe completion + * through {@code QwpWebSocketSender.isSlotLockReleased()}, free the slot + * index in {@code SenderPool.reprobeRetiredSlots()}, and admit a replacement + * sender whose {@code SlotLock.acquire} then collided with the still-open + * flock fd — a spurious "sf slot already in use" construction failure naming + * the process's own pid as the holder. + * + *

    The window between the publish and the {@code Files.close(fd)} is + * microseconds wide in the wild; {@code setBeforeFlockReleaseHook} makes it + * deterministic. The closing thread is parked between terminal cleanup and + * the flock release, and the test asserts from outside the window's two + * halves of the contract: + *

      + *
    • inside the window: {@code isCloseCompleted()} is still false AND a + * fresh {@code SlotLock.acquire} on the slot fails (proving the flock + * is genuinely held — i.e. reporting completion here would have been + * a lie);
      • + *
    • after the window: {@code isCloseCompleted()} is true AND a fresh + * {@code SlotLock.acquire} succeeds (completion still implies + * reusability — the reorder did not break the happy path).
      • + *
    + */ +public class EngineClosePublishAfterFlockReleaseTest { + + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-publish-order-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir == null) return; + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + @Test(timeout = 30_000L) + public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + CountDownLatch inWindow = new CountDownLatch(1); + CountDownLatch proceed = new CountDownLatch(1); + engine.setBeforeFlockReleaseHook(() -> { + inWindow.countDown(); + try { + // Bounded so a failed assertion on the main thread can + // never wedge the closer past the test timeout. + proceed.await(20, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + AtomicReference closerError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + engine.close(); + } catch (Throwable t) { + closerError.set(t); + } + }, "engine-closer"); + closer.start(); + + try { + assertTrue("closer thread never reached the cleanup/release window", + inWindow.await(10, TimeUnit.SECONDS)); + + // Inside the window: terminal cleanup has run, the flock has + // NOT been released. Completion must not be observable yet — + // this is the exact read a pool thread performs before + // freeing the slot index. + assertFalse("closeCompleted was published before the flock release; " + + "a pool observing this would free the slot index and a " + + "replacement sender would collide with the still-held flock", + engine.isCloseCompleted()); + + // Prove the window is real: the flock is genuinely still + // held, so acquisition by a "replacement engine" fails. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("scaffolding error: expected the slot flock to still be " + + "held inside the pre-release window, but a fresh " + + "SlotLock.acquire succeeded"); + } catch (IllegalStateException expected) { + // good — slot is locked, which is why completion must + // not have been published yet. + } + } finally { + proceed.countDown(); + closer.join(10_000L); + } + assertFalse("closer thread did not finish", closer.isAlive()); + assertNull("engine.close() threw", closerError.get()); + + // After the window: the release is confirmed, so completion must + // now be latched, and completion must still imply reusability. + assertTrue("closeCompleted must latch once the flock release is confirmed", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — completion implies the slot dir is acquirable. + } catch (IllegalStateException stillHeld) { + fail("closeCompleted reported true but the slot flock is still held: " + + stillHeld.getMessage()); + } + }); + } + + /** + * The error half of the publish-after-release contract: when + * {@code SlotLock.release()} reports failure (the OS refused the explicit + * unlock, so the flock may still be held), {@code closeCompleted} must + * stay false until a later close confirms release. A pool observing + * {@code isCloseCompleted() == false} keeps the slot retired while the + * flock is genuinely held, then recovers it once the retry completes. + *

    + * The lock's fd is swapped to a known-bad descriptor for a deterministic + * native unlock failure, then restored before retrying through the engine's + * public close API. + */ + @Test(timeout = 30_000L) + public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); + slotLockField.setAccessible(true); + SlotLock slotLock = (SlotLock) slotLockField.get(engine); + assertNotNull("disk-mode engine must hold a slot lock", slotLock); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + assertTrue("precondition: live flock fd", realFd >= 0); + try { + // A non-negative fd no process has open: close(2) fails EBADF, + // so finishClose's release() confirmation fails. + fdField.setInt(slotLock, 1_000_000_000); + engine.close(); + assertFalse( + "closeCompleted was published despite an unconfirmed flock release; " + + "a pool observing this would free the slot index while the " + + "flock fd is still open", + engine.isCloseCompleted()); + // The REAL flock is still held — the incomplete report is true. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — incomplete close really means "still locked". + } + + // Remove the injected fault. The retry must only re-attempt + // the retained fd release: ring/watermark cleanup stays + // exactly-once, while completion and slot reusability recover. + fdField.setInt(slotLock, realFd); + engine.close(); + assertTrue("retried close() must complete after the flock release succeeds", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — eventual completion implies reusable capacity. + } + } finally { + // If an assertion failed before the successful retry, restore + // and release the real fd so the test never leaks a flock. + if (!engine.isCloseCompleted()) { + fdField.setInt(slotLock, realFd); + assertTrue("restored fd must release cleanly", slotLock.release()); + } + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java new file mode 100644 index 00000000..0037437f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -0,0 +1,508 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FlockReleaseRetryDriverTest { + + private final List sfDirs = new ArrayList<>(); + + @After + public void tearDown() { + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + for (String sfDir : sfDirs) { + removeDir(sfDir); + } + } + + /** + * Driver-start failure must not strand retired capacity until process + * exit. {@code Sender.close()} is a one-shot no-op by contract, so the + * only recovery surface a pool has is its retired-slot probe + * ({@code isSlotLockReleased()}, called from the housekeeper tick and + * capacity-starved borrows) — that probe must re-arm the shared retry + * driver once thread creation works again. + */ + @Test(timeout = 30_000L) + public void testDriverStartFailureRecoversViaPoolProbe() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("probe-rearm"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(engine, true); + AtomicReference rearmedDriver = new AtomicReference<>(); + boolean recovered = false; + try { + setFd(slotLock, 1_000_000_000); + sender.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // Sender.close() is idempotent: repeat calls never reach the + // engine again, so they cannot restart the failed driver. + sender.close(); + assertEquals("no-op close must not retry the driver start", 1, starts.get()); + + // While the fault persists, each pool probe re-attempts the + // re-arm (and fails again) without publishing a release. + assertFalse("failed unlock must keep the slot reported as held", + sender.isSlotLockReleased()); + assertTrue("pool probe must re-attempt the driver start", + starts.get() >= 2); + + // The transient condition clears: thread creation works again + // (the failed start drained the queue, so the factory swap is + // legal) and the flock fd is back. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-rearmed-flock-release-retry"); + rearmedDriver.set(thread); + return thread; + }); + setFd(slotLock, realFd); + CountDownLatch released = new CountDownLatch(1); + engine.setSlotLockReleaseListener(released::countDown); + + // Production recovery surface: the pool re-probes the retired + // slot through isSlotLockReleased(). + sender.isSlotLockReleased(); + assertTrue("pool probe must re-arm the flock-release retry after " + + "a driver start failure", + released.await(10, TimeUnit.SECONDS)); + assertTrue(engine.isCloseCompleted()); + assertTrue("probe must expose the recovered release", + sender.isSlotLockReleased()); + recovered = true; + } finally { + Thread driver = rearmedDriver.get(); + if (driver != null) { + driver.join(10_000L); + } + if (!recovered) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresShareOneRetryThread() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int engineCount = 32; + AtomicInteger retryFailures = new AtomicInteger(); + AtomicInteger threadsCreated = new AtomicInteger(); + AtomicReference retryThreadRef = new AtomicReference<>(); + CountDownLatch retryFailuresObserved = new CountDownLatch(engineCount * 2); + CountDownLatch retryThreadStarted = new CountDownLatch(1); + CountDownLatch runRetryDriver = new CountDownLatch(1); + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(() -> { + retryFailures.incrementAndGet(); + retryFailuresObserved.countDown(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + threadsCreated.incrementAndGet(); + Thread thread = new Thread(() -> { + retryThreadStarted.countDown(); + try { + runRetryDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + task.run(); + }, "test-shared-flock-release-retry"); + retryThreadRef.set(thread); + return thread; + }); + + List engines = new ArrayList<>(); + List slotLocks = new ArrayList<>(); + List realFds = new ArrayList<>(); + boolean fdsRestored = false; + try { + for (int i = 0; i < engineCount; i++) { + String sfDir = newSfDir("persistent-" + i); + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + engines.add(engine); + slotLocks.add(slotLock); + realFds.add(realFd); + setFd(slotLock, 1_000_000_000); + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + if (i == 0) { + assertTrue("retry thread was not started", + retryThreadStarted.await(10, TimeUnit.SECONDS)); + } + } + + assertEquals("persistent failures must share one retry thread", + 1, threadsCreated.get()); + runRetryDriver.countDown(); + assertTrue("driver did not perform two failed rounds", + retryFailuresObserved.await(10, TimeUnit.SECONDS)); + assertTrue("driver did not retain persistent failures", + retryFailures.get() >= engineCount * 2); + for (CursorSendEngine engine : engines) { + assertFalse("failed releases must remain unpublished", + engine.isCloseCompleted()); + } + + restoreFds(slotLocks, realFds); + fdsRestored = true; + Thread retryThread = retryThreadRef.get(); + retryThread.join(10_000L); + assertFalse("shared retry thread retained lifecycle resources after drain", + retryThread.isAlive()); + for (CursorSendEngine engine : engines) { + assertTrue("driver must release every restored flock", + engine.isCloseCompleted()); + } + assertEquals("retries must not create another thread", + 1, threadsCreated.get()); + } finally { + if (!fdsRestored) { + restoreFds(slotLocks, realFds); + } + runRetryDriver.countDown(); + Thread retryThread = retryThreadRef.get(); + if (retryThread != null) { + retryThread.join(10_000L); + } + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + /** + * Schedule inspection for the shared driver's retry cadence: the + * inter-round park must grow exponentially from 100 ms and cap at 5 s, + * so a persistent unlock failure does not burn 10 syscalls per second + * per engine forever. The park override replaces the real park, so the + * test coordinates rounds without wall-clock waits. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffDoublesToCap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-backoff-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("backoff-cap"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + boolean fdRestored = false; + try { + setFd(slotLock, 1_000_000_000); + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + + // Eight failed rounds: enough to observe the full ramp and + // two capped parks. + for (int round = 1; round <= 8; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 8) { + proceed.release(); + } + } + + setFd(slotLock, realFd); + fdRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after the release succeeded", + driver.isAlive()); + assertTrue("restored flock must be released", engine.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(800_000_000L); + expected.add(1_600_000_000L); + expected.add(3_200_000_000L); + expected.add(5_000_000_000L); + expected.add(5_000_000_000L); + assertEquals("retry parks must double from 100ms and cap at 5s", + expected, parks); + } finally { + if (!fdRestored) { + setFd(slotLock, realFd); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + /** + * A successful release in a round is progress: the driver must reset its + * backoff to the base so the remaining engines are retried promptly + * while the failure condition is clearing. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffResetsOnProgress() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-reset-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engineA = new CursorSendEngine( + newSfDir("backoff-reset-a"), 4L * 1024 * 1024); + CursorSendEngine engineB = new CursorSendEngine( + newSfDir("backoff-reset-b"), 4L * 1024 * 1024); + SlotLock slotLockA = slotLock(engineA); + SlotLock slotLockB = slotLock(engineB); + int realFdA = fd(slotLockA); + int realFdB = fd(slotLockB); + boolean fdARestored = false; + boolean fdBRestored = false; + try { + setFd(slotLockA, 1_000_000_000); + setFd(slotLockB, 1_000_000_001); + engineA.close(); + engineB.close(); + + // Three failed rounds ramp the backoff to 400ms. + for (int round = 1; round <= 3; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 3) { + proceed.release(); + } + } + + // Engine A recovers; round 4 has one success and one failure, + // so its park must be back at the 100ms base. + setFd(slotLockA, realFdA); + fdARestored = true; + proceed.release(); + assertTrue("driver did not park after the mixed round", + parked.tryAcquire(10, TimeUnit.SECONDS)); + assertTrue("recovered engine must publish completion", + engineA.isCloseCompleted()); + + setFd(slotLockB, realFdB); + fdBRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after both releases succeeded", + driver.isAlive()); + assertTrue(engineB.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(100_000_000L); + assertEquals("a successful release must reset the backoff to base", + expected, parks); + } finally { + if (!fdARestored) { + setFd(slotLockA, realFdA); + } + if (!fdBRestored) { + setFd(slotLockB, realFdB); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + @Test(timeout = 30_000L) + public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("start-failure"), 4L * 1024 * 1024); + SlotLock slotLock = slotLock(engine); + int realFd = fd(slotLock); + try { + setFd(slotLock, 1_000_000_000); + engine.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // This also proves the failed driver cleared its queue: the + // setter rejects replacement while any engine remains queued. + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + engine.close(); + assertTrue("explicit close must recover after retry-thread start failure", + engine.isCloseCompleted()); + } finally { + if (!engine.isCloseCompleted()) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + setFd(slotLock, realFd); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + }); + } + + private static int fd(SlotLock slotLock) throws Exception { + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + return fdField.getInt(slotLock); + } + + private static void removeDir(String sfDir) { + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + private static void restoreFds(List slotLocks, List realFds) throws Exception { + for (int i = 0; i < slotLocks.size(); i++) { + SlotLock slotLock = slotLocks.get(i); + synchronized (slotLock) { + setFd(slotLock, realFds.get(i)); + } + } + } + + private static void setFd(SlotLock slotLock, int fd) throws Exception { + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + fdField.setInt(slotLock, fd); + } + + private static SlotLock slotLock(CursorSendEngine engine) throws Exception { + Field slotLockField = CursorSendEngine.class.getDeclaredField("slotLock"); + slotLockField.setAccessible(true); + return (SlotLock) slotLockField.get(engine); + } + + private String newSfDir(String suffix) { + String sfDir = Paths.get( + System.getProperty("java.io.tmpdir"), + "qdb-flock-release-retry-" + suffix + "-" + System.nanoTime() + ).toString(); + sfDirs.add(sfDir); + return sfDir; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 177ee5f6..cff8249b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -128,7 +128,7 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { assertTrue(expected.getMessage(), expected.getMessage().contains("pre-allocation failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); assertEquals("allocate must run exactly once", 1, ff.allocateCalls); assertEquals("fd must be closed on allocate failure", 1, ff.closeCalls); assertEquals("file must be removed on allocate failure", 1, ff.removeCalls); @@ -138,28 +138,31 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { } @Test - public void testCreateFailsCleanlyWhenOpenCleanRWReturnsMinusOne() throws Exception { + public void testCreateFailsCleanlyWhenExclusiveOpenReturnsMinusOne() throws Exception { TestUtils.assertMemoryLeak(() -> { String path = tmpDir + "/seg-noopen.sfa"; long sizeBytes = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 64; FaultyFilesFacade ff = new FaultyFilesFacade(); - ff.failOnOpenCleanRW = true; + ff.failOnOpenRWExclusive = true; try { MmapSegment.create(ff, path, 0L, sizeBytes).close(); - fail("expected MmapSegmentException from openCleanRW returning -1"); + fail("expected MmapSegmentException from openRWExclusive returning -1"); } catch (MmapSegmentException expected) { assertTrue(expected.getMessage(), - expected.getMessage().contains("openCleanRW failed")); + expected.getMessage().contains("exclusive create failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); - assertEquals("allocate must not run after openCleanRW failure", + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); + assertEquals("allocate must not run after exclusive-create failure", 0, ff.allocateCalls); assertEquals("close must not be called when no fd was opened", 0, ff.closeCalls); - assertEquals("remove must not be called when openCleanRW failed", + // With O_EXCL semantics a create failure can mean "path already + // exists and belongs to another lifecycle" -- create() must NOT + // unlink a file it never owned. + assertEquals("remove must not be called when exclusive create failed", 0, ff.removeCalls); - assertFalse("no file should exist when openCleanRW failed", + assertFalse("no file should exist when exclusive create failed", Files.exists(path)); }); } @@ -198,7 +201,7 @@ public void testCreateRepeatedAllocateFailuresDoNotAccumulateOrphans() throws Ex } assertEquals("no orphan files may survive repeated allocate failures", 0, survivors); - assertEquals(attempts, ff.openCleanRWCalls); + assertEquals(attempts, ff.openRWExclusiveCalls); assertEquals(attempts, ff.allocateCalls); assertEquals(attempts, ff.closeCalls); assertEquals(attempts, ff.removeCalls); @@ -495,9 +498,29 @@ private static final class FaultyFilesFacade implements FilesFacade { int closeCalls; boolean failOnAllocate; boolean failOnOpenCleanRW; + boolean failOnOpenRWExclusive; int openCleanRWCalls; + int openRWExclusiveCalls; int removeCalls; + @Override + public int openRWExclusive(String path) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(pathPtr); + } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 3d7c6a7c..cee8e41d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -204,6 +204,391 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the claim-time registration gate at the top of + * {@code SegmentManager.serviceRing}: a snapshot entry whose ring was + * deregistered BEFORE the worker claims it must be skipped entirely — + * never claimed as {@code inService}, never handed to + * {@code serviceRing0}. This is the exact guarantee + * {@code CursorSendEngine.close()} relies on when it releases the ring, + * watermark and slot flock right after + * {@code awaitRingQuiescence(ring) == true}: the deregistering thread may + * already be freeing those resources, so a stale snapshot entry must not + * be touched at all (spare install, watermark write, drainTrimmable, or + * path building under the slot dir). + *

    + * Deterministic shape: three rings A, B, C registered before start, so + * the worker's first snapshot is exactly [A, B, C]. The worker parks in + * A's spare-install pass; while it is parked, B is deregistered (and a + * quiescence barrier for B must pass immediately — no pass for B is in + * flight). After release the worker walks the rest of its stale + * snapshot: it must skip B and service C. Every serviced ring is + * recorded from inside the worker's own pass (via the trim-sync hook + * reading {@code inService}), so the assertion is exact — no timing + * grace, no sleeps. + */ + @Test(timeout = 15_000L) + public void testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + SegmentRing[] rings = new SegmentRing[3]; + String[] slots = new String[3]; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Rings the worker actually claimed and serviced, recorded from + // the worker thread itself at the trim-sync point every service + // pass reaches (spare needed or not). + java.util.Set serviced = + java.util.Collections.synchronizedSet( + java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>())); + boolean managerClosed = false; + try { + for (int i = 0; i < 3; i++) { + slots[i] = tmpDir + "/claim-skip-slot-" + i; + Assert.assertEquals(0, Files.mkdir(slots[i], Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create( + slots[i] + "/sf-initial.sfa", 0L, segSize); + rings[i] = new SegmentRing(initial, segSize); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.setBeforeTrimSyncHook(() -> { + try { + Object ring = readInServiceRing(manager); + if (ring != null) { + serviced.add(ring); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Register all three BEFORE start: the worker's first snapshot + // is [A, B, C] and every fresh ring wants a hot spare, so the + // install hook parks the worker inside A's pass. + manager.register(rings[0], slots[0]); + manager.register(rings[1], slots[1]); + manager.register(rings[2], slots[2]); + manager.start(); + Assert.assertTrue("worker did not reach A's install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // B is deregistered while its snapshot entry is still ahead of + // the worker's cursor. The quiescence barrier must pass at + // once: the in-flight pass is A's, not B's — this is the state + // in which an engine owner frees B's resources. + manager.deregister(rings[1]); + Assert.assertTrue("no pass for B is in flight — the barrier must pass immediately", + manager.awaitRingQuiescence(rings[1])); + + releaseWorker.countDown(); + + // Positive marker that the worker walked PAST B's snapshot + // slot: C sits after B in the same snapshot, so once C has + // been serviced, B's claim-or-skip decision has been made. + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!serviced.contains(rings[2])) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("worker never serviced ring C after release"); + } + Thread.sleep(1); + } + + Assert.assertTrue("ring A must have been serviced", serviced.contains(rings[0])); + Assert.assertFalse( + "worker claimed and serviced a snapshot entry that was deregistered " + + "before its pass started — the deregistering thread may already " + + "be releasing the ring/watermark/slot lock, so a stale snapshot " + + "entry must be skipped at claim time", + serviced.contains(rings[1])); + // Defense in depth: nothing may have been installed into the + // deregistered ring either. + Assert.assertNull("no hot spare may be installed into a deregistered ring", + readHotSpare(rings[1])); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("worker-side hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + manager.close(); + } + for (SegmentRing ring : rings) { + if (ring != null) { + ring.close(); + } + } + } + }); + } + + /** + * Pins the scratch-handoff half of the timed-out-close contract in + * isolation: after {@code close()} gives up on the bounded join and hands + * {@code pathScratch} ownership to the worker, the WORKER's exit block + * alone must free the native buffer — with no retried {@code close()} + * ever running. The sibling test + * ({@link #testCloseDoesNotFreePathScratchWhenWorkerStillAlive}) retries + * {@code close()} before asserting the free, so a regression that dropped + * the worker-side free (leaving reclaim to a retry nobody is required to + * make) would stay green there: production owners do NOT retry — a + * timed-out close returns to the pool and the worker is the only thread + * left that can reclaim the allocation. + */ + @Test(timeout = 15_000L) + public void testWorkerAloneFreesPathScratchAfterTimedOutClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/worker-frees-scratch-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: hands scratch ownership to the worker and + // returns. This is the last close() call this test makes. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Thread worker = readWorkerThread(manager); + Assert.assertTrue("worker must still be live after the timed-out close", + worker != null && worker.isAlive()); + Assert.assertTrue("scratch must still be allocated while the worker may use it", + readPathScratchImpl(manager) != 0L); + + // Release the pass. running=false already, so the worker + // finishes the pass and exits — its exit block must free the + // handed-over scratch buffer without ANY further caller action. + releaseWorker.countDown(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("worker never exited after release", worker.isAlive()); + Assert.assertEquals( + "worker exit block must free the handed-over path scratch — no retried " + + "close() runs in production after a timed-out close, so leaving " + + "the free to a retry leaks the native buffer for the process " + + "lifetime", + 0L, readPathScratchImpl(manager)); + + // Reap the dead thread for tidiness; must not double-free. + manager.close(); + Assert.assertNull("retried close must reap the exited worker", + readWorkerThread(manager)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.close(); + ring.close(); + } + }); + } + + /** + * Pins the {@link SegmentManager#deferUntilWorkerExit} handoff contract + * that {@code CursorSendEngine.close()}'s slot-ownership transfer depends + * on: + *
      + *
    • rejects the handoff ({@code false}) when no worker ever started — + * the caller must clean up inline;
      • + *
    • accepts it ({@code true}) while the worker is live-but-slow + * mid service pass after a timed-out close(), and runs the cleanup + * exactly when the worker exits — never while the pass is still in + * flight;
      • + *
    • rejects it again once the worker loop has exited/been reaped.
      • + *
    + */ + @Test(timeout = 15_000L) + public void testDeferUntilWorkerExitRunsCleanupAfterFinalPass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/defer-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch cleanupRan = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + // Never-started manager: no worker will ever run the cleanup, + // and none can touch a slot — the caller must do it inline. + Assert.assertFalse("never-started manager must reject the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: running=false, worker parked mid-pass — the + // exact state CursorSendEngine.close() hands ownership over in. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Assert.assertFalse("worker must not be reaped while parked mid-pass", + manager.isWorkerReaped()); + + Assert.assertTrue("live-but-slow worker must accept the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + Assert.assertEquals("cleanup must not run while the pass is still in flight", + 1, cleanupRan.getCount()); + + // Release the pass; the worker loop observes running=false, + // exits, and must run the deferred cleanup on its way out. + releaseWorker.countDown(); + Assert.assertTrue("cleanup never ran on worker exit", + cleanupRan.await(10, TimeUnit.SECONDS)); + + // Reap the exited worker, then: no live worker, no handoff. + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + managerClosed = true; + Assert.assertFalse("reaped manager must reject the handoff", + manager.deferUntilWorkerExit(() -> { + })); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + + /** + * Pins the {@link SegmentManager#awaitRingQuiescence} contract that + * {@code CursorSendEngine.close()} depends on: + *
      + *
    • returns {@code false} (never {@code true}) while a service pass + * for the ring is provably in flight and the timeout elapses;
      • + *
    • preserves a pending caller interrupt without aborting;
      • + *
    • returns {@code true} once the in-flight pass has finished.
      • + *
    + */ + @Test(timeout = 15_000L) + public void testAwaitRingQuiescenceBlocksWhileServicePassInFlight() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/quiesce-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + manager.deregister(ring); + manager.setWorkerJoinTimeoutMillis(50L); + Thread.currentThread().interrupt(); + Assert.assertFalse( + "awaitRingQuiescence returned true while the worker was parked " + + "inside the service pass for this ring", + manager.awaitRingQuiescence(ring)); + Assert.assertTrue("awaitRingQuiescence must preserve the caller's interrupt", + Thread.interrupted()); + + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + Assert.assertTrue( + "awaitRingQuiescence must return true once the in-flight pass finished", + manager.awaitRingQuiescence(ring)); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + @Test(timeout = 15_000L) public void testInterruptedCallerDoesNotAbandonReapableWorker() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -306,6 +691,24 @@ private static void cleanupRecursively(String dir) { } } + private static Object readHotSpare(SegmentRing ring) throws Exception { + Field f = SegmentRing.class.getDeclaredField("hotSpare"); + f.setAccessible(true); + return f.get(ring); + } + + private static Object readInServiceRing(SegmentManager manager) throws Exception { + Field inServiceF = SegmentManager.class.getDeclaredField("inService"); + inServiceF.setAccessible(true); + Object entry = inServiceF.get(manager); + if (entry == null) { + return null; + } + Field ringF = entry.getClass().getDeclaredField("ring"); + ringF.setAccessible(true); + return ringF.get(entry); + } + private static long readPathScratchImpl(SegmentManager manager) throws Exception { Field pathScratchF = SegmentManager.class.getDeclaredField("pathScratch"); pathScratchF.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java new file mode 100644 index 00000000..44a348f5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java @@ -0,0 +1,529 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + *******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class SegmentManagerCrashConsistencyTest { + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim acknowledged segments"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static void awaitValue(AtomicInteger value, int expected, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (value.get() < expected) { + if (System.nanoTime() > deadline) { + throw new AssertionError(message + " [expected=" + expected + ", actual=" + value.get() + ']'); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static SegmentRing createRing(String root, long segmentSize, long payload, int sealedCount) { + SegmentRing ring = null; + boolean success = false; + try { + ring = new SegmentRing(MmapSegment.create(root + "/sf-0.sfa", 0, segmentSize), segmentSize); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 1)); + for (int i = 1; i <= sealedCount; i++) { + ring.installHotSpare(MmapSegment.create(root + "/sf-" + i + ".sfa", i, segmentSize)); + Assert.assertEquals(i, ring.appendOrFsn(payload, 1)); + } + ring.installHotSpare(MmapSegment.create(root + "/sf-" + (sealedCount + 1) + ".sfa", + sealedCount + 1L, segmentSize)); + Assert.assertTrue(ring.acknowledge(sealedCount - 1L)); + success = true; + return ring; + } finally { + if (!success && ring != null) ring.close(); + } + } + + private static AckWatermark openWatermark(FilesFacade ff, String root) throws Exception { + Method method = AckWatermark.class.getDeclaredMethod("open", FilesFacade.class, String.class); + method.setAccessible(true); + return (AckWatermark) method.invoke(null, ff, root); + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(dir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + @Test(timeout = 15_000L) + public void testBackgroundTrimDurabilityOrder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-order-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync", + "segment-remove", "segment-remove", "dir-fsync"), ff.events); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testBarrierFailuresPreserveCrashSafetyAndRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{0, 1, 2, 5}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-fault-" + failAt + "-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root, failAt); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + ff.awaitFailure(); + if (failAt == 5) { + Assert.assertNotNull("post-unlink barrier failure committed ring removal", + ring.firstSealed()); + ff.advance(TimeUnit.SECONDS.toNanos(2)); + manager.wakeWorker(); + awaitTrimmed(ring); + } + manager.close(); + } + if (failAt <= 2) { + Assert.assertEquals("unlink started before covering barrier failed", 0, ff.removeCalls.get()); + Assert.assertNotNull("barrier failure removed ring bookkeeping", ring.firstSealed()); + } else { + Assert.assertNull("post-unlink barrier retry did not commit ring removal", ring.firstSealed()); + } + Assert.assertFalse("segment deletion began without a durable covering watermark", + ff.removeCalls.get() > 0 && !ff.durableWatermark); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + @Test(timeout = 15_000L) + public void testDiskTrimWithoutWatermarkIsPreserved() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-no-watermark-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 1); + CountDownLatch servicePass = new CountDownLatch(1); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.setBeforeTrimSyncHook(servicePass::countDown); + manager.register(ring, root, null); + manager.start(); + Assert.assertTrue("manager did not reach the trim service pass", + servicePass.await(5, TimeUnit.SECONDS)); + } + Assert.assertNotNull(ring.firstSealed()); + Assert.assertTrue(Files.exists(root + "/sf-0.sfa")); + } finally { + if (ring != null) ring.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testMoreThanOneQuantumBatchesBarriers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-batch-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 65); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 70L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(65, ff.removeCalls.get()); + Assert.assertEquals("watermark must sync once per quantum", 2, ff.msyncCalls.get()); + Assert.assertEquals("directory barriers must be twice per quantum", 4, ff.dirSyncCalls.get()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testPersistentFailureDoesNotStarveSibling() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String base = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-sibling-" + System.nanoTime()).toString(); + String badRoot = base + "/bad"; + String goodRoot = base + "/good"; + Assert.assertEquals(0, Files.mkdir(base, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(badRoot, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(goodRoot, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(badRoot, "dir-fsync", 0); + AckWatermark badWatermark = null; + AckWatermark goodWatermark = null; + SegmentRing badRing = null; + SegmentRing goodRing = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + badRing = createRing(badRoot, segmentSize, payload, 1); + goodRing = createRing(goodRoot, segmentSize, payload, 1); + badWatermark = openWatermark(ff, badRoot); + goodWatermark = openWatermark(ff, goodRoot); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 16L, ff, ff::ticks)) { + manager.register(badRing, badRoot, badWatermark); + manager.register(goodRing, goodRoot, goodWatermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "bad sibling failure was not attempted"); + awaitTrimmed(goodRing); + Assert.assertNotNull("failed sibling unexpectedly trimmed", badRing.firstSealed()); + } + } finally { + if (badRing != null) badRing.close(); + if (goodRing != null) goodRing.close(); + if (badWatermark != null) badWatermark.close(); + if (goodWatermark != null) goodWatermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(badRoot); + removeRecursive(goodRoot); + removeRecursive(base); + } + }); + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresBackOffAndRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String[] failures = {"watermark-msync", "segment-remove", "post-dir-fsync"}; + for (String failure : failures) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-" + failure + '-' + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade( + root, failure, "segment-remove".equals(failure) ? 2 : 0); + if ("watermark-msync".equals(failure)) { + ff.ticks.set(Long.MAX_VALUE - 2_000_000L); + } + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + AtomicInteger logs = new AtomicInteger(); + AtomicInteger passes = new AtomicInteger(); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.setBeforeTrimSyncHook(passes::incrementAndGet); + manager.setRetryLogHook(logs::incrementAndGet); + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "initial persistent failure was not attempted"); + awaitValue(logs, 1, "initial failure transition was not logged"); + if ("segment-remove".equals(failure)) { + Assert.assertEquals("successful unlink prefix was not committed", 1L, + ring.firstSealed().baseSeq()); + Assert.assertFalse(Files.exists(root + "/sf-0.sfa")); + Assert.assertTrue(Files.exists(root + "/sf-1.sfa")); + } + + int operations = ff.operationCalls(); + for (int i = 0; i < 4; i++) { + int nextPass = passes.get() + 1; + manager.wakeWorker(); + awaitValue(passes, nextPass, "deferred retry pass did not run"); + } + Assert.assertEquals("deferred passes performed filesystem work", + operations, ff.operationCalls()); + Assert.assertEquals("deferred passes emitted logs", 1, logs.get()); + + long delay = 4_000_000L; + for (int attempt = 2; attempt <= 11; attempt++) { + int nextPass = passes.get() + 1; + operations = ff.operationCalls(); + ff.advance(delay - 1); + manager.wakeWorker(); + awaitValue(passes, nextPass, "pre-deadline pass did not run"); + Assert.assertEquals("pre-deadline pass performed filesystem work", + operations, ff.operationCalls()); + nextPass = passes.get() + 1; + ff.advance(1); + manager.wakeWorker(); + awaitValue(ff.failureCalls, attempt, "retry deadline did not enable attempt"); + awaitValue(passes, nextPass, "retry pass did not run"); + Assert.assertEquals("persistent failure log was not throttled", 1, logs.get()); + delay = Math.min(delay * 2, 1_024_000_000L); + } + + ff.failureEnabled = false; + ff.advance(delay); + manager.wakeWorker(); + awaitTrimmed(ring); + awaitValue(logs, 2, "recovery transition was not logged"); + Assert.assertEquals("recovery transition was not logged once", 2, logs.get()); + } + Assert.assertNull(ring.firstSealed()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + private static final class OrderingFilesFacade implements FilesFacade { + private final AtomicInteger dirSyncCalls = new AtomicInteger(); + private final List events = new ArrayList<>(); + private final int failAt; + private final AtomicInteger failureCalls = new AtomicInteger(); + private final AtomicInteger msyncCalls = new AtomicInteger(); + private final AtomicInteger removeCalls = new AtomicInteger(); + private final String persistentEvent; + private final int persistentRemoveOrdinal; + private final String root; + private final AtomicLong ticks = new AtomicLong(); + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private boolean expectPreDirSync = true; + private volatile boolean failureEnabled; + private volatile boolean failureObserved; + private int watermarkFd = -1; + + private OrderingFilesFacade(String root) { + this(root, -1); + } + + private OrderingFilesFacade(String root, int failAt) { + this.root = root; + this.failAt = failAt; + this.persistentEvent = null; + this.persistentRemoveOrdinal = 0; + } + + private OrderingFilesFacade(String root, String persistentEvent, int persistentRemoveOrdinal) { + this.root = root; + this.failAt = -1; + this.persistentEvent = persistentEvent; + this.persistentRemoveOrdinal = persistentRemoveOrdinal; + this.failureEnabled = true; + } + + private void advance(long nanos) { + ticks.addAndGet(nanos); + } + + private void awaitFailure() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!failureObserved) { + if (System.nanoTime() > deadline) { + throw new AssertionError("injected barrier failure was not reached"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private boolean fail(String event) { + events.add("post-dir-fsync".equals(event) ? "dir-fsync" : event); + boolean persistentMatch = failureEnabled && event.equals(persistentEvent) + && (!"segment-remove".equals(event) || persistentRemoveOrdinal <= removeCalls.get()); + boolean failed = eventIndex++ == failAt || persistentMatch; + if (failed) { + failureCalls.incrementAndGet(); + failureObserved = true; + } + return failed; + } + + private int operationCalls() { + return msyncCalls.get() + dirSyncCalls.get() + removeCalls.get(); + } + + private long ticks() { + return ticks.get(); + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { + if (active && fd == watermarkFd && fail("watermark-fsync")) return -1; + return INSTANCE.fsync(fd); + } + @Override public int fsyncDir(String dir) { + if (active && root.equals(dir)) { + dirSyncCalls.incrementAndGet(); + String event = expectPreDirSync ? "dir-fsync" : "post-dir-fsync"; + boolean failed = fail(event); + if (expectPreDirSync) { + if (!failed) { + durableWatermark = true; + expectPreDirSync = false; + } + } else { + expectPreDirSync = true; + if (!failed) durableSegments = false; + } + if (failed) return -1; + } + return INSTANCE.fsyncDir(dir); + } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int msync(long addr, long len, boolean async) { + if (active) { + msyncCalls.incrementAndGet(); + if (fail("watermark-msync")) return -1; + } + return INSTANCE.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + removeCalls.incrementAndGet(); + if (fail("segment-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java new file mode 100644 index 00000000..1bb6b899 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +/** + * Standalone comparison of the service-pass ownership barriers used by + * SegmentManager before and after its per-entry state change. This deliberately + * has no pass/fail threshold: it reports target-JVM costs while the production + * state machine and lifecycle tests establish correctness. + * + * 
    + * mvn -pl core test-compile
+ * mvn -pl core exec:java -Dexec.classpathScope=test \
+ *   -Dexec.mainClass=io.questdb.client.test.cutlass.qwp.client.sf.cursor.SegmentManagerPassBarrierBenchmark \
+ *   -Dexec.args="--passes=10000000"
+ *
    + */ +public final class SegmentManagerPassBarrierBenchmark { + + private static final Object MONITOR = new Object(); + private static final AtomicIntegerFieldUpdater STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(Entry.class, "state"); + private static volatile long checksum; + private static volatile int serviceProbe; + + public static void main(String[] args) { + int passes = 10_000_000; + for (String arg : args) { + if (arg.startsWith("--passes=")) { + passes = Integer.parseInt(arg.substring("--passes=".length())); + } else { + throw new IllegalArgumentException("unknown argument: " + arg); + } + } + if (passes < 1) { + throw new IllegalArgumentException("passes must be positive"); + } + + int warmup = Math.max(100_000, passes / 10); + for (int rings : new int[]{1, 32, 256}) { + measureMonitor(rings, warmup); + measureAtomic(rings, warmup); + long monitorNanos = measureMonitor(rings, passes); + long atomicNanos = measureAtomic(rings, passes); + System.out.printf( + "rings=%d passes=%d monitor(two enters)=%.2f ns/pass atomic(two CAS)=%.2f ns/pass ratio=%.2f%n", + rings, + passes, + (double) monitorNanos / passes, + (double) atomicNanos / passes, + (double) monitorNanos / atomicNanos); + } + System.out.println("checksum=" + checksum); + } + + private static Entry[] entries(int count) { + Entry[] entries = new Entry[count]; + for (int i = 0; i < count; i++) { + entries[i] = new Entry(); + } + return entries; + } + + private static long measureAtomic(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + if (!STATE_UPDATER.compareAndSet(entry, 0, 1)) { + throw new AssertionError("claim failed"); + } + service(entry); + if (!STATE_UPDATER.compareAndSet(entry, 1, 0)) { + throw new AssertionError("completion failed"); + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static long measureMonitor(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + synchronized (MONITOR) { + entry.state = 1; + } + service(entry); + synchronized (MONITOR) { + entry.state = 0; + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static void service(Entry entry) { + // Models the non-trivial serviceRing0 call between claim and complete + // and prevents the JVM from coarsening both monitor regions into one. + serviceProbe = entry.state; + } + + private static final class Entry { + volatile int state; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java index 0c102fce..a00faea4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; @@ -36,6 +37,10 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -73,6 +78,65 @@ public void tearDown() { Files.remove(tmpDir); } + @Test + public void testAckBatchDoesNotDelaySiblingSpareProvisioning() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int sealedCount = 128; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + CountDownLatch busyDrained = new CountDownLatch(1); + CountDownLatch siblingInstall = new CountDownLatch(1); + AtomicInteger trimPass = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + try (SegmentRing busyRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentRing siblingRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60))) { + for (int i = 0; i < sealedCount; i++) { + assertEquals(i, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + } + assertEquals(sealedCount, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + busyRing.acknowledge(sealedCount - 1L); + + manager.register(busyRing, null); + manager.register(siblingRing, null); + manager.setBeforeInstallSyncHook(() -> { + try { + assertEquals("the busy ring must stop at the 64-segment quantum before sibling service", + 64, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + siblingInstall.countDown(); + } + }); + manager.setBeforeTrimSyncHook(() -> { + if (trimPass.incrementAndGet() == 4) { + try { + assertEquals("the next fair pass must fully drain the busy ring", + 0, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + busyDrained.countDown(); + } + } + }); + manager.start(); + assertTrue("manager did not reach sibling spare provisioning", + siblingInstall.await(10, TimeUnit.SECONDS)); + assertTrue("manager parked instead of immediately rescheduling the remaining ACK batch", + busyDrained.await(10, TimeUnit.SECONDS)); + if (failure.get() != null) { + throw new AssertionError("ACK batch fairness witness failed", failure.get()); + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testManagerProvisionsSpareWithinPollingTick() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -136,10 +200,12 @@ public void testManagerTrimsAckedSegmentFiles() throws Exception { String seg0Path = tmpDir + "/0000000000000000.sfa"; MmapSegment seg0 = MmapSegment.create(seg0Path, 0, segSize); long buf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L)) { + assertTrue("watermark must open", watermark != null); mgr.start(); - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Fill seg0 (2 frames) and force rotation by appending a third. for (int i = 0; i < 2; i++) ring.appendOrFsn(buf, 32); @@ -170,11 +236,13 @@ public void testMaxTotalBytesCapBlocksProvisioningUntilTrimFrees() throws Except long cap = 3 * segSize; MmapSegment seg0 = MmapSegment.create(tmpDir + "/0000000000000000.sfa", 0, segSize); long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L, cap)) { + assertTrue("watermark must open", watermark != null); mgr.start(); // register seeds totalBytes = 1*segSize (initial active). - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Manager provisions spare 1 → totalBytes = 2*segSize. assertTrue(waitFor(() -> !ring.needsHotSpare(), 2000)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java index 8cc05407..e622ac1b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java @@ -43,7 +43,6 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -186,11 +185,8 @@ public void testTrimPathDoesNotDoubleSubtractAfterDeregister() throws Exception + "`totalBytes -= sz` on a stillRegistered re-check " + "under the same lock that covers deregister.", 0L, observed); - assertFalse("stale SegmentManager snapshot skipped drainTrimmable() " - + "after deregister and left a fully-acked sealed " - + "segment on disk. The registration guard should " - + "protect watermark/accounting only; trim ownership " - + "transfer must still close and unlink " + activePath, + assertTrue("deregister before the durable barrier must preserve the segment " + + "for owner-side quiescent cleanup " + activePath, Files.exists(activePath)); } finally { mgr.setBeforeTrimSyncHook(null); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java new file mode 100644 index 00000000..67e0be65 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -0,0 +1,350 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SegmentManagerUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-segmgr-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test + public void testEnumerationFindNextFailureRefusesGenerationAllocation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/enumeration"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String lowerName = "sf-0000000000000000.sfa"; + String lowerPath = dir + "/" + lowerName; + String higherPath = dir + "/sf-0000000000000007.sfa"; + MmapSegment lower = MmapSegment.create(lowerPath, 0L, segmentSize); + lower.close(); + MmapSegment initial = MmapSegment.create(higherPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + byte[] originalHigher = java.nio.file.Files.readAllBytes(Paths.get(higherPath)); + FailingFilesFacade facade = new FailingFilesFacade(null, dir, lowerName); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 4, facade)) { + try { + manager.register(ring, dir); + Assert.fail("register accepted a partially enumerated SF directory"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("could not fully enumerate")); + } + Assert.assertTrue("fault did not occur after the lower generation was observed", + facade.partialLowerObserved); + Assert.assertTrue("failed enumeration cursor was not closed", facade.partialFindClosed); + Assert.assertEquals("enumeration failure must not allocate or truncate a path", + 0, facade.openCleanCalls); + Assert.assertTrue("higher generation disappeared", Files.exists(higherPath)); + Assert.assertArrayEquals("partial enumeration changed the unseen higher generation", + originalHigher, java.nio.file.Files.readAllBytes(Paths.get(higherPath))); + } finally { + ring.close(); + } + }); + } + + @Test(timeout = 15_000L) + public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/unlink"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String failedPath = dir + "/sf-0000000000000000.sfa"; + String activePath = dir + "/sf-0000000000000001.sfa"; + MmapSegment initial = MmapSegment.create(failedPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + AckWatermark watermark = AckWatermark.open(dir); + Assert.assertNotNull(watermark); + watermark.write(-1L); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + fill(payload, 32, (byte) 0x11); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 32)); + ring.installHotSpare(MmapSegment.create(activePath, 1L, segmentSize)); + Assert.assertEquals(1L, ring.appendOrFsn(payload, 32)); + ring.acknowledge(0L); + byte[] original = java.nio.file.Files.readAllBytes(Paths.get(failedPath)); + + FailingFilesFacade facade = new FailingFilesFacade(failedPath, null, null); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + manager.register(ring, dir, watermark); + manager.start(); + Assert.assertTrue("manager never attempted the injected unlink", + facade.removeAttempted.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("failed unlink must retain conservative registered bytes", + ring.totalSegmentBytes(), readTotalBytes(manager)); + } + + Assert.assertTrue("failed unlink path must remain observable", Files.exists(failedPath)); + Assert.assertTrue("failed unlink changed the acknowledged segment bytes", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + Assert.assertNotNull("failed unlink removed the segment from ring bookkeeping", + ring.firstSealed()); + Assert.assertEquals(failedPath, ring.firstSealed().path()); + Assert.assertEquals("failed unlink must remain covered by the durable cumulative watermark", + 0L, watermark.read()); + + fill(payload, 32, (byte) 0x5A); + Assert.assertEquals("non-DEDUP successor must continue at the next FSN", + 2L, ring.appendOrFsn(payload, 32)); + String successorPath = ring.getActive().path(); + Assert.assertNotEquals("successor reused the acknowledged path", + failedPath, successorPath); + Assert.assertEquals(dir + "/sf-0000000000000002.sfa", successorPath); + Assert.assertTrue("successor segment was not created", Files.exists(successorPath)); + Assert.assertTrue("distinct non-DEDUP payload was not written to the successor", + containsRun(java.nio.file.Files.readAllBytes(Paths.get(successorPath)), + (byte) 0x5A, 32)); + Assert.assertTrue("successor write overwrote the failed-unlink segment", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + + try (SegmentManager retryManager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + retryManager.register(ring, dir, watermark); + retryManager.start(); + retryManager.wakeWorker(); + awaitRetryCommit(failedPath, watermark); + } + MmapSegment firstSealed = ring.firstSealed(); + Assert.assertTrue("successful retry retained the acknowledged segment", + firstSealed == null || !failedPath.equals(firstSealed.path())); + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + ring.close(); + watermark.close(); + } + }); + } + + private static void awaitRetryCommit(String failedPath, AckWatermark watermark) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (Files.exists(failedPath) || watermark.read() != 0L) { + if (System.nanoTime() > deadline) { + throw new AssertionError("unlink retry did not remove the file and advance the watermark"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static boolean containsRun(byte[] bytes, byte value, int length) { + int run = 0; + for (byte b : bytes) { + run = b == value ? run + 1 : 0; + if (run == length) { + return true; + } + } + return false; + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static long readTotalBytes(SegmentManager manager) throws Exception { + Field field = SegmentManager.class.getDeclaredField("totalBytes"); + field.setAccessible(true); + Field lockField = SegmentManager.class.getDeclaredField("lock"); + lockField.setAccessible(true); + Object lock = lockField.get(manager); + synchronized (lock) { + return field.getLong(manager); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static final class FailingFilesFacade implements FilesFacade { + private static final long PARTIAL_FIND_PTR = Long.MAX_VALUE; + private final String enumerationFailureDir; + private final String partialLowerName; + private final String unlinkFailurePath; + private final CountDownLatch removeAttempted = new CountDownLatch(1); + private int openCleanCalls; + private boolean partialFindClosed; + private long partialFindNamePtr; + private boolean partialLowerObserved; + private int unlinkFailuresRemaining = 1; + + private FailingFilesFacade( + String unlinkFailurePath, + String enumerationFailureDir, + String partialLowerName + ) { + this.unlinkFailurePath = unlinkFailurePath; + this.enumerationFailureDir = enumerationFailureDir; + this.partialLowerName = partialLowerName; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + INSTANCE.freeNativePath(partialFindNamePtr); + partialFindNamePtr = 0L; + partialFindClosed = true; + } else { + INSTANCE.findClose(findPtr); + } + } + @Override + public long findFirst(String dir) { + if (dir.equals(enumerationFailureDir)) { + partialFindNamePtr = INSTANCE.allocNativePath(partialLowerName); + return PARTIAL_FIND_PTR; + } + return INSTANCE.findFirst(dir); + } + @Override + public long findName(long findPtr) { + return findPtr == PARTIAL_FIND_PTR ? partialFindNamePtr : INSTANCE.findName(findPtr); + } + @Override + public int findNext(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + partialLowerObserved = true; + return -1; + } + return INSTANCE.findNext(findPtr); + } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int openCleanRW(String path) { + openCleanCalls++; + return INSTANCE.openCleanRW(path); + } + @Override + public int openCleanRW(long pathPtr) { + openCleanCalls++; + return INSTANCE.openCleanRW(pathPtr); + } + @Override + public int openRW(String path) { return INSTANCE.openRW(path); } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (path.equals(unlinkFailurePath) && unlinkFailuresRemaining > 0) { + unlinkFailuresRemaining--; + removeAttempted.countDown(); + return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java new file mode 100644 index 00000000..21116e68 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -0,0 +1,827 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +/** + * Regression tests for the SF recovery fail-open findings: directory + * enumeration errors, per-file open/mmap errors and boundary (leading / + * trailing) segment loss must fail startup without mutating the slot, while + * positively-identified corruption is quarantined only after the surviving + * chain validates. Also pins the crash-window states of the manifest + * protocol (fresh start, rotation, clean-drain) as recoverable. + */ +public class SegmentRecoveryIntegrityTest { + private static final String MANIFEST_NAME = "sf-manifest.bin"; + private static final long SEGMENT_SIZE = 64 * 1024; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-integrity-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + // ------------------------------------------------------------------ + // Operational failures must fail closed without touching a byte. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the SF directory cannot be enumerated"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testPartialFindNextFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int calls; + + @Override + public int findNext(long findPtr) { + // First advance succeeds, then the listing "fails" the way + // a readdir I/O error does. Recovery must treat the + // partial listing as fatal, not as end-of-directory. + return ++calls >= 2 ? -1 : super.findNext(findPtr); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the directory listing is incomplete"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not fully enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testOpenRWFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public int openRW(String path) { + // EMFILE/EACCES-style failure on a perfectly valid file. + return victim.equals(path) ? -1 : super.openRW(path); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be opened"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testMmapFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int victimFd = Integer.MIN_VALUE; + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (victim.equals(path)) { + victimFd = fd; + } + return fd; + } + + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + if (fd == victimFd) { + return Files.FAILED_MMAP_ADDRESS; + } + return super.mmap(fd, len, offset, flags, memoryTag); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be mapped"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testUnsupportedVersionFailsRecoveryWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String foreign = tmpDir + "/sf-0000000000000001.sfa"; + writeRawSegmentHeader(foreign, MmapSegment.FILE_MAGIC, (byte) 99, 2L); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("a segment written by a different client version must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + // NOT renamed .corrupt: the file belongs to the client build that + // can read it; recovery keeps the slot intact for that writer. + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Corruption is quarantined, but never before validation decides. + // ------------------------------------------------------------------ + + @Test + public void testCorruptStrayFileQuarantinedInLegacyDirAndSiblingsRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + String stray = tmpDir + "/zz-stray.sfa"; + writeRawSegmentHeader(stray, 0xDEADBEEF, (byte) 1, 0L); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("valid chain must recover around a corrupt stray file", ring); + try { + Assert.assertEquals("all five durable frames must be recovered", + 4, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertFalse("corrupt stray must be quarantined away from the .sfa scan", + Files.exists(stray)); + Assert.assertTrue("quarantine preserves the bytes as evidence", + Files.exists(stray + ".corrupt")); + Assert.assertTrue("legacy recovery must migrate the slot to the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptChainSegmentWithManifestFailsWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String seg = tmpDir + "/sf-initial.sfa"; + writeSegmentWithFrames(seg, 0, 3); + // Migrate once so the manifest exists and the segment is flagged. + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + // Bit-rot the magic in place. + overwriteInt(seg, 0, 0xBADC0DE); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("losing the only chain segment to corruption must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "corrupt"); + } + // Deferred quarantine: a FAILED recovery must not have renamed, + // deleted, or otherwise mutated anything. + assertDirUnchanged(before); + }); + } + + @Test + public void testFlaggedSegmentWithDeletedManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.remove(tmpDir + "/" + MANIFEST_NAME)); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a manifest-required segment without a manifest must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + // ------------------------------------------------------------------ + // Boundary evasion: missing leading/trailing segments must be caught. + // ------------------------------------------------------------------ + + @Test + public void testMissingActiveSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Chain [0..2) exists but the manifest says the active starts at 2. + // Pre-manifest recovery would silently promote the highest present + // segment and hand out overlapping FSNs. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing trailing/active segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + }); + } + + @Test + public void testMissingHeadSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The manifest promises a head at base 0, but only [2..5) survived. + // Pre-manifest recovery would pass contiguity on the remainder and + // silently lose the leading rows. + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing leading/head segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF head"); + } + }); + } + + @Test + public void testInteriorGapStillFailsRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000002.sfa", 5, 2); + writeManifest(1, 0, 5); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an interior FSN gap must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "FSN gap"); + } + }); + } + + // ------------------------------------------------------------------ + // Legal crash-window states must recover without operator action. + // ------------------------------------------------------------------ + + @Test + public void testFreshStartCrashWithTwoEmptyBaseZeroSegmentsRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Fresh engine start provisions sf-initial plus a hot spare, both + // empty at baseSeq 0. A process kill in that window must not brick + // the slot on "ambiguous" empties. + MmapSegment a = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + a.close(); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + MmapSegment b = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 0, SEGMENT_SIZE); + b.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("two equivalent empty segments must not brick recovery", ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + Assert.assertEquals(0, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + Assert.assertEquals("the redundant empty must have been cleaned up", + 1, countSfaFiles()); + }); + } + + @Test + public void testRotationCrashWindowEmptyActiveAtChainEndRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Crash after rotation committed (manifest fsync'd, spare header + // synced) but before any frame reached the new active: sealed + // chain [0..2) plus an empty active at base 2. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + spare.close(); + writeManifest(1, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("freshly-rotated empty active is a legal crash state", ring); + try { + Assert.assertEquals(1, ring.publishedFsn()); + Assert.assertEquals(2, ring.getActive().baseSeq()); + Assert.assertNotNull(ring.firstSealed()); + Assert.assertEquals(0, ring.firstSealed().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testDrainWindowManifestWithoutSegmentsRecoversEmptyAndRemovesManifest() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain close unlinks the last segment before the manifest; + // a crash between the two leaves this state. + writeManifest(3, 7, 9); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("segment-less slot must recover as EMPTY", ring); + Assert.assertFalse("the stale manifest must be discarded", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Engine crash between creating sf-initial.sfa (unflagged) and + // creating the manifest: recovers via legacy migration. + MmapSegment initial = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + initial.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("legacy migration must create the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + // ------------------------------------------------------------------ + // Engine level: enumeration failure must not truncate the durable log. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureCannotCreateInitialSegment() throws Exception { + TestUtils.assertMemoryLeak(() -> { + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + Assert.assertFalse("startup failure created sf-initial.sfa", + Files.exists(tmpDir + "/sf-initial.sfa")); + } + } finally { + manager.close(); + } + }); + } + + @Test + public void testFindFirstFailureDoesNotTruncateExistingLog() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The original failure mode: enumeration error -> treated as empty + // -> fresh start openCleanRW(O_TRUNC) destroys the durable log. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 4); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + // expected + } + } finally { + manager.close(); + } + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + /** Creates a real segment at {@code path} and appends {@code frames} 64-byte frames. */ + private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment segment = MmapSegment.create(path, baseSeq, SEGMENT_SIZE); + try { + for (int i = 0; i < frames; i++) { + for (int b = 0; b < 64; b++) { + Unsafe.getUnsafe().putByte(buf + b, (byte) (i * 31 + b)); + } + Assert.assertTrue("test frame must fit", segment.tryAppend(buf, 64) >= 0); + } + } finally { + segment.close(); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } + + /** Writes a raw 64-byte pseudo segment header (+zero padding) for corruption tests. */ + private static void writeRawSegmentHeader(String path, int magic, byte version, long baseSeq) { + int fd = Files.openCleanRW(path); + Assert.assertTrue("could not create " + path, fd >= 0); + try { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, magic); + Unsafe.getUnsafe().putByte(buf + 4, version); + Unsafe.getUnsafe().putLong(buf + 8, baseSeq); + Assert.assertEquals(64, Files.write(fd, buf, 64, 0)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** Overwrites a single int at {@code offset} in an existing file. */ + private static void overwriteInt(String path, long offset, int value) { + int fd = Files.openRW(path); + Assert.assertTrue("could not open " + path, fd >= 0); + try { + long buf = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(buf, value); + Assert.assertEquals(4, Files.write(fd, buf, 4, offset)); + } finally { + Unsafe.free(buf, 4, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Writes a valid {@code sf-manifest.bin} with one CRC-protected record, + * mirroring SfManifest's on-disk layout (two alternating 64-byte records + * in a 128-byte file; record slot = generation & 1). + */ + private void writeManifest(long generation, long headBase, long activeBase) { + String path = tmpDir + "/" + MANIFEST_NAME; + // openRW (not openCleanRW): callers may layer a second generation's + // record into the sibling slot of an existing manifest. + int fd = Files.openRW(path); + Assert.assertTrue("could not create manifest", fd >= 0); + try { + if (Files.length(path) < 128) { + Assert.assertTrue(Files.truncate(fd, 128)); + } + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, 0x314d4653); // SFM1 + Unsafe.getUnsafe().putInt(buf + 4, 1); // version + Unsafe.getUnsafe().putLong(buf + 8, generation); + Unsafe.getUnsafe().putLong(buf + 16, headBase); + Unsafe.getUnsafe().putLong(buf + 24, activeBase); + int crc = Crc32c.update(Crc32c.INIT, buf, 60); + Unsafe.getUnsafe().putInt(buf + 60, crc); + long offset = (generation & 1L) * 64; + Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Snapshot of the durable SF payload files (segments, quarantined + * segments, manifest): name -> content. Lifecycle noise such as the slot + * lock and ack watermark is deliberately excluded -- "recovery must not + * mutate the slot" is a statement about the durable log, not about lock + * bookkeeping. + */ + private Map snapshotDir() { + Map out = new HashMap<>(); + Path dir = Paths.get(tmpDir); + try (java.util.stream.Stream stream = java.nio.file.Files.list(dir)) { + stream.filter(java.nio.file.Files::isRegularFile) + .filter(p -> { + String name = p.getFileName().toString(); + return name.endsWith(".sfa") || name.endsWith(".corrupt") + || MANIFEST_NAME.equals(name); + }) + .forEach(p -> { + try { + out.put(p.getFileName().toString(), java.nio.file.Files.readAllBytes(p)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return out; + } + + /** Asserts the directory holds exactly the snapshotted files, byte for byte. */ + private void assertDirUnchanged(Map before) { + Map after = snapshotDir(); + Assert.assertEquals("file set must be unchanged", before.keySet(), after.keySet()); + for (Map.Entry e : before.entrySet()) { + Assert.assertArrayEquals("bytes of " + e.getKey() + " must be unchanged", + e.getValue(), after.get(e.getKey())); + } + } + + private int countSfaFiles() { + int count = 0; + for (String name : snapshotDir().keySet()) { + if (name.endsWith(".sfa")) { + count++; + } + } + return count; + } + + // ------------------------------------------------------------------ + // Manifest crash windows and record selection + // ------------------------------------------------------------------ + + @Test + public void testCreationCrashZeroByteManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // kill -9 between the manifest's O_EXCL create and its first + // durable record leaves a zero-byte file. No boundary was ever + // committed, so nothing can depend on it: startup must self-heal, + // not demand an operator delete the file. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("creation-crash manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("debris must be quarantined for postmortem", + Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue("a fresh valid manifest must replace the debris", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCreationCrashRecordlessManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Same window, later stage: allocate() completed (128 zero bytes) + // but the first record write/fsync never landed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 128)); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("record-less manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testRecordlessManifestWithFlaggedSegmentsFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Flagged segments prove a manifest create COMPLETED at some + // point, so a record-less manifest here is double-slot bit rot, + // not creation debris -- boundaries were lost. Fail closed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); // truncates + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 128)); + Files.close(fd); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("boundary loss next to flagged segments must fail closed"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + @Test + public void testDrainCrashSurvivingSpareRecoversEmpty() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain crash window: boundaries collapsed to head==active, + // the active-base file already unlinked, but an empty hot spare + // (provisional base > activeBase) survived. Everything was acked; + // this must recover as EMPTY, not brick on "missing active". + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000007.sfa", 5, SEGMENT_SIZE); + spare.close(); + writeManifest(4, 3, 3); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("drain crash window must recover as EMPTY", ring); + Assert.assertEquals("surviving spare must be cleaned up", 0, countSfaFiles()); + Assert.assertFalse("collapsed manifest must be removed", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptActiveWithSameBaseEmptyStandInFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The active (holding unacked frames) is corrupted while a + // leftover spare coincidentally carries the same base. Accepting + // the clean empty as the "rotation crash" active would quarantine + // the unacked frames and re-issue their FSNs -- recovery must + // fail closed instead. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String corruptActive = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(corruptActive, 2, 2); + overwriteInt(corruptActive, 0, 0xBADC0DE); // bit-rot the magic + MmapSegment standIn = MmapSegment.create(tmpDir + "/sf-0000000000000002.sfa", 2, SEGMENT_SIZE); + standIn.close(); + writeManifest(1, 0, 2); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an empty stand-in must not mask a corrupt active"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testManifestHigherGenerationRecordWins() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) says active=0; gen2 (slot 0) says active=2. If + // selection picked gen1, the empty at base 2 would sit beyond the + // committed active boundary; gen2 accepts it as the active. + writeManifest(1, 0, 0); + writeManifest(2, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals("the higher-generation record must win", + 2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testManifestTornNewerRecordFallsBackToOlder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) is valid and matches the segments; gen2 (slot 0) + // was torn mid-write (bad CRC). Selection must fall back to gen1 + // rather than reject the manifest or trust torn boundaries. + writeManifest(1, 0, 2); + writeManifest(2, 0, 4); + overwriteInt(tmpDir + "/" + MANIFEST_NAME, 60, 0xBADC0DE); // tear gen2's CRC (slot 0) + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("a torn newer record must fall back to the older slot", ring); + try { + Assert.assertEquals(2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + /** FilesFacade delegating everything to the production instance. */ + private static class DelegatingFacade implements FilesFacade { + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { return INSTANCE.openRW(path); } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java index 9f533fc7..a226eb0f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java @@ -76,11 +76,17 @@ public void tearDown() { } @Test - public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { + public void testRecoveryReusesSoleEmptyOrphanAsInitialActive() throws Exception { TestUtils.assertMemoryLeak(() -> { // Simulate a crashed prior session that left an unrotated hot spare // (valid SF01 header, frameCount=0). MmapSegment.create stamps the // header but writes no frames. + // + // Contract (manifest-era): a clean empty leftover is REUSED as the + // fresh ring's initial active instead of being unlinked and + // re-created. The anti-leak guarantee this test was written for + // still holds -- crash cycles converge to exactly one segment + // file, they don't accumulate. String orphanPath = tmpDir + "/sf-orphan.sfa"; MmapSegment empty = MmapSegment.create(orphanPath, 0L, SEGMENT_SIZE); empty.close(); @@ -89,12 +95,17 @@ public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { SegmentRing recovered = SegmentRing.openExisting(tmpDir, SEGMENT_SIZE); - Assert.assertNull( - "recovery returned a ring even though the only segment was empty", + Assert.assertNotNull( + "recovery must reuse the clean empty leftover as the initial active", recovered); - Assert.assertFalse( - "recovery left the empty orphan .sfa on disk — disk leak grows " - + "with every crash cycle", + try { + Assert.assertEquals("reused ring must be empty", -1, recovered.publishedFsn()); + Assert.assertEquals(0, recovered.getActive().baseSeq()); + Assert.assertEquals(orphanPath, recovered.getActive().path()); + } finally { + recovered.close(); + } + Assert.assertTrue("the reused segment file must still exist", Files.exists(orphanPath)); }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index f1a0fcde..c6466de7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -38,6 +38,8 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -428,9 +430,10 @@ public void testAcknowledgeIsMonotonic() throws Exception { } @Test - public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws Exception { + public void testNextSealedAfterWalksThousandsOfSegmentsInLinearOperations() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Regression for "sealed snapshot grew unexpectedly large". + // Regression for both "sealed snapshot grew unexpectedly large" + // and a later quadratic list traversal at every segment boundary. // The cursor I/O loop used to copy the entire sealed list into a // fixed-size array (initial 16, grown once to 32) on every advance. // Under load — producer outpacing the WS sender, no maxTotalBytes @@ -462,8 +465,18 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws // After the loop we have `sealedCount` sealed segments and one // active (containing nothing yet — its base = sealedCount). // Now walk: oldest sealed, then nextSealedAfter() repeatedly. + SegmentRing.resetNextSealedComparisons(); MmapSegment cursor = ring.firstSealed(); assertNotNull(cursor); + for (int i = 1; i < sealedCount / 2; i++) { + cursor = ring.nextSealedAfter(cursor); + assertNotNull(cursor); + } + long halfWalkOperations = SegmentRing.getNextSealedComparisons(); + + SegmentRing.resetNextSealedComparisons(); + cursor = ring.firstSealed(); + assertNotNull(cursor); assertEquals(0, cursor.baseSeq()); int visited = 1; long prevBase = cursor.baseSeq(); @@ -478,8 +491,14 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws visited++; } assertEquals("must visit every sealed segment", sealedCount, visited); - // Walking past the last sealed → null (caller falls through to active). - assertNull(ring.nextSealedAfter(cursor)); + // The loop terminated when walking past the last sealed returned null. + long comparisons = SegmentRing.getNextSealedComparisons(); + assertTrue("walk inspected " + comparisons + " entries for " + sealedCount + + " segments; successor traversal must remain O(N)", + comparisons <= 2L * sealedCount); + assertTrue("doubling the walk grew operations from " + halfWalkOperations + + " to " + comparisons + "; expected linear scaling", + comparisons <= 2L * halfWalkOperations + 2L); } } finally { Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); @@ -528,6 +547,73 @@ public void testNextSealedAfterStillReturnsCorrectlyWhenCursorWasTrimmed() throw }); } + @Test(timeout = 30_000L) + public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 16); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment seg0 = MmapSegment.create(tmpDir + "/race-0.sfa", 0, segSize); + try (SegmentRing ring = new SegmentRing(seg0, segSize)) { + fillPattern(buf, 16, 0); + for (int i = 0; i < 3; i++) { + assertEquals(i, ring.appendOrFsn(buf, 16)); + ring.installHotSpare(MmapSegment.create( + tmpDir + "/race-" + (i + 1) + ".sfa", + ring.nextSeqHint(), segSize)); + } + MmapSegment cursor = ring.firstSealed(); + assertNotNull(cursor); + ring.acknowledge(1); + + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread rotate = new Thread(() -> { + ready.countDown(); + try { + start.await(); + assertEquals(3L, ring.appendOrFsn(buf, 16)); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-rotate"); + Thread trim = new Thread(() -> { + ready.countDown(); + try { + start.await(); + ObjList drained = ring.drainTrimmable(); + assertNotNull(drained); + assertEquals(2, drained.size()); + for (int i = 0; i < drained.size(); i++) { + drained.get(i).close(); + } + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-trim"); + + synchronized (ring) { + rotate.start(); + trim.start(); + ready.await(); + start.countDown(); + } + rotate.join(); + trim.join(); + assertNull("concurrent mutation failed: " + failure.get(), failure.get()); + + MmapSegment next = ring.nextSealedAfter(cursor); + assertNotNull(next); + assertEquals("trim fallback must skip both removed successors", 2L, next.baseSeq()); + assertEquals("rotation must leave the former active sealed", 1, ring.getSealedSegments().size()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + /** * Open-time sort regression: at the documented {@code sf_max_total_bytes * / sf_max_bytes} ceiling (~16K segments) an O(N²) sort over the @@ -600,6 +686,147 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { }); } + /** + * Adversarial-order companion to + * {@link #testLargeSegmentCountReopensInOrder}: that test covers the + * already-sorted readdir order median-of-three was chosen for; this one + * covers the orders median-of-three does NOT defend. On the + * pre-introsort quicksort, exact simulation at N=16384 measured ~22.6M + * comparisons for organ-pipe, ~134M (full N²/2) for mass-duplicate + * baseSeqs and ~50.7M for Musser's median-of-three-killer permutation. + * The heapsort fallback caps the worst of these at ~3.8·N·log₂(N) + * (~860K), so the 8·N·log₂(N) bound below keeps >2x headroom against + * harmless implementation drift while sitting ~26x under the mildest + * quadratic blow-up. Also pins unsigned key ordering: baseSeqs with the + * sign bit set must sort above {@code Long.MAX_VALUE}, not below zero. + *

    + * A healthy client cannot produce these orders (recovery validation + * rejects duplicate or non-contiguous baseSeqs moments after the sort), + * but corrupted-yet-parseable headers or operator file copies feed the + * sort BEFORE validation runs, so the sort itself must stay log-linear. + * Sorts in-memory segments directly: staging 16K adversarial header + * files on disk per pattern would dominate the test's runtime without + * adding coverage. + */ + @Test + public void testAdversarialSegmentOrdersSortLogLinear() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 16384; + final long bound = 8L * n * (long) (Math.log(n) / Math.log(2)); + + // Organ pipe: 0,1,...,n/2-1,n/2-1,...,1,0. + long[] organPipe = new long[n]; + for (int i = 0; i < n / 2; i++) { + organPipe[i] = i; + organPipe[n - 1 - i] = i; + } + assertAdversarialSortWithinBound("organ-pipe", organPipe, bound); + + long[] allDuplicates = new long[n]; + for (int i = 0; i < n; i++) { + allDuplicates[i] = 42; + } + assertAdversarialSortWithinBound("all-duplicates", allDuplicates, bound); + + long[] fewDistinct = new long[n]; + for (int i = 0; i < n; i++) { + fewDistinct[i] = i % 4; + } + assertAdversarialSortWithinBound("few-distinct", fewDistinct, bound); + + // Musser's median-of-three killer permutation of 1..n. + long[] med3Killer = new long[n]; + int half = n / 2; + for (int i = 1; i <= half; i++) { + if (i % 2 == 1) { + med3Killer[i - 1] = i; + med3Killer[i] = half + i; + } + med3Killer[half + i - 1] = 2L * i; + } + assertAdversarialSortWithinBound("median-of-three-killer", med3Killer, bound); + + // High-bit keys interleaved with small ones: exercises the + // unsigned comparison contract alongside the comparison bound. + long[] unsignedMix = new long[n]; + for (int i = 0; i < n; i++) { + unsignedMix[i] = (i % 2 == 0) ? (0x8000000000000000L | i) : i; + } + assertAdversarialSortWithinBound("unsigned-mix", unsignedMix, bound); + }); + } + + private static void assertAdversarialSortWithinBound(String label, long[] baseSeqs, long bound) { + final long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; + ObjList list = new ObjList<>(); + try { + for (long baseSeq : baseSeqs) { + list.add(MmapSegment.createInMemory(baseSeq, segSize)); + } + SegmentRing.resetSortComparisons(); + SegmentRing.sortByBaseSeqForTest(list); + long comparisons = SegmentRing.getSortComparisons(); + for (int i = 1, size = list.size(); i < size; i++) { + assertTrue(label + ": unsigned baseSeq order violated at index " + i, + Long.compareUnsigned(list.get(i - 1).baseSeq(), list.get(i).baseSeq()) <= 0); + } + assertTrue(label + " sort took " + comparisons + " comparisons (expected < " + bound + + " = 8 * N * log2(N) for N=" + baseSeqs.length + + "); regression suggests the introsort heapsort fallback stopped engaging", + comparisons < bound); + } finally { + for (int i = 0, size = list.size(); i < size; i++) { + list.get(i).close(); + } + } + } + + @Test + public void testRemovingAcknowledgedPrefixMovesLinearReferences() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + long previousMoves = -1; + for (int sealedCount = 64; sealedCount <= 256; sealedCount *= 2) { + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + for (int i = 0; i < 2 * sealedCount; i++) { + assertEquals(i, ring.appendOrFsn(buf, 1)); + ring.installHotSpare(MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + assertEquals(2L * sealedCount, ring.appendOrFsn(buf, 1)); + ring.acknowledge(sealedCount - 1L); + SegmentRing.resetTrimMovedReferences(); + MmapSegment segment; + int removed = 0; + while ((segment = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(segment)); + segment.close(); + removed++; + } + assertEquals(sealedCount, removed); + assertEquals("unacknowledged suffix must remain live", sealedCount, + ring.getSealedSegments().size()); + assertEquals(sealedCount, ring.firstSealed().baseSeq()); + long moves = SegmentRing.getTrimMovedReferences(); + assertTrue("removing " + sealedCount + " entries moved " + moves + + " references; expected amortized linear work", + moves <= 2L * sealedCount); + if (previousMoves >= 0) { + assertTrue("doubling the acknowledged prefix grew moved references from " + + previousMoves + " to " + moves, + moves <= 2L * previousMoves + sealedCount); + } + previousMoves = moves; + } + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testMaxBytesPerSegmentReturnsConfiguredValue() throws Exception { // Direct constructor path: the value passed in must round-trip through diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644a7463..20930f50 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -34,6 +34,7 @@ import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -103,6 +104,71 @@ public void testCloseReleasesLock() throws Exception { }); } + @Test + public void testReleaseConfirmsAndIsIdempotent() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/verified-release"; + SlotLock lock = SlotLock.acquire(slot); + assertTrue("first release must confirm success", lock.release()); + // Idempotent: an already-released lock keeps reporting true — + // callers gating a "slot reusable" signal on it must never see + // a spurious false after a confirmed release. + assertTrue("repeat release must stay true", lock.release()); + // close() after release() is a safe no-op (QuietCloseable path). + lock.close(); + // Confirmed release means the slot is genuinely acquirable. + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + + /** + * The {@code release() == false} branch: when the OS reports an explicit + * unlock failure, release must (a) return {@code false} so owners gating a + * "slot reusable" signal never see a lie, (b) retain the fd because the + * non-consuming unlock can safely be retried, and (c) keep returning + * {@code false} while the failure persists. Once unlock succeeds, release + * confirms and stays confirmed. Swapping in a known-bad descriptor gives + * the slot-specific native primitive a deterministic unlock failure. + */ + @Test + public void testFailedUnlockRetainsFdAndReportsFalse() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-release"; + SlotLock lock = SlotLock.acquire(slot); + java.lang.reflect.Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(lock); + assertTrue("precondition: acquire must hold a live fd", realFd >= 0); + try { + // A non-negative descriptor no process has open makes the + // explicit flock/UnlockFileEx operation fail without consuming + // the real descriptor that continues to hold the slot lock. + fdField.setInt(lock, 1_000_000_000); + assertFalse("release must report false when explicit unlock fails", + lock.release()); + assertEquals("failed unlock must retain the fd for a safe retry", + 1_000_000_000, fdField.getInt(lock)); + assertFalse("repeat release must stay false while unlock keeps failing", + lock.release()); + // While the release is unconfirmed the real flock remains held. + try (SlotLock ignored = SlotLock.acquire(slot)) { + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good - unconfirmed release really means "still locked". + } + } finally { + fdField.setInt(lock, realFd); + } + assertTrue("release must confirm once explicit unlock succeeds", lock.release()); + assertTrue("confirmed release must stay confirmed", lock.release()); + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..139a445f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -64,6 +64,8 @@ public class TestWebSocketServer implements Closeable { // client-side pool actually closed the connections it opened. private final AtomicInteger liveConnections = new AtomicInteger(); private final int port; + private final AtomicInteger roleRejectCount = new AtomicInteger(); + private final CountDownLatch roleRejectLatch = new CountDownLatch(1); private final AtomicBoolean running = new AtomicBoolean(false); private final ServerSocket serverSocket; private final CountDownLatch startLatch = new CountDownLatch(1); @@ -165,6 +167,10 @@ public TestWebSocketServer(WebSocketServerHandler handler, this.port = serverSocket.getLocalPort(); } + public boolean awaitRoleReject(long timeout, TimeUnit unit) throws InterruptedException { + return roleRejectLatch.await(timeout, unit); + } + public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedException { return startLatch.await(timeout, unit); } @@ -221,6 +227,13 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Number of HTTP 421 role-reject responses sent over the server's lifetime. + */ + public int roleRejectCount() { + return roleRejectCount.get(); + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -586,6 +599,8 @@ private boolean performHandshake() throws IOException { "\r\n"; out.write(sb.getBytes(StandardCharsets.US_ASCII)); out.flush(); + roleRejectCount.incrementAndGet(); + roleRejectLatch.countDown(); return false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java new file mode 100644 index 00000000..c4a33e9c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java @@ -0,0 +1,191 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +/** + * P2 regression: {@link QuestDBImpl#close()} must not return before shutdown + * has completed, even when two threads call it concurrently. {@code closed} is + * volatile and set BEFORE the pool teardown chain runs, so a naive guard lets a + * second concurrent caller observe {@code closed == true} and RETURN while the + * first caller is still inside {@code closeQuietly(senderPool)} tearing down the + * flock/mmap/I/O-thread resources. After ANY {@code close()} returns -- the + * losing concurrent caller included -- callers must be able to assume shutdown + * has completed. + *

    + * The window is opened deterministically by injecting (via the {@code @TestOnly} + * senderFactory seam) a fake delegate whose {@code close()} parks on a latch. + * The last teardown step, {@code senderPool.close()}, closes the prewarmed idle + * delegate on the closing thread OUTSIDE the pool lock, so thread A parks there + * with {@code closed} already raised. Thread B then calls {@code close()}: + *

      + *
    • pre-fix -- B reads {@code closed == true} and returns immediately while A + * is still tearing down (premature return);
      • + *
    • fixed -- B blocks until A finishes the teardown, then returns.
      • + *
    + * Latch-coordinated (no {@code Thread.sleep} for correctness) with a JUnit + * timeout on the two-thread interleaving. + */ +public class QuestDBImplCloseTest { + + // Non-SF http config: the injected senderFactory replaces the native build, + // but the constructor's eager config probe must still parse it. + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + // RED (closed set before teardown, no serialization): while thread A is + // parked inside the delegate teardown of the final senderPool.close() step, + // thread B's close() sees closed==true and RETURNS -- closeReturnedEarly is + // true and the first assertion fails. GREEN (close() serialized through + // completion): B blocks on A until the teardown finishes, so its close() + // does not return until delegateCloses == 1. + @Test(timeout = 30_000) + public void concurrentCloseSecondCallerBlocksUntilShutdownCompletes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger delegateCloses = new AtomicInteger(); + CountDownLatch inDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + IntFunction senderFactory = slotIndex -> + parkingCloseSender(delegateCloses, inDelegateClose, releaseDelegateClose); + // queryMin = 0 -> QueryClientPool prewarms nothing, so the connect + // hook is never reached and its teardown is a no-op; the parking + // delegate is the only blocking teardown step. + Consumer connectHook = client -> { + }; + + QuestDBImpl questDB = newQuestDB(senderFactory, connectHook); + + // Thread A: enter close() and park inside the final teardown step + // (senderPool.close() -> idle delegate close()), with closed + // already raised. + Thread closerA = new Thread(questDB::close, "questdb-closer-A"); + closerA.start(); + Assert.assertTrue("closer A never reached the delegate teardown", + inDelegateClose.await(10, TimeUnit.SECONDS)); + + // Thread B: a concurrent close(). It must NOT return while A is + // still tearing down. + Thread closerB = new Thread(questDB::close, "questdb-closer-B"); + closerB.start(); + closerB.join(300); + boolean closeReturnedEarly = !closerB.isAlive(); + int closesWhenBReturned = delegateCloses.get(); + + // Always unpark the teardown so the test fails on the assertion, not + // its own timeout. + releaseDelegateClose.countDown(); + + Assert.assertFalse( + "concurrent close() returned while the first caller was still tearing down " + + "(delegateCloses=" + closesWhenBReturned + " when B returned): " + + "close() must serialize through shutdown completion", + closeReturnedEarly); + + // Once the teardown completes, B must return promptly and the + // delegate must have been torn down exactly once. + closerB.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("concurrent close() did not return after the teardown completed", + closerB.isAlive()); + closerA.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("first close() did not return after the teardown completed", + closerA.isAlive()); + Assert.assertEquals("the prewarmed delegate must be torn down exactly once", + 1, delegateCloses.get()); + }); + } + + private static QuestDBImpl newQuestDB( + IntFunction senderFactory, Consumer connectHook + ) { + return new QuestDBImpl( + SENDER_CFG, QUERY_CFG, + /*senderMin*/ 1, /*senderMax*/ 1, + /*queryMin*/ 0, /*queryMax*/ 1, + /*acquireTimeoutMillis*/ 250L, + /*idleTimeoutMillis*/ Long.MAX_VALUE, + /*maxLifetimeMillis*/ Long.MAX_VALUE, + /*housekeeperIntervalMillis*/ Long.MAX_VALUE, + senderFactory, connectHook); + } + + /** + * Proxy-backed fake Sender whose {@code close()} signals {@code inClose}, + * parks on {@code releaseClose}, then bumps {@code closes} -- a delegate + * teardown frozen mid-close so the test can probe what a concurrent + * close() does while it runs. + */ + private static Sender parkingCloseSender( + AtomicInteger closes, + CountDownLatch inClose, + CountDownLatch releaseClose + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + inClose.countDown(); + if (!releaseClose.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("test never released the parked close"); + } + closes.incrementAndGet(); + return null; + case "toString": + return "ParkingCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index 81055bb6..38561c46 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -32,6 +32,8 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.file.Paths; import java.util.concurrent.atomic.AtomicBoolean; @@ -83,6 +85,75 @@ public void preWarmClosesBuiltDelegatesWhenBuildThrowsError() throws Exception { }); } + @Test(timeout = 30_000) + public void elasticBorrowClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + })) { + try { + pool.borrow(); + Assert.fail("borrow must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("borrow must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed elastic creation must close its delegate", 1, closeCalls.get()); + } + }); + } + + @Test(timeout = 30_000) + public void preWarmClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try { + new SenderPool( + CFG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + }); + Assert.fail("prewarm must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("prewarm must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed prewarm creation must close its delegate", 1, closeCalls.get()); + }); + } + + @Test(timeout = 30_000) + public void recoveryClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AssertionError closeFailure = new AssertionError("injected close failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, closeFailure), false, + () -> { + throw failure; + })) { + Method createRecoverer = SenderPool.class.getDeclaredMethod("createRecoverer", int.class); + createRecoverer.setAccessible(true); + try { + createRecoverer.invoke(pool, -1); + Assert.fail("recovery creation must propagate the post-factory failure"); + } catch (InvocationTargetException e) { + Assert.assertSame("recovery must preserve throwable identity", failure, e.getCause()); + Assert.assertArrayEquals("close failure must remain secondary", + new Throwable[]{closeFailure}, failure.getSuppressed()); + } + Assert.assertEquals("failed recovery creation must close its delegate", 1, closeCalls.get()); + } + }); + } + // Companion to the catch (RuntimeException) -> track-normal-completion fix in // PooledSender.close(). flush() can exit with an Error (AssertionError under // -ea, OutOfMemoryError, ...) as well as a RuntimeException; the wrapper is @@ -232,6 +303,42 @@ public void borrowReleasesSfSlotIndexWhenCreationFails() throws Exception { }); } + private static Sender closeCountingSender(AtomicInteger closeCalls, Throwable closeFailure) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCalls.incrementAndGet(); + if (closeFailure != null) { + throw closeFailure; + } + return null; + } + if ("toString".equals(method.getName())) { + return "CloseCountingSender"; + } + if ("hashCode".equals(method.getName())) { + return System.identityHashCode(proxy); + } + if ("equals".equals(method.getName())) { + return proxy == args[0]; + } + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + }); + } + private static Sender fakeSender(AtomicBoolean closedFlag) { return (Sender) Proxy.newProxyInstance( Sender.class.getClassLoader(), diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index cfef2feb..ea7e901a 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -30,7 +30,11 @@ import ch.qos.logback.core.read.ListAppender; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -45,14 +49,17 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -658,19 +665,1129 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw }); } + @Test + public void testRetiredSlotRecoveredByHousekeeperAfterLateFlockRelease() throws Exception { + // Recovery twin of testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock: + // a slot retired because close() returned with the flock still held is + // NOT lost until process exit. Engine cleanup may complete later on a + // worker/I/O-thread exit path (isSlotLockReleased() re-probes the + // retained engine), and the housekeeper's reapIdle() tick must then + // return the index to the free set: leakedSlots back down, slotInUse + // cleared, full capacity restored. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + // Forge the retire: real teardown first (no native leaks), + // then clear slotLockReleased so discardBroken retires the + // slot as leaked. + Sender delegate = getDelegate(a); + delegate.close(); + setBooleanField(delegate, "slotLockReleased", false); + invokeDiscardBroken(pool, a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // Forge the late release: the deferred cleanup finished and + // the delegate now reports the flock dropped (in production + // this flip comes from isSlotLockReleased() re-probing the + // retained engine after the manager worker exited). + setBooleanField(delegate, "slotLockReleased", true); + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertFalse("recovered slot index 0 must return to the free set", + slotInUse[0]); + + // Full capacity restored: with maxSize=2, two concurrent + // borrows must succeed again (index 0 is reusable — its + // flock is genuinely free). + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + } + } + }); + } + + @Test + public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { + // Borrow-path twin of the housekeeper recovery test: a borrow that + // would otherwise park on the cap check must re-probe retired slots + // before waiting, so a late flock release converts a guaranteed + // borrow timeout into an immediate creation on the recovered index — + // no housekeeper tick required. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + delegate.close(); + setBooleanField(delegate, "slotLockReleased", false); + invokeDiscardBroken(pool, a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // One live borrow + one retired slot = cap reached. + PooledSender b = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-1"))); + try { + // Late release lands while the pool is capacity-starved. + setBooleanField(delegate, "slotLockReleased", true); + + // The next borrow hits the cap check, re-probes, frees + // index 0, and must create on it instead of timing out. + PooledSender c = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + } + } finally { + b.close(); + } + } + } + }); + } + + @Test(timeout = 30_000) + public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender lease = pool.borrow(); + Sender delegate = getDelegate(lease); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + Assert.assertTrue("precondition: live flock fd", realFd >= 0); + + CountDownLatch borrowerAcquired = new CountDownLatch(1); + CountDownLatch borrowerParked = new CountDownLatch(1); + CountDownLatch releaseBorrower = new CountDownLatch(1); + AtomicReference borrowerFailure = new AtomicReference<>(); + AtomicReference recovered = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + recovered.set(pool.borrow()); + borrowerAcquired.countDown(); + if (!releaseBorrower.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release borrower"); + } + } catch (Throwable t) { + borrowerFailure.compareAndSet(null, t); + } finally { + PooledSender sender = recovered.get(); + if (sender != null) { + sender.close(); + } + } + }, "sender-pool-deferred-release-waiter"); + + try { + synchronized (slotLock) { + fdField.setInt(slotLock, 1_000_000_000); + } + invokeDiscardBroken(pool, lease); + Assert.assertEquals("failed release must retire the only slot", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + pool.setBeforeBorrowWaitHook(borrowerParked::countDown); + borrower.start(); + Assert.assertTrue("borrower must reach the condition wait with a long timeout", + borrowerParked.await(5, TimeUnit.SECONDS)); + // The hook runs under the pool lock immediately before awaitNanos. + // Acquiring that lock here proves awaitNanos atomically enqueued the + // borrower and released the lock before restoration can start. This + // read-only operation neither changes pool state nor signals a waiter. + pool.availableSize(); + + // This restored fd is the only source of progress: the retry driver + // confirms the release. There is no housekeeper or pool mutation. + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + Assert.assertTrue("deferred flock release must wake the parked borrower", + borrowerAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertNull("borrower must not fail", borrowerFailure.get()); + Assert.assertEquals("release wakeup must recover retired capacity", + 0, pool.leakedSlotCount()); + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrower.countDown(); + if (!engine.isCloseCompleted()) { + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + } + borrower.join(TimeUnit.SECONDS.toMillis(1)); + if (borrower.isAlive()) { + borrower.interrupt(); + borrower.join(TimeUnit.SECONDS.toMillis(5)); + } + Assert.assertFalse("borrower thread must finish", borrower.isAlive()); + } + if (borrowerFailure.get() != null) { + throw new AssertionError("borrower failed", borrowerFailure.get()); + } + } + } + }); + } + + @Test + public void testDirectRetiredSlotCallbacksHaveLinearProbeCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + final int slotCount = 32; + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, slotCount, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[slotCount]; + Sender[] delegates = new Sender[slotCount]; + Runnable[] callbacks = new Runnable[slotCount]; + for (int i = 0; i < slotCount; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + Assert.assertNotNull(callbacks[i]); + } + for (int i = 0; i < slotCount; i++) { + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(slotCount, pool.leakedSlotCount()); + setLongField(pool, "retiredSlotProbeCount", 0); + + int[] geometricCheckpoints = {4, 8, 16, 32}; + int checkpoint = 0; + for (int i = 0; i < slotCount; i++) { + setBooleanField(delegates[i], "slotLockReleased", true); + callbacks[i].run(); + if (i + 1 == geometricCheckpoints[checkpoint]) { + Assert.assertEquals("direct release probes must grow linearly", + i + 1, getLongField(pool, "retiredSlotProbeCount")); + checkpoint++; + } + } + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + } + } + }); + } + + @Test + public void testDirectRetiredSlotCallbackFallbackAndIdempotence() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 0, 3, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[3]; + Sender[] delegates = new Sender[3]; + Runnable[] callbacks = new Runnable[3]; + for (int i = 0; i < 3; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = (Runnable) getField(delegates[i], "slotLockReleaseListener"); + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(3, pool.leakedSlotCount()); + setLongField(pool, "retiredSlotProbeCount", 0); + + // Simulate callback registration becoming unavailable. The + // periodic housekeeper scan must remain a complete fallback. + ((QwpWebSocketSender) delegates[0]).setSlotLockReleaseListener(null); + setBooleanField(delegates[0], "slotLockReleased", true); + pool.reapIdle(); + Assert.assertEquals(2, pool.leakedSlotCount()); + + // A premature callback must not remove an unreleased slot. + callbacks[1].run(); + Assert.assertEquals(2, pool.leakedSlotCount()); + setBooleanField(delegates[1], "slotLockReleased", true); + callbacks[1].run(); + Assert.assertEquals(1, pool.leakedSlotCount()); + + // Duplicate and stale callbacks are idempotent and do not + // probe or mutate the slot after its direct removal. + setBooleanField(delegates[2], "slotLockReleased", true); + callbacks[2].run(); + long probesAfterRecovery = getLongField(pool, "retiredSlotProbeCount"); + callbacks[2].run(); + callbacks[1].run(); + Assert.assertEquals(probesAfterRecovery, + getLongField(pool, "retiredSlotProbeCount")); + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(((List) getField(pool, "retiredSlots")).isEmpty()); + } + } + }); + } + + @Test + public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 8, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[8]; + Sender[] delegates = new Sender[8]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + // Release the real native resources, then forge the + // delayed publication which makes the pool retire the + // slot. The idempotent second close in discardBroken() + // leaves the forged state unchanged. + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals(8, pool.leakedSlotCount()); + + // Mix completed and incomplete entries throughout the + // retired list. A recovery pass must remove exactly these + // four without skipping a swapped entry or corrupting the + // bitmap/count relationship. + int[] released = {0, 2, 5, 7}; + for (int i = 0; i < released.length; i++) { + setBooleanField(delegates[released[i]], "slotLockReleased", true); + } + pool.reapIdle(); + + Assert.assertEquals(4, pool.leakedSlotCount()); + Assert.assertEquals(4, ((List) getField(pool, "retiredSlots")).size()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + for (int i = 0; i < slotInUse.length; i++) { + boolean mustRemainRetired = i == 1 || i == 3 || i == 4 || i == 6; + Assert.assertEquals("slot reservation mismatch at index " + i, + mustRemainRetired, slotInUse[i]); + } + + // Exactly the four restored reservations are reusable. + PooledSender[] recovered = new PooledSender[4]; + try { + for (int i = 0; i < recovered.length; i++) { + recovered[i] = pool.borrow(); + } + Assert.assertEquals(4, pool.totalSize()); + Assert.assertEquals(4, pool.leakedSlotCount()); + } finally { + for (int i = 0; i < recovered.length; i++) { + if (recovered[i] != null) { + recovered[i].close(); + } + } + } + } + } + }); + } + + @Test(timeout = 30_000) + public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 6, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[6]; + Sender[] delegates = new Sender[6]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + delegates[i].close(); + setBooleanField(delegates[i], "slotLockReleased", false); + invokeDiscardBroken(pool, leases[i]); + } + Assert.assertEquals("all capacity must start retired", + 6, pool.leakedSlotCount()); + + final int borrowerCount = 3; + CountDownLatch allAcquired = new CountDownLatch(borrowerCount); + CountDownLatch allDone = new CountDownLatch(borrowerCount); + CountDownLatch firstAcquired = new CountDownLatch(1); + CountDownLatch initialWaiters = new CountDownLatch(borrowerCount); + CountDownLatch releaseBorrowers = new CountDownLatch(1); + CountDownLatch reparkedWaiters = new CountDownLatch(borrowerCount - 1); + AtomicReference failure = new AtomicReference<>(); + ConcurrentHashMap initialWaiterThreads = new ConcurrentHashMap<>(); + ConcurrentHashMap reparkedWaiterThreads = new ConcurrentHashMap<>(); + PooledSender[] recovered = new PooledSender[borrowerCount]; + pool.setBeforeBorrowWaitHook(() -> { + if (initialWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + initialWaiters.countDown(); + } + }); + for (int i = 0; i < borrowerCount; i++) { + final int borrower = i; + Thread thread = new Thread(() -> { + try { + recovered[borrower] = pool.borrow(); + allAcquired.countDown(); + firstAcquired.countDown(); + if (!releaseBorrowers.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release recovered borrower"); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + try { + if (recovered[borrower] != null) { + recovered[borrower].close(); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + allDone.countDown(); + } + } + }, "sender-pool-retired-waiter-" + i); + thread.start(); + } + + try { + Assert.assertTrue("all borrowers must reach the condition wait", + initialWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("three distinct borrowers must reach the wait path", + borrowerCount, initialWaiterThreads.size()); + + // The hook runs while each borrower holds the pool lock, + // immediately before awaitNanos atomically releases that + // lock and enqueues it. Once the last latch count lands, + // the first reapIdle() cannot acquire the lock until all + // three borrowers are definitely condition waiters. + pool.setBeforeBorrowWaitHook(() -> { + if (reparkedWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + reparkedWaiters.countDown(); + } + }); + setBooleanField(delegates[2], "slotLockReleased", true); + pool.reapIdle(); + + // One restored index admits exactly one borrower. The + // other two must consume signalAll(), lose the capacity + // race, and deterministically re-enter the wait path. + Assert.assertTrue("one borrower must take the first restored index", + firstAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("two distinct borrowers must re-park after the first recovery", + reparkedWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("exactly two distinct borrowers must re-enter the wait path", + borrowerCount - 1, reparkedWaiterThreads.size()); + Assert.assertEquals("exactly one borrower should hold restored capacity", + borrowerCount - 1, allAcquired.getCount()); + + // Recover two non-contiguous entries in one reverse / + // swap-remove pass. A single signal() here would wake + // only one of the two proven waiters; signalAll() must + // wake both and let them claim the two restored indices. + pool.setBeforeBorrowWaitHook(null); + setBooleanField(delegates[0], "slotLockReleased", true); + setBooleanField(delegates[5], "slotLockReleased", true); + pool.reapIdle(); + Assert.assertTrue("all waiting borrowers must receive restored capacity", + allAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(3, pool.totalSize()); + Assert.assertEquals(3, pool.leakedSlotCount()); + + boolean[] seen = new boolean[6]; + for (int i = 0; i < recovered.length; i++) { + int slotIndex = getIntField(slotOf(recovered[i]), "slotIndex"); + Assert.assertFalse("borrowers must receive distinct restored indices", + seen[slotIndex]); + seen[slotIndex] = true; + } + Assert.assertTrue("slot 0 must be restored", seen[0]); + Assert.assertTrue("slot 2 must be restored", seen[2]); + Assert.assertTrue("slot 5 must be restored", seen[5]); + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrowers.countDown(); + Assert.assertTrue("borrower threads must finish", + allDone.await(10, TimeUnit.SECONDS)); + } + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } + } + }); + } + + @Test + public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 1, 2_000, + Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Sender delegate = getDelegate(a); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + SlotLock slotLock = (SlotLock) getField(engine, "slotLock"); + Field fdField = SlotLock.class.getDeclaredField("fd"); + fdField.setAccessible(true); + int realFd = fdField.getInt(slotLock); + Assert.assertTrue("precondition: live flock fd", realFd >= 0); + try { + // Inject one persistent explicit-unlock failure. + // Delegate close must retire the only pool slot rather + // than publish a release while the real flock remains held. + synchronized (slotLock) { + fdField.setInt(slotLock, 1_000_000_000); + } + invokeDiscardBroken(pool, a); + Assert.assertEquals("failed release must retire pool capacity", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + // Remove the fault without calling close again: the + // engine's error-path retry driver runs outside the + // pool lock and must eventually publish completion. + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + } + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("flock-release retry never completed"); + } + Thread.sleep(1L); + } + + // A capacity-starved public borrow re-probes the + // retained delegate, frees index 0, and proves the + // recovered flock is genuinely acquirable. + PooledSender recovered = pool.borrow(); + try { + Assert.assertEquals("retry must restore pool capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(1, countSlotDirs()); + } finally { + recovered.close(); + } + } finally { + if (!engine.isCloseCompleted()) { + synchronized (slotLock) { + fdField.setInt(slotLock, realFd); + Assert.assertTrue("restored fd must release cleanly", + slotLock.release()); + } + } + } + } + } + }); + } + + @Test + public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Exception { + // Full-stack twin of the two forged-flag recovery tests above: no + // reflection-forged slotLockReleased anywhere. The REAL mechanism is + // driven end to end — the delegate's owned SegmentManager worker is + // wedged mid service pass (test hook), the delegate's close() takes + // the real timed-out-join → worker-exit handoff path, the pool + // retires the slot off the delegate's genuine isSlotLockReleased() + // report, the worker's deferred cleanup releases the real flock, and + // the housekeeper re-probe restores capacity — proving the recovered + // index is genuinely reusable by borrowing on it again (a forged flag + // would pass the accounting asserts but collide on the flock here). + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + CursorSendEngine engine = (CursorSendEngine) getField(delegate, "cursorEngine"); + Assert.assertNotNull("SF delegate must own a cursor engine", engine); + SegmentManager manager = (SegmentManager) getField(engine, "manager"); + + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + // Park the manager worker inside a service pass for + // the delegate's ring. The trim-sync point is reached + // on every ~1ms tick, so this wedges deterministically. + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + Assert.assertTrue("manager worker never entered a service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Real teardown against the wedged worker: the + // delegate's engine close times out its bounded join, + // hands cleanup to the worker's exit path, and reports + // the retained flock; the pool must retire the slot. + // Fault the old callback-allocation/registration path. + // C3 makes the owned-engine handoff preallocated, so + // this hook must no longer be reached by production + // teardown. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + invokeDiscardBroken(pool, a); + Assert.assertEquals( + "pool must retire the slot while the delegate's manager " + + "worker holds the deferred cleanup", + 1, pool.leakedSlotCount()); + Assert.assertFalse("engine cleanup must still be pending", + engine.isCloseCompleted()); + + // Un-wedge and deterministically reap the manager. + // No sender/engine close retry is allowed: worker exit + // itself must own and finish the preallocated handoff. + releaseWorker.countDown(); + manager.close(); + Assert.assertTrue("manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must run deferred cleanup despite the " + + "old allocation fault injection", + engine.isCloseCompleted()); + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertFalse("recovered slot index 0 must return to the free set", + slotInUse[0]); + + // The proof a forged flag cannot fake: both indices — + // including the recovered one, whose flock was really + // dropped by the worker-exit cleanup — admit live + // senders again. + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + if (hookErr.get() != null) { + throw new AssertionError("trim hook failed", hookErr.get()); + } + } finally { + manager.setBeforeExitCleanupRegistrationHook(null); + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + } + } + } + }); + } + + @Test + public void testPreallocatedExitHandoffCleansInRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(0, 1); + } + + @Test + public void testPreallocatedExitHandoffCleansOutOfRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(1, 1); + } + // ---------------------------------------------------------------------- // Recovery: stable slot ids let a re-created pool re-adopt unacked data. // ---------------------------------------------------------------------- @Test - public void testRecoveryReplayThroughPooledSlot() throws Exception { - // Phase 1: write rows to a slot against a silent server (no acks), so - // the data persists unacked on disk under default-0. Close. - // Phase 2: a new pool against an ack-ing server re-adopts default-0 - // (stable index) and replays the unacked frames. Stable, deterministic - // slot ids are exactly what make this recovery possible. + public void testRecoveryReplayThroughPooledSlot() throws Exception { + // Phase 1: write rows to a slot against a silent server (no acks), so + // the data persists unacked on disk under default-0. Close. + // Phase 2: a new pool against an ack-ing server re-adopts default-0 + // (stable index) and replays the unacked frames. Stable, deterministic + // slot ids are exactly what make this recovery possible. + TestUtils.assertMemoryLeak(() -> { + // Phase 1 -- silent server. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + // Data must be on disk, unacked, under default-0. + Assert.assertTrue("unacked data must persist on disk", hasSegmentFile(slot("default-0"))); + + // Phase 2 -- ack-ing server, brand-new pool, same sf_dir. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(cfg2, 1, 1, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + try { + // Drain replays the recovered, unacked frames. + s.drain(5_000); + Assert.assertTrue("recovered frames must be replayed to the new server", + awaitAtLeast(handler.frames, 1, 5_000)); + } finally { + s.close(); + } + } + } + }); + } + + @Test + public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { + // M1 regression: a startup-recovery delegate runs on the PoolHousekeeper + // thread, so its build() must NOT inherit the user's SYNC initial-connect + // mode (auto-enabled by any reconnect_* knob). SYNC would retry the + // connect for the whole reconnect budget inside build() -- far past + // PoolHousekeeper.STOP_TIMEOUT_MILLIS -- so a close() landing during that + // build would make the housekeeper join time out and leave the recoverer + // holding the slot flock after close() returned. The recovery factory + // forces initial_connect_mode=OFF (at most one connect attempt); the + // normal factory must still honour the user's promoted SYNC mode. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + // reconnect_max_duration_millis set, initial_connect_mode unset + // -> the builder promotes to SYNC for ordinary senders. + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + + ";reconnect_max_duration_millis=30000;"; + // min=0 (no prewarm connect), no stranded data (recovery no-op). + try (SenderPool pool = new SenderPool(cfg, 0, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + // Normal managed-slot delegate: inherits the promoted SYNC. + Sender normal = invokeBuildSlotDelegate(pool, "defaultSender", 0); + try { + Assert.assertEquals( + "ordinary pooled sender must honour the user's promoted SYNC mode", + Sender.InitialConnectMode.SYNC, readInitialConnectMode(normal)); + } finally { + normal.close(); + } + // Recovery delegate on a different slot: forced OFF. + Sender recoverer = invokeBuildSlotDelegate(pool, "defaultRecoverySender", 1); + try { + Assert.assertEquals( + "recovery delegate must force OFF so build() makes at most one connect attempt", + Sender.InitialConnectMode.OFF, readInitialConnectMode(recoverer)); + } finally { + recoverer.close(); + } + } + } + }); + } + + @Test + public void testSharedManagerPassCompletionRecoversRetiredPoolSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + long segSize = io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.HEADER_SIZE + + io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + AtomicReference engineRef = new AtomicReference<>(); + manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertEquals(0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT)); + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + IntFunction factory = slotIndex -> { + CursorSendEngine engine = new CursorSendEngine( + slot("default-" + slotIndex), segSize, manager); + engineRef.set(engine); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port); + sender.setCursorEngine(engine, true); + return sender; + }; + SenderPool pool = new SenderPool( + config, 0, 1, 500, 0, Long.MAX_VALUE, factory); + try { + PooledSender lease = pool.borrow(); + Assert.assertTrue("shared manager never entered the sender ring's service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + lease.close(); + + manager.setWorkerJoinTimeoutMillis(50L); + pool.reapIdle(); + Assert.assertEquals("pool must retire the slot while its shared-manager pass is live", + 1, pool.leakedSlotCount()); + CursorSendEngine engine = engineRef.get(); + Assert.assertFalse("engine cleanup must remain pending during the live pass", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + Assert.assertTrue("deferred cleanup did not finish with the ring pass", + cleanupFinished.await(5, TimeUnit.SECONDS)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + + pool.reapIdle(); + Assert.assertEquals("pass completion must drive deferred engine cleanup and restore capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("deferred cleanup must publish the released flock", + engine.isCloseCompleted()); + try (PooledSender recovered = pool.borrow()) { + Assert.assertTrue("recovered slot must be reusable", Files.exists(slot("default-0"))); + } + } finally { + releaseWorker.countDown(); + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + pool.close(); + manager.close(); + } + } + }); + } + + @Test + public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception { + // C1 regression: the startup recovery loop MUST mirror discardBroken / + // reapIdle. When a recoverer's delegate close() returns with the SF + // flock still held (the I/O thread refused to stop), the recovered slot + // index must be retired (leakedSlots++, slotInUse stays + // set) -- NOT freed. Freeing it would let a later borrow re-pick the + // still-locked dir and resurrect "sf slot already in use", the exact + // failure class this PR exists to kill. Pre-fix the recovery finally + // set slotInUse[i]=false unconditionally; this test is RED until it + // consults flockReleased() like the other two close-and-reclaim paths. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: leave unacked data on disk under default-0 so startup + // recovery treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: ack-ing server + a new pool whose injected factory forges + // the exact leak symptom for the recovery build of slot 0. The + // factory returns a real, flock-holding QwpWebSocketSender but + // pre-sets closed=true, so the recovery close() is a complete no-op + // (checkNotClosed short-circuits drain too): the flock stays held + // and slotLockReleased never flips -- precisely a refused I/O-thread + // stop. flockReleased(recoverer) must therefore report false. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + forged.set(real); + } + return real; + }; + + // minSize=0 so prewarm never adopts slot 0 -- recovery is the + // only builder of slot 0. maxSize=2 so a later borrow can still + // get a fresh slot (default-1), proving capacity dropped by one. + SenderPool pool = newPoolWithFactory(cfg2, 0, 2, 500, factory); + try { + // The forge must actually have reached recovery's build. + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + // The retire branch must have fired during construction. + Assert.assertEquals("recovery must retire the still-locked slot as leaked", + 1, getIntField(pool, "leakedSlots")); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertTrue("retired slot 0 must stay reserved", slotInUse[0]); + Assert.assertFalse("slot 1 must remain free", slotInUse[1]); + + // A later borrow must take the fresh slot 1, never re-pick + // the still-locked default-0 (which would throw "sf slot + // already in use"). + PooledSender b = pool.borrow(); + try { + Assert.assertTrue("borrow must use a fresh slot dir", + Files.exists(slot("default-1"))); + // Capacity stays reduced while the flock is held: + // max=2, one leaked + one live => the next borrow times + // out rather than colliding on the locked dir. + try { + pool.borrow(); + Assert.fail("capacity must be reduced by the leaked slot"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + } + } finally { + b.close(); + } + } finally { + pool.close(); + // Release the forged recoverer's real flock + native + // resources: pool.close() never saw it (recovery never + // added the recoverer to `all`), so un-forge closed and + // close it for real, otherwise assertMemoryLeak trips. + Sender leaked = forged.get(); + if (leaked != null) { + setBooleanField(leaked, "closed", false); + leaked.close(); + } + } + } + }); + } + + @Test + public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Exception { + // Recovery twin of + // testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld: a + // slot retired by STARTUP recovery (recoverer close() returned with + // the flock still held) must not stay lost until process exit. + // isSlotLockReleased() is no longer a one-shot snapshot -- deferred + // engine cleanup on a worker exit path can release the flock later -- + // so the pool must keep the recoverer in retiredSlots and re-probe it. + // Pre-fix, startup recovery only ticked leakedSlots and dropped the + // recoverer: at maxSize=1 every later borrow timed out forever even + // after the flock dropped. This test is RED until the recoverer is + // retained. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1 -- the worst case, where the single slot's + // retirement starves the whole pool. Forge the retention exactly + // like the retire test (closed=true makes the recovery drain and + // close a no-op, so the real flock stays held and slotLockReleased + // stays false), but only for the FIRST build of slot 0: the + // post-recovery borrow below must get a working delegate on the + // recovered index. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 500, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // While the flock is genuinely held, borrows time out: the + // cap-check re-probe finds the flock still reported held. + try { + pool.borrow(); + Assert.fail("borrow must time out while the slot is retired"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + } + + // The late release: the "wedged worker" finishes and the + // flock genuinely drops (un-forge and close the recoverer + // for real; in production this flip comes from + // isSlotLockReleased() re-probing the retained engine after + // the worker exited). + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + // The capacity-starved borrow must re-probe the startup- + // retired slot, recover its capacity, and create on the + // freed index -- no housekeeper tick required. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the startup-retired slot's capacity", + 0, pool.leakedSlotCount()); + boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); + Assert.assertTrue("recovered index 0 must carry the new borrow", slotInUse[0]); + Assert.assertEquals(1, countSlotDirs()); + } finally { + b.close(); + } + } + } + }); + } + + @Test + public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Exception { + // Boundary twin of testStartupRetiredSlotRecoveredAfterLateFlockRelease: + // acquireTimeoutMillis=0 is a valid try-once borrow (builder rejects only + // < 0). Pre-fix, borrow() ran the terminal timeout check BEFORE + // reprobeRetiredSlots(), so a zero-budget borrow threw "timed out" + // without its one probe -- even when the retired slot's flock had + // already dropped and a probe would have restored capacity and admitted + // a creation. Deterministic: no housekeeper runs in this test, so + // borrow() is the only reprobe driver. Recovery is driven manually via + // the deferred-pool step helper because the inline path reuses + // acquireTimeoutMillis as its recovery budget -- 0 would skip recovery + // outright. This test is RED until the probe is hoisted above the + // timeout check. TestUtils.assertMemoryLeak(() -> { - // Phase 1 -- silent server. + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int silentPort = silent.getPort(); silent.start(); @@ -686,71 +1803,62 @@ public void testRecoveryReplayThroughPooledSlot() throws Exception { s.close(); } } - // Data must be on disk, unacked, under default-0. - Assert.assertTrue("unacked data must persist on disk", hasSegmentFile(slot("default-0"))); + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); - // Phase 2 -- ack-ing server, brand-new pool, same sf_dir. + // Phase 2: maxSize=1, zero acquire budget, deferred recovery driven + // step-by-step (the housekeeper's per-tick unit, budgeted by + // RECOVERY_DRAIN_BUDGET_MILLIS, not the zero acquire budget). Forge + // the retirement (closed=true makes the recovery drain and close a + // no-op, so the real flock stays held), then release the flock for + // real BEFORE borrowing: the recovery is discoverable, but only via + // a probe. CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { int ackPort = ack.getPort(); ack.start(); Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; - try (SenderPool pool = new SenderPool(cfg2, 1, 1, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - PooledSender s = pool.borrow(); - try { - // Drain replays the recovered, unacked frames. - s.drain(5_000); - Assert.assertTrue("recovered frames must be replayed to the new server", - awaitAtLeast(handler.frames, 1, 5_000)); - } finally { - s.close(); + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + setBooleanField(real, "closed", true); + } catch (Exception e) { + throw new RuntimeException(e); + } } - } - } - }); - } + return real; + }; - @Test - public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { - // M1 regression: a startup-recovery delegate runs on the PoolHousekeeper - // thread, so its build() must NOT inherit the user's SYNC initial-connect - // mode (auto-enabled by any reconnect_* knob). SYNC would retry the - // connect for the whole reconnect budget inside build() -- far past - // PoolHousekeeper.STOP_TIMEOUT_MILLIS -- so a close() landing during that - // build would make the housekeeper join time out and leave the recoverer - // holding the slot flock after close() returned. The recovery factory - // forces initial_connect_mode=OFF (at most one connect attempt); the - // normal factory must still honour the user's promoted SYNC mode. - TestUtils.assertMemoryLeak(() -> { - CountingAckHandler handler = new CountingAckHandler(); - try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { - int ackPort = ack.getPort(); - ack.start(); - Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); - // reconnect_max_duration_millis set, initial_connect_mode unset - // -> the builder promotes to SYNC for ordinary senders. - String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir - + ";reconnect_max_duration_millis=30000;"; - // min=0 (no prewarm connect), no stranded data (recovery no-op). - try (SenderPool pool = new SenderPool(cfg, 0, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - // Normal managed-slot delegate: inherits the promoted SYNC. - Sender normal = invokeBuildSlotDelegate(pool, "defaultSender", 0); - try { - Assert.assertEquals( - "ordinary pooled sender must honour the user's promoted SYNC mode", - Sender.InitialConnectMode.SYNC, readInitialConnectMode(normal)); - } finally { - normal.close(); + try (SenderPool pool = newDeferredPoolWithFactory(cfg2, 0, 1, 0, factory)) { + //noinspection StatementWithEmptyBody + while (invokeRunStartupRecoveryStep(pool)) { + // drive the whole backlog, one housekeeper-tick unit at a time } - // Recovery delegate on a different slot: forced OFF. - Sender recoverer = invokeBuildSlotDelegate(pool, "defaultRecoverySender", 1); + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // The late release: un-forge and close the recoverer for + // real. The flock genuinely drops, but nothing signals the + // pool -- the release happens in the delegate, volatile + // writes only. + Sender recoverer = forged.get(); + setBooleanField(recoverer, "closed", false); + recoverer.close(); + + // Try-once borrow: its single pass must probe, recover the + // capacity, and create on the freed index. Pre-fix this + // threw "timed out" without ever probing. + PooledSender b = pool.borrow(); try { - Assert.assertEquals( - "recovery delegate must force OFF so build() makes at most one connect attempt", - Sender.InitialConnectMode.OFF, readInitialConnectMode(recoverer)); + Assert.assertEquals("zero-timeout borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); } finally { - recoverer.close(); + b.close(); } } } @@ -758,19 +1866,16 @@ public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { } @Test - public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception { - // C1 regression: the startup recovery loop MUST mirror discardBroken / - // reapIdle. When a recoverer's delegate close() returns with the SF - // flock still held (the I/O thread refused to stop), the recovered slot - // index must be retired permanently (leakedSlots++, slotInUse stays - // set) -- NOT freed. Freeing it would let a later borrow re-pick the - // still-locked dir and resurrect "sf slot already in use", the exact - // failure class this PR exists to kill. Pre-fix the recovery finally - // set slotInUse[i]=false unconditionally; this test is RED until it - // consults flockReleased() like the other two close-and-reclaim paths. + public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception { + // Positive-timeout twin of the zero-timeout test. A borrower parks in + // awaitNanos while the retired slot's flock is genuinely held and + // sleeps out its full budget. A test hook releases the flock after the + // wait reports expiry but before the terminal loop pass. Pre-fix that + // pass hit the timeout check before reprobeRetiredSlots() and threw -- + // missing capacity that had already come back. Post-fix the terminal + // pass probes first, recovers the index, and admits the creation. TestUtils.assertMemoryLeak(() -> { - // Phase 1: leave unacked data on disk under default-0 so startup - // recovery treats it as a candidate orphan and builds a recoverer. + // Phase 1: strand unacked data under default-0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int silentPort = silent.getPort(); silent.start(); @@ -789,13 +1894,9 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th Assert.assertTrue("unacked data must persist under default-0", hasSegmentFile(slot("default-0"))); - // Phase 2: ack-ing server + a new pool whose injected factory forges - // the exact leak symptom for the recovery build of slot 0. The - // factory returns a real, flock-holding QwpWebSocketSender but - // pre-sets closed=true, so the recovery close() is a complete no-op - // (checkNotClosed short-circuits drain too): the flock stays held - // and slotLockReleased never flips -- precisely a refused I/O-thread - // stop. flockReleased(recoverer) must therefore report false. + // Phase 2: maxSize=1 and a positive acquire budget. A test hook + // releases the flock only after awaitNanos() has returned with that + // budget exhausted, so the terminal wake-up pass is deterministic. CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { int ackPort = ack.getPort(); @@ -806,60 +1907,48 @@ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() th AtomicReference forged = new AtomicReference<>(); IntFunction factory = idx -> { Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); - if (idx == 0) { + if (idx == 0 && forged.compareAndSet(null, real)) { try { setBooleanField(real, "closed", true); } catch (Exception e) { throw new RuntimeException(e); } - forged.set(real); } return real; }; - // minSize=0 so prewarm never adopts slot 0 -- recovery is the - // only builder of slot 0. maxSize=2 so a later borrow can still - // get a fresh slot (default-1), proving capacity dropped by one. - SenderPool pool = newPoolWithFactory(cfg2, 0, 2, 500, factory); - try { - // The forge must actually have reached recovery's build. + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 100, factory)) { Assert.assertNotNull("recovery must have built slot 0", forged.get()); - // The retire branch must have fired during construction. - Assert.assertEquals("recovery must retire the still-locked slot as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("retired slot 0 must stay reserved", slotInUse[0]); - Assert.assertFalse("slot 1 must remain free", slotInUse[1]); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); - // A later borrow must take the fresh slot 1, never re-pick - // the still-locked default-0 (which would throw "sf slot - // already in use"). - PooledSender b = pool.borrow(); + AtomicBoolean waitExpired = new AtomicBoolean(); + pool.setBorrowWaitExpiredHook(() -> { + Assert.assertTrue("expired-wait hook must run exactly once", + waitExpired.compareAndSet(false, true)); + Sender recoverer = forged.get(); + try { + setBooleanField(recoverer, "closed", false); + } catch (Exception e) { + throw new RuntimeException(e); + } + // The flock drops only after the positive awaitNanos() + // budget is exhausted. This delegate-side release does + // not signal slotReleased. + recoverer.close(); + }); try { - Assert.assertTrue("borrow must use a fresh slot dir", - Files.exists(slot("default-1"))); - // Capacity is permanently reduced by the leaked slot: - // max=2, one leaked + one live => the next borrow times - // out rather than colliding on the locked dir. + PooledSender b = pool.borrow(); try { - pool.borrow(); - Assert.fail("capacity must be reduced by the leaked slot"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + Assert.assertTrue("borrow must exhaust its positive wait budget", + waitExpired.get()); + Assert.assertEquals("wake-up probe must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); } } finally { - b.close(); - } - } finally { - pool.close(); - // Release the forged recoverer's real flock + native - // resources: pool.close() never saw it (recovery never - // added the recoverer to `all`), so un-forge closed and - // close it for real, otherwise assertMemoryLeak trips. - Sender leaked = forged.get(); - if (leaked != null) { - setBooleanField(leaked, "closed", false); - leaked.close(); + pool.setBorrowWaitExpiredHook(null); } } } @@ -1306,6 +2395,163 @@ public void testDefaultConfigRecoversOutOfRangeSlotsAfterShrink() throws Excepti }); } + @Test + public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exception { + // A deferred pool can already have its sole in-range sender borrowed when + // startup recovery reaches an out-of-range slot left by a larger pool. + // A transient recoverer build failure must leave that candidate pending: + // after the primary lease returns, the SAME pool must retry and drain it. + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-1;close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", 1L).atNow(); + seed.flush(); + } + } + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean primaryReturned = new AtomicBoolean(); + AtomicInteger recoveryAttempts = new AtomicInteger(); + IntFunction factory = idx -> { + if (idx == 1) { + recoveryAttempts.incrementAndGet(); + if (!primaryReturned.get()) { + throw new LineSenderException("transient out-of-range recovery failure"); + } + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + PooledSender primary = pool.borrow(); + Object primarySlot = slotOf(primary); + + Assert.assertFalse("first recovery attempt must stop at the transient failure", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("exactly one out-of-range recovery attempt", 1, + recoveryAttempts.get()); + Assert.assertTrue("failed recovery must preserve the candidate", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("out-of-range failure must not consume in-range capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range recoverer must not enter retired-slot bookkeeping", + ((List) getField(pool, "retiredSlots")).isEmpty()); + + primary.close(); + primaryReturned.set(true); + invokeRunStartupRecoveryOnce(pool); + + Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, + recoveryAttempts.get()); + Assert.assertFalse("retry must drain the preserved out-of-range data", + hasSegmentFile(slot("default-1"))); + Assert.assertTrue("retry must deliver the recovered frame", handler.frames.get() >= 1); + Assert.assertEquals("successful out-of-range retry must not consume capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range retry must leave retired slots untouched", + ((List) getField(pool, "retiredSlots")).isEmpty()); + + PooledSender next = pool.borrow(); + try { + Assert.assertSame("normal borrow must reuse the returned primary slot", + primarySlot, slotOf(next)); + } finally { + next.close(); + } + } + } + }); + } + + @Test + public void testDrainFailureRetriesInRangeAndOutOfRangeCandidates() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + if (attempts[idx].incrementAndGet() == 1) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("drain".equals(method.getName())) { + throw new LineSenderException("transient drain failure for slot " + idx); + } + if ("close".equals(method.getName())) { + return null; + } + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + }); + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertFalse("failed in-range drain must defer the same candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals(1, attempts[0].get()); + Assert.assertEquals(0, attempts[1].get()); + Assert.assertTrue("failed in-range drain must preserve durable data", + hasSegmentFile(slot("default-0"))); + + Assert.assertTrue("successful in-range retry may continue scanning", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("same live pool must retry the in-range candidate", + 2, attempts[0].get()); + Assert.assertFalse("in-range retry must drain the preserved data", + hasSegmentFile(slot("default-0"))); + + Assert.assertFalse("failed out-of-range drain must defer the same candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals(1, attempts[1].get()); + Assert.assertTrue("failed out-of-range drain must preserve durable data", + hasSegmentFile(slot("default-1"))); + + Assert.assertTrue("successful out-of-range retry may finish the candidate", + invokeRunStartupRecoveryStep(pool)); + Assert.assertEquals("same live pool must retry the out-of-range candidate", + 2, attempts[1].get()); + Assert.assertFalse("out-of-range retry must drain the preserved data", + hasSegmentFile(slot("default-1"))); + Assert.assertFalse("final scan step must mark recovery complete", + invokeRunStartupRecoveryStep(pool)); + Assert.assertTrue("both recovered frames must be delivered", handler.frames.get() >= 2); + } + } + }); + } + @Test public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer @@ -1806,6 +3052,134 @@ private String slot(String name) { return sfDir + "/" + name; } + private void assertPreallocatedExitHandoffCleansStartupRecoverer( + int strandedIndex, int maxSize) throws Exception { + TestUtils.assertMemoryLeak(() -> { + String strandedId = "default-" + strandedIndex; + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=100;"; + Sender sender = Sender.builder(config).senderId(strandedId).build(); + sender.table("recover").longColumn("v", strandedIndex).atNow(); + sender.flush(); + try { + sender.close(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("drain timed out")); + } + } + Assert.assertTrue("startup-recovery fixture must contain an unacked segment", + hasSegmentFile(slot(strandedId))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean instrumented = new AtomicBoolean(); + AtomicReference engineRef = new AtomicReference<>(); + AtomicReference managerRef = new AtomicReference<>(); + AtomicReference hookErr = new AtomicReference<>(); + CountDownLatch releaseWorker = new CountDownLatch(1); + IntFunction factory = idx -> { + Sender sender = Sender.builder(config).senderId("default-" + idx).build(); + if (idx == strandedIndex && instrumented.compareAndSet(false, true)) { + try { + CursorSendEngine engine = (CursorSendEngine) getField(sender, "cursorEngine"); + SegmentManager manager = (SegmentManager) getField(engine, "manager"); + CountDownLatch workerBlocked = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting to release recovery worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + if (!workerBlocked.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("recovery manager never entered a service pass"); + } + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engineRef.set(engine); + managerRef.set(manager); + } catch (Throwable t) { + try { + sender.close(); + } catch (Throwable ignored) { + } + throw new RuntimeException(t); + } + } + return sender; + }; + + SenderPool pool = null; + try { + pool = newPoolWithFactory(config, 0, maxSize, 2_000, factory); + CursorSendEngine engine = engineRef.get(); + SegmentManager manager = managerRef.get(); + Assert.assertNotNull("startup recovery must build the stranded slot", engine); + if (strandedIndex < maxSize) { + Assert.assertEquals("in-range recoverer must remain retired while worker is live", + 1, pool.leakedSlotCount()); + } else { + Assert.assertEquals("out-of-range recovery must not consume pool capacity", + 0, pool.leakedSlotCount()); + } + Assert.assertFalse("cleanup must remain pending while the worker is live", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + manager.close(); + Assert.assertTrue("recovery manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must complete startup-recoverer cleanup without " + + "a sender or engine close retry [index=" + strandedIndex + "]", + engine.isCloseCompleted()); + if (hookErr.get() != null) { + throw new AssertionError("recovery worker hook failed", hookErr.get()); + } + if (strandedIndex < maxSize) { + pool.reapIdle(); + Assert.assertEquals("late cleanup must restore in-range pool capacity", + 0, pool.leakedSlotCount()); + } + try (SlotLock ignored = SlotLock.acquire(slot(strandedId))) { + // Completion must mean the real slot flock is reusable. + } + } finally { + releaseWorker.countDown(); + SegmentManager manager = managerRef.get(); + if (manager != null) { + manager.setBeforeExitCleanupRegistrationHook(null); + manager.setBeforeTrimSyncHook(null); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + } + CursorSendEngine engine = engineRef.get(); + if (engine != null && !engine.isCloseCompleted()) { + engine.close(); + } + if (pool != null) { + pool.close(); + } + } + } + }); + } + private int countSlotDirs() { if (!Files.exists(sfDir)) { return 0; @@ -1940,12 +3314,24 @@ private static void setBooleanField(Object target, String name, boolean value) t f.setBoolean(target, value); } + private static void setLongField(Object target, String name, long value) throws Exception { + Field f = target.getClass().getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + private static int getIntField(Object target, String name) throws Exception { Field f = target.getClass().getDeclaredField(name); f.setAccessible(true); return f.getInt(target); } + private static long getLongField(Object target, String name) throws Exception { + Field f = target.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.getLong(target); + } + private static Object getField(Object target, String name) throws Exception { Field f = target.getClass().getDeclaredField(name); f.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java new file mode 100644 index 00000000..3a737f50 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -0,0 +1,627 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.network; + +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.KqueueFacade; +import io.questdb.client.network.KqueueFacadeImpl; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.NetworkFacadeImpl; +import io.questdb.client.network.PlainSocket; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Os; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class SocketTrafficShutdownTest { + private static final NetworkFacade NF = NetworkFacadeImpl.INSTANCE; + + private static class CompatibilityNetworkFacade implements NetworkFacade { + private final AtomicInteger closeCount; + + private CompatibilityNetworkFacade(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public int close(int fd) { + closeCount.incrementAndGet(); + return 0; + } + + @Override + public void close(int fd, org.slf4j.Logger logger) { + close(fd); + } + + @Override + public void configureKeepAlive(int fd) { + } + + @Override + public int configureNonBlocking(int fd) { + return 0; + } + + @Override + public int connect(int fd, long pSockaddr) { + return 0; + } + + @Override + public int connectAddrInfo(int fd, long pAddrInfo) { + return 0; + } + + @Override + public int connectAddrInfoTimeout(int fd, long pAddrInfo, int timeoutMillis) { + return 0; + } + + @Override + public int errno() { + return 0; + } + + @Override + public void freeAddrInfo(long pAddrInfo) { + } + + @Override + public void freeSockAddr(long pSockaddr) { + } + + @Override + public long getAddrInfo(CharSequence host, int port) { + return 0; + } + + @Override + public int getSndBuf(int fd) { + return 0; + } + + @Override + public int recvRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendToRaw(int fd, long lo, int len, long socketAddress) { + return 0; + } + + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + return 0; + } + + @Override + public int setMulticastInterface(int fd, int ipv4Address) { + return 0; + } + + @Override + public int setMulticastTtl(int fd, int ttl) { + return 0; + } + + @Override + public boolean setSndBuf(int fd, int size) { + return false; + } + + @Override + public int setTcpNoDelay(int fd, boolean noDelay) { + return 0; + } + + @Override + public long sockaddr(int address, int port) { + return 0; + } + + @Override + public int socketTcp(boolean blocking) { + return 0; + } + + @Override + public int socketUdp() { + return 0; + } + + @Override + public boolean testConnection(int fd, long buffer, int bufferSize) { + return false; + } + } + + private static class CompatibilitySocket implements Socket { + private final AtomicInteger closeCount; + + private CompatibilitySocket(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + + @Override + public int getFd() { + return -1; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void of(int fd) { + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) throws TlsSessionInitFailedException { + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + } + + @Test + public void testCompatibilityDefaultsDoNotBypassCustomTransportOwnership() { + AtomicInteger facadeCloseCount = new AtomicInteger(); + NetworkFacade customFacade = new CompatibilityNetworkFacade(facadeCloseCount); + + PlainSocket plainSocket = new PlainSocket(customFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + plainSocket.of(42); + assertUnsupported(plainSocket::closeTraffic); + Assert.assertEquals("facade compatibility fallback must not release a synthetic descriptor", + 0, facadeCloseCount.get()); + Assert.assertEquals(42, plainSocket.getFd()); + plainSocket.close(); + Assert.assertEquals(1, facadeCloseCount.get()); + + AtomicInteger socketCloseCount = new AtomicInteger(); + Socket customSocket = new CompatibilitySocket(socketCloseCount); + assertUnsupported(customSocket::closeTraffic); + Assert.assertEquals("socket compatibility fallback must not run destructive close", + 0, socketCloseCount.get()); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownAfterPeerDisconnectRetainsFd() throws Exception { + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + + long buffer = 0; + int fd = -1; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + + peer.close(); + Assert.assertTrue("client must observe the peer disconnect", socket.recv(buffer, 1) < 0); + + socket.closeTraffic(); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + socket.close(); + if (buffer != 0) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + + @Test + public void testPlainSocketShutdownFailureStillThrows() { + AtomicInteger closeCount = new AtomicInteger(); + NetworkFacade failingFacade = new CompatibilityNetworkFacade(closeCount) { + @Override + public int errno() { + return 1234; + } + + @Override + public int shutdown(int fd) { + Assert.assertEquals(42, fd); + return -1; + } + }; + PlainSocket socket = new PlainSocket(failingFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + socket.of(42); + + try { + socket.closeTraffic(); + Assert.fail("expected genuine traffic shutdown failure"); + } catch (IllegalStateException expected) { + Assert.assertEquals("could not shut down socket traffic [fd=42, errno=1234]", expected.getMessage()); + } finally { + socket.close(); + } + + Assert.assertTrue(socket.isClosed()); + Assert.assertEquals("full close must release facade ownership exactly once", 1, closeCount.get()); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesWindowsRecvAndRetainsFd() throws Exception { + Assume.assumeTrue("real Winsock cancellation coverage runs on Windows", Os.type == Os.WINDOWS); + + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + AtomicBoolean recvDone = new AtomicBoolean(); + AtomicInteger recvResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference recvFailure = new AtomicReference<>(); + CountDownLatch recvStarted = new CountDownLatch(1); + + long buffer = 0; + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + long recvBuffer = buffer; + + waiter = new Thread(() -> { + recvStarted.countDown(); + try { + recvResult.set(socket.recv(recvBuffer, 1)); + } catch (Throwable t) { + recvFailure.set(t); + } finally { + recvDone.set(true); + } + }, "socket-traffic-windows-recv-waiter"); + waiter.setDaemon(true); + waiter.start(); + + Assert.assertTrue("waiter did not reach the receive call", + recvStarted.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the receive complete", recvDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native receive", waiter.isAlive()); + Assert.assertNull("native receive failed", recvFailure.get()); + Assert.assertTrue("shutdown must disconnect the native receive", recvResult.get() < 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the Winsock descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released Winsock descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + if (waiter != null && waiter.isAlive()) { + try { + socket.closeTraffic(); + } catch (Throwable ignored) { + // Full close below is the final wake-up fallback. + } + } + socket.close(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + if (buffer != 0 && (waiter == null || !waiter.isAlive())) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + + @Test + public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exception { + AtomicInteger closeCount = new AtomicInteger(); + AtomicInteger shutdownCount = new AtomicInteger(); + NetworkFacade facade = (NetworkFacade) Proxy.newProxyInstance( + NetworkFacade.class.getClassLoader(), + new Class[]{NetworkFacade.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCount.incrementAndGet(); + return method.getReturnType() == int.class ? 0 : null; + } + if ("shutdown".equals(method.getName())) { + shutdownCount.incrementAndGet(); + return 0; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == long.class) { + return 0L; + } + return null; + } + ); + Socket socket = JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + facade, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + ); + socket.of(42); + Field sslEngineField = socket.getClass().getDeclaredField("sslEngine"); + Field stateField = socket.getClass().getDeclaredField("state"); + sslEngineField.setAccessible(true); + stateField.setAccessible(true); + sslEngineField.set(socket, SSLContext.getDefault().createSSLEngine()); + stateField.setInt(socket, 2); // JavaTlsClientSocket.STATE_TLS + Object[] tlsState = snapshotTlsState(socket); + + try { + socket.closeTraffic(); + + Object[] stateAfterTrafficClose = snapshotTlsState(socket); + for (int i = 0; i < tlsState.length; i++) { + if (i == 2) { + Assert.assertEquals("traffic cancellation must preserve TLS state", tlsState[i], stateAfterTrafficClose[i]); + } else { + Assert.assertSame("traffic cancellation must preserve SSLEngine/buffer references", + tlsState[i], stateAfterTrafficClose[i]); + } + } + Assert.assertEquals(1, shutdownCount.get()); + Assert.assertEquals("traffic cancellation must not release the delegate fd", 0, closeCount.get()); + Assert.assertEquals(42, socket.getFd()); + Assert.assertFalse(socket.isClosed()); + } finally { + // Restore a valid plaintext state so full close does not attempt a + // synthetic TLS close_notify with uninitialised session buffers. + sslEngineField.set(socket, null); + stateField.setInt(socket, 1); // JavaTlsClientSocket.STATE_PLAINTEXT + socket.close(); + } + Assert.assertEquals(1, closeCount.get()); + Assert.assertTrue(socket.isClosed()); + } + + @Test(timeout = 30_000L) + public void testTlsSocketTrafficGateUsesDelegateShutdownAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + NF, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + )); + } + + private static void assertShutdownWakesMacOsKqueue(Socket socket) throws Exception { + Assume.assumeTrue("real kqueue cancellation coverage runs on macOS", Os.type == Os.DARWIN); + + AtomicBoolean pollDone = new AtomicBoolean(); + AtomicInteger pollResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference pollFailure = new AtomicReference<>(); + CountDownLatch pollEntered = new CountDownLatch(1); + KqueueFacade facade = new KqueueFacade() { + private final KqueueFacade delegate = KqueueFacadeImpl.INSTANCE; + + @Override + public NetworkFacade getNetworkFacade() { + return delegate.getNetworkFacade(); + } + + @Override + public int kevent(int kq, long changeList, int nChanges, long eventList, int nEvents, int timeout) { + if (eventList != 0 && nEvents > 0) { + pollEntered.countDown(); + } + return delegate.kevent(kq, changeList, nChanges, eventList, nEvents, timeout); + } + + @Override + public int kqueue() { + return delegate.kqueue(); + } + }; + + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfoTimeout(fd, addrInfo, 5_000)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept(); Kqueue kqueue = new Kqueue(facade, 1)) { + try { + Assert.assertEquals(0, NF.configureNonBlocking(fd)); + socket.of(fd); + int retainedFd = fd; + fd = -1; + + kqueue.setWriteOffset(0); + kqueue.readFD(retainedFd, 0); + Assert.assertEquals(0, kqueue.register(1)); + + waiter = new Thread(() -> { + try { + pollResult.set(kqueue.poll(10_000)); + } catch (Throwable t) { + pollFailure.set(t); + } finally { + pollDone.set(true); + } + }, "socket-traffic-kqueue-waiter"); + waiter.start(); + + Assert.assertTrue("waiter did not enter the native kqueue wait", + pollEntered.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the read wait ready", pollDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native kqueue wait", waiter.isAlive()); + Assert.assertNull("native kqueue wait failed", pollFailure.get()); + Assert.assertTrue("shutdown must produce a readiness event", pollResult.get() > 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + } finally { + socket.closeTraffic(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + socket.close(); + } + } + } finally { + if (fd != -1) { + NF.close(fd); + } + } + } + + private static void assertUnsupported(Runnable operation) { + try { + operation.run(); + Assert.fail("expected unsupported traffic shutdown"); + } catch (UnsupportedOperationException expected) { + // Expected compatibility behavior: no native or destructive fallback. + } + } + + private static Object[] snapshotTlsState(Socket socket) throws Exception { + String[] fieldNames = { + "callerOutputBuffer", + "sslEngine", + "state", + "unwrapInputBuffer", + "unwrapOutputBuffer", + "wrapInputBuffer", + "wrapOutputBuffer" + }; + Object[] state = new Object[fieldNames.length]; + for (int i = 0; i < fieldNames.length; i++) { + Field field = socket.getClass().getDeclaredField(fieldNames[i]); + field.setAccessible(true); + state[i] = field.get(socket); + } + return state; + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java index b2efff7b..cc8c6e8b 100644 --- a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java +++ b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java @@ -28,6 +28,8 @@ import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; + /** * */ @@ -58,6 +60,21 @@ public void testRemoveFromTo() { Assert.assertEquals(list(), remove(list("a", "b", "c"), 4, 10)); } + @Test + public void testRemoveFromToClearsFinalBackingSlot() throws Exception { + ObjList values = new ObjList<>(16); + for (int i = 0; i < 16; i++) { + values.add(new Object()); + } + + values.remove(0, 0); + + Field bufferField = ObjList.class.getDeclaredField("buffer"); + bufferField.setAccessible(true); + Object[] buffer = (Object[]) bufferField.get(values); + Assert.assertNull(buffer[buffer.length - 1]); + } + private static ObjList remove(ObjList o, int from, int to) { o.remove(from, to); return o;