From a168643972c56fca4fc287f522b65718463cb51e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 00:00:35 +0100 Subject: [PATCH 01/14] fix(browser-db-sqlite-persistence): fairly schedule cold hydrations --- .../fix-shared-sqlite-hydration-fairness.md | 7 + .github/workflows/e2e-tests.yml | 5 + .../e2e/shared-driver-fairness.opfs.html | 13 + .../e2e/shared-driver-fairness.opfs.spec.ts | 73 ++ .../e2e/shared-driver-fairness.opfs.ts | 151 ++++ .../package.json | 3 + .../playwright.opfs.config.ts | 25 + .../src/browser-coordinator.ts | 28 +- .../src/wa-sqlite-driver.ts | 34 +- .../tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md | 355 +++++++++ .../shared-driver-fairness-oracle.test.ts | 286 +++++++ .../tests/shared-driver-fairness-oracle.ts | 708 ++++++++++++++++++ .../tsconfig.json | 10 +- .../vite.opfs.config.ts | 30 + .../src/persisted.ts | 342 ++++++--- .../src/sqlite-core-adapter.ts | 276 ++++++- .../tests/persisted.test.ts | 405 ++++++++++ .../tests/shared-logical-scheduling.test.ts | 124 +++ .../src/electron-coordinator.ts | 28 +- pnpm-lock.yaml | 3 + 20 files changed, 2783 insertions(+), 123 deletions(-) create mode 100644 .changeset/fix-shared-sqlite-hydration-fairness.md create mode 100644 packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html create mode 100644 packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts create mode 100644 packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts create mode 100644 packages/browser-db-sqlite-persistence/playwright.opfs.config.ts create mode 100644 packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md create mode 100644 packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts create mode 100644 packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts create mode 100644 packages/browser-db-sqlite-persistence/vite.opfs.config.ts create mode 100644 packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts diff --git a/.changeset/fix-shared-sqlite-hydration-fairness.md b/.changeset/fix-shared-sqlite-hydration-fairness.md new file mode 100644 index 0000000000..d2055d929a --- /dev/null +++ b/.changeset/fix-shared-sqlite-hydration-fairness.md @@ -0,0 +1,7 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/browser-db-sqlite-persistence': patch +'@tanstack/electron-db-sqlite-persistence': patch +--- + +Prevent cold SQLite hydrations from being starved by unrelated queued writes when collections share a browser driver. Schedule each complete hydrate fairly while preserving transaction atomicity and leader-local persistence coordination. diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a340502b46..6b3756ff74 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -74,6 +74,11 @@ jobs: cd examples/react/start-ssr-e2e pnpm exec playwright install --with-deps chromium + - name: Run Browser SQLite OPFS fairness E2E tests + run: | + cd packages/browser-db-sqlite-persistence + pnpm test:opfs-fairness + - name: Run React Start SSR E2E tests run: | cd examples/react/start-ssr-e2e diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html new file mode 100644 index 0000000000..215878cfd3 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html @@ -0,0 +1,13 @@ + + + + + + + Shared driver OPFS fairness oracle + + + running + + + diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts new file mode 100644 index 0000000000..4b99a1db15 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts @@ -0,0 +1,73 @@ +import { expect, test } from '@playwright/test' +import type { Page } from '@playwright/test' +import type { OPFSOracleResult } from './shared-driver-fairness.opfs' + +async function readOracleResult( + page: Page, + mode: `neutral` | `storm`, +): Promise { + await page.goto(`/e2e/shared-driver-fairness.opfs.html?mode=${mode}`) + await page.waitForFunction( + () => window.__tanstackDriverFairnessOracle !== undefined, + ) + return page.evaluate(() => window.__tanstackDriverFairnessOracle!) +} + +function expectedHydratedCollections(scenarioId: string, count: number) { + return Array.from({ length: count }, (_, index) => ({ + collectionId: `${scenarioId}-hydrate-${index}`, + rows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + })) +} + +test(`real Chromium OPFS fixture reaches and cleans up cold hydration`, async ({ + page, +}) => { + const result = await readOracleResult(page, `neutral`) + + if (result.status !== `complete`) throw new Error(result.primaryFailure) + expect(result.provider).toBe(`Chromium OPFSCoopSyncVFS worker`) + expect(result.observation.admittedHydrateIds).toHaveLength(2) + // These are actual public Collection rows captured after preload, compared + // with seed values built independently by this browser assertion. + expect(result.observation.hydratedCollections).toEqual( + expectedHydratedCollections(`opfs-neutral-reach`, 2), + ) + expect( + result.observation.rawDequeues.some((entry) => + entry.sql.startsWith(`SELECT key, value, metadata, row_version FROM`), + ), + ).toBe(true) + expect(result.observation.cleanupFailures).toEqual([]) + expect(result.opfsCleanupFailures).toEqual([]) +}) + +test(`real Chromium OPFS fixture bounds pending cold hydration behind persists`, async ({ + page, +}) => { + const result = await readOracleResult(page, `storm`) + + if (result.status !== `complete`) throw new Error(result.primaryFailure) + expect(result.provider).toBe(`Chromium OPFSCoopSyncVFS worker`) + expect(result.observation.admittedHydrateIds).toHaveLength(4) + expect(result.observation.hydratedCollections).toEqual( + expectedHydratedCollections(`opfs-fixed-persist-storm`, 4), + ) + // This is the semantic RED checkpoint. Setup, wall time, and cleanup are + // reported independently and cannot satisfy this assertion. + if (result.violation !== undefined) { + throw new Error( + `real OPFS fairness mismatch: ${JSON.stringify(result.violation)}; ` + + `logical completion order: ${JSON.stringify(result.observation.logicalCompletionOrder)}; ` + + `driver admissions: ${result.observation.driverAdmissions.length}; ` + + `raw dequeues: ${result.observation.rawDequeues.length}; ` + + `driver cleanup diagnostics: ${JSON.stringify(result.observation.cleanupFailures)}; ` + + `OPFS cleanup diagnostics: ${JSON.stringify(result.opfsCleanupFailures)}`, + ) + } + expect(result.observation.cleanupFailures).toEqual([]) + expect(result.opfsCleanupFailures).toEqual([]) +}) diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts new file mode 100644 index 0000000000..533155912c --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts @@ -0,0 +1,151 @@ +import { openBrowserWASQLiteOPFSDatabase } from '../src/index' +import { + findSharedDriverFairnessViolation, + observeSharedDriverFairness, +} from '../tests/shared-driver-fairness-oracle' +import type { + SharedDriverFairnessObservation, + SharedDriverFairnessScenario, + SharedDriverFairnessViolation, +} from '../tests/shared-driver-fairness-oracle' + +export type OPFSOracleResult = + | { + status: `complete` + provider: `Chromium OPFSCoopSyncVFS worker` + observation: SharedDriverFairnessObservation + violation: SharedDriverFairnessViolation | undefined + opfsCleanupFailures: ReadonlyArray + } + | { + status: `failed-before-checkpoint` + provider: `Chromium OPFSCoopSyncVFS worker` + primaryFailure: string + opfsCleanupFailures: ReadonlyArray + } + +declare global { + interface Window { + __tanstackDriverFairnessOracle?: OPFSOracleResult + } +} + +async function removeOPFSArtifacts( + databaseName: string, +): Promise> { + const failures: Array = [] + const root = await navigator.storage.getDirectory() + for (const suffix of [``, `-journal`, `-wal`]) { + try { + await root.removeEntry(`${databaseName}${suffix}`) + } catch (error) { + if (!(error instanceof DOMException && error.name === `NotFoundError`)) { + failures.push( + `${databaseName}${suffix}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + } + + // OPFSCoopSyncVFS creates one private temporary access-handle directory per + // worker. This page owns its isolated origin for the fixture run. + const iterableRoot = root as FileSystemDirectoryHandle & { + entries: () => AsyncIterableIterator<[string, FileSystemHandle]> + } + for await (const [name, handle] of iterableRoot.entries()) { + if (handle.kind !== `directory` || !name.startsWith(`.ahp-`)) continue + try { + await root.removeEntry(name, { recursive: true }) + } catch (error) { + failures.push( + `${name}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + return failures +} + +function scenarioFromLocation(): SharedDriverFairnessScenario { + const mode = new URL(location.href).searchParams.get(`mode`) + if (mode === `neutral`) { + return { + id: `opfs-neutral-reach`, + work: [0, 1].map((index) => ({ + kind: `hydrate` as const, + id: `hydrate-${index}`, + seededRows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + })), + } + } + return { + id: `opfs-fixed-persist-storm`, + work: [ + ...Array.from({ length: 5 }, (_, index) => ({ + kind: `persist` as const, + id: `persist-${index}`, + mutationsPerPersist: 2, + })), + ...Array.from({ length: 4 }, (_, index) => ({ + kind: `hydrate` as const, + id: `hydrate-${index}`, + seededRows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + })), + ], + } +} + +async function run(): Promise { + const status = document.querySelector(`#oracle-status`) + const databaseName = `ws5b-${crypto.randomUUID()}.sqlite` + let observation: SharedDriverFairnessObservation | undefined + let violation: SharedDriverFairnessViolation | undefined + let primaryFailure: string | undefined + let opfsCleanupFailures: ReadonlyArray = [] + try { + observation = await observeSharedDriverFairness( + () => openBrowserWASQLiteOPFSDatabase({ databaseName }), + scenarioFromLocation(), + ) + // Freeze the reached semantic checkpoint before attempting OPFS cleanup. + violation = findSharedDriverFairnessViolation(observation) + } catch (error) { + primaryFailure = error instanceof Error ? error.message : String(error) + } + + try { + opfsCleanupFailures = await removeOPFSArtifacts(databaseName) + } catch (cleanupError) { + opfsCleanupFailures = [ + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError), + ] + } + + if (observation) { + window.__tanstackDriverFairnessOracle = { + status: `complete`, + provider: `Chromium OPFSCoopSyncVFS worker`, + observation, + violation, + opfsCleanupFailures, + } + if (status) status.value = `complete` + } else { + window.__tanstackDriverFairnessOracle = { + status: `failed-before-checkpoint`, + provider: `Chromium OPFSCoopSyncVFS worker`, + primaryFailure: primaryFailure ?? `unknown failure before checkpoint`, + opfsCleanupFailures, + } + if (status) status.value = `failed-before-checkpoint` + } +} + +void run() diff --git a/packages/browser-db-sqlite-persistence/package.json b/packages/browser-db-sqlite-persistence/package.json index 9bae7bb83c..26b2b70fef 100644 --- a/packages/browser-db-sqlite-persistence/package.json +++ b/packages/browser-db-sqlite-persistence/package.json @@ -23,6 +23,8 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", + "test:oracles": "vitest --run tests/shared-driver-fairness-oracle.test.ts", + "test:opfs-fairness": "playwright test --config playwright.opfs.config.ts", "test:e2e": "pnpm --filter @tanstack/db-ivm build && pnpm --filter @tanstack/db build && pnpm --filter @tanstack/db-sqlite-persistence-core build && pnpm --filter @tanstack/browser-db-sqlite-persistence build && vitest --config vitest.e2e.config.ts --run" }, "type": "module", @@ -56,6 +58,7 @@ }, "devDependencies": { "@journeyapps/wa-sqlite": "^1.4.1", + "@playwright/test": "^1.60.0", "@types/better-sqlite3": "^7.6.13", "@vitest/coverage-istanbul": "^3.2.4", "better-sqlite3": "^12.6.2" diff --git a/packages/browser-db-sqlite-persistence/playwright.opfs.config.ts b/packages/browser-db-sqlite-persistence/playwright.opfs.config.ts new file mode 100644 index 0000000000..ecfa8af0a6 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/playwright.opfs.config.ts @@ -0,0 +1,25 @@ +import { defineConfig } from '@playwright/test' + +const baseURL = `http://127.0.0.1:4185` +const browserChannel = + process.env.PLAYWRIGHT_CHANNEL ?? (process.env.CI ? undefined : `chrome`) + +export default defineConfig({ + testDir: `./e2e`, + testMatch: `shared-driver-fairness.opfs.spec.ts`, + timeout: 60_000, + fullyParallel: false, + workers: 1, + use: { + baseURL, + ...(browserChannel ? { channel: browserChannel } : {}), + headless: true, + trace: `retain-on-failure`, + }, + webServer: { + command: `vite --config vite.opfs.config.ts --host 127.0.0.1 --port 4185`, + reuseExistingServer: false, + timeout: 120_000, + url: `${baseURL}/e2e/shared-driver-fairness.opfs.html`, + }, +}) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 1babddc5a7..995a63ac94 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -1,6 +1,7 @@ import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' import type { ApplyLocalMutationsResponse, + HydrationPersistenceAdapter, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, @@ -221,9 +222,15 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId: string, signature: string, spec: PersistedIndexSpec, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + // A scoped adapter is a leader-local capability and never crosses RPC. + await (scopedAdapter ?? this.requireAdapter()).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -270,13 +277,19 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina async pullSince( collectionId: string, fromRowVersion: number, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { if (this.isLeader(collectionId)) { - return this.handlePullSince(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) + // A scoped adapter is a leader-local capability and never crosses RPC. + return this.handlePullSince( + collectionId, + { + type: `rpc:pullSince:req`, + rpcId: safeRandomUUID(), + fromRowVersion, + }, + scopedAdapter, + ) } return this.sendRPC(collectionId, { @@ -733,10 +746,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina rpcId: string fromRowVersion: number }, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = scopedAdapter ?? this.requireAdapter() if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, diff --git a/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts b/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts index c1ae91b9e0..3a6be4c901 100644 --- a/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts +++ b/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts @@ -1,4 +1,7 @@ -import { InvalidPersistedCollectionConfigError } from '@tanstack/db-sqlite-persistence-core' +import { + InvalidPersistedCollectionConfigError, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, +} from '@tanstack/db-sqlite-persistence-core' import type { SQLiteDriver } from '@tanstack/db-sqlite-persistence-core' export type BrowserWASQLiteDatabase = { @@ -36,6 +39,7 @@ function assertDatabaseShape( } export class BrowserWASQLiteDriver implements SQLiteDriver { + readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] = {} private readonly database: BrowserWASQLiteDatabase private queue: Promise = Promise.resolve() private nextSavepointId = 1 @@ -46,29 +50,33 @@ export class BrowserWASQLiteDriver implements SQLiteDriver { this.database = options.database } - async exec(sql: string): Promise { - await this.enqueue(async () => { + exec(sql: string): Promise { + return this.enqueue(async () => { await this.database.execute(sql) }) } - async query( + query( sql: string, params: ReadonlyArray = [], ): Promise> { return this.enqueue(() => this.database.execute(sql, params)) } - async run(sql: string, params: ReadonlyArray = []): Promise { - await this.enqueue(async () => { + run(sql: string, params: ReadonlyArray = []): Promise { + return this.enqueue(async () => { await this.database.execute(sql, params) }) } - async transaction( + transaction( fn: (transactionDriver: SQLiteDriver) => Promise, ): Promise { - assertTransactionCallbackHasDriverArg(fn) + try { + assertTransactionCallbackHasDriverArg(fn) + } catch (error) { + return Promise.reject(error) + } return this.enqueue(async () => { await this.database.execute(`BEGIN IMMEDIATE`) @@ -87,7 +95,7 @@ export class BrowserWASQLiteDriver implements SQLiteDriver { }) } - async transactionWithDriver( + transactionWithDriver( fn: (transactionDriver: SQLiteDriver) => Promise, ): Promise { return this.transaction(fn) @@ -144,6 +152,14 @@ export class BrowserWASQLiteDriver implements SQLiteDriver { private enqueue(operation: () => Promise | T): Promise { const queuedOperation = this.queue.then(operation, operation) + // Brand the exact Promise returned to callers. Transparent wrappers may + // preserve this identity for late discovery; wrappers that create a new + // Promise must forward the driver key before adapter construction. + Object.defineProperty( + queuedOperation, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { value: this[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] }, + ) this.queue = queuedOperation.then( () => undefined, () => undefined, diff --git a/packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md b/packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md new file mode 100644 index 0000000000..a04f607d68 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md @@ -0,0 +1,355 @@ +# RFC #1659 Workstream 5B: RED oracle ledger + +## Frozen baseline and scope + +- Repository/worktree: `powersync-v2-main/.worktrees/rfc-1659-ws5b-driver-fairness-oracle` +- Branch: `rfc-1659-ws5b-driver-fairness-oracle` +- `HEAD`, `origin/main`, merge base, and `github/main`: `76d766e84afbfcde2900a661233dd59e1decd5c2` +- Baseline production driver: `packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts`; its single promise FIFO is unchanged. +- Required guidance read in full before task work: root `AGENTS.md`, `docs/contributing/oracle-tests.md`, and `docs/contributing/oracle-coverage.md`. +- Issue #1752: open, updated `2026-08-19T20:42:53Z`; reports a shared WA-SQLite/OPFS FIFO persist storm delaying hydration. +- RFC #1659: open; Workstream 5 authorizes an explicit bounded-fairness oracle and requires the current persist-first storm as a fault control. +- PR #1837 is evidence-only: open/clean, base `7f6b6438cd3a5b2cfc54ea1d8ad8a2102ea9d699`, head `83fc42f3ef98c11d49c097ec07ca8040259ac147`, updated `2026-09-17T13:36:35Z`. Its files are offline-runtime work, not the browser driver's FIFO. No code or history was borrowed from it or any other candidate. +- Phase 1 plus the explicitly approved circular-gate repair only. No production source, old RFC branch, merge, rebase, or cherry-pick was used. Nothing was committed or pushed. + +## Oracle card + +**Law and source.** RFC #1659 says queued persist work must not starve cold hydration on the shared browser driver. The approved executable law permits the non-preemptible persist already running when hydration is requested, then bounds completed persists at every public hydration-completion checkpoint. The maintainer-approved bound is `K = 1` additional persist between successive complete logical cold-hydrate units. + +**Authority limit.** The maintainer chose `K = 1` and one complete logical cold hydrate, including setup operations, as the priority unit. This does not authorize priority for all reads, a bidirectional policy, transaction preemption, or changes to generic `SQLiteDriver` FIFO/transaction semantics. A query-only priority queue remains insufficient because genuinely cold startup performs registry/schema/metadata/index setup as well as row loading and local application. + +**Legal ordered histories.** The generated input is an explicit sequence of logical `{ kind: hydrate | persist, id, payload }` work, not a tuple of counts. A storm's first item is the persist already non-preemptibly running; the remaining two to seven cold hydrates and one to six additional persists are randomly interleaved. Persist transactions contain one to three legal insert mutations. Every hydrate carries one independently generated collection identity and two independently specified seed rows. A seed connection closes before a fresh adapter/driver admits the generated history. Only unrelated persist tables are prewarmed. + +**Independent ordered reference.** The reference consumes only the explicit history. It preserves FIFO identity order within the persist and hydrate lanes. At hydrate checkpoint `i`, its permitted completed-persist prefix is the already-running persist plus at most `i*K` subsequent persist identities from the history. It also derives the exact expected hydrate identity and minimum pending-persist count. It does not inspect or call the production FIFO. Public row expectations are derived independently from the history's seed payload and compared exactly, including collection identity, missing/extra rows, row order, IDs, and values. + +**Production path and reach.** The fixture uses the real `BrowserWASQLiteDriver`, `createSQLiteCorePersistenceAdapter`, `persistedCollectionOptions`, `createCollection`, and public `preload()` path over one shared database. The circular gate was surgically revised after explicit approval: it records each logical hydrate only after the public `preload()` request returns its Promise, and releases the held `BEGIN IMMEDIATE` after all such requests are pending. It no longer requires underlying registry queries to overtake the held non-preemptible persist. A test-only delegating `SQLiteDriver` still records every actual driver method admission, and the database wrapper separately records raw SQL dequeue order; neither schedules production observations. + +**Checkpoints.** Each public hydration completion records its logical completion ordinal, exact completed persist identities, bounded pending count, and raw dequeue count. Actual hydrated rows are captured from `collection.toArray` after public `preload()` completion; expected rows are never returned as observations. Wall-clock duration is never a verdict. + +**Neutral and hostile controls.** The neutral lane uses a fresh adapter with no queued persists and proves exact public rows plus a real persisted-row query. The hostile lane wraps the real `BrowserWASQLiteDriver` in a test-only global-FIFO scheduler, then traverses the same core adapter, persisted collection, and `preload()` fixture. This preserves the current persist-first fault even after production becomes fair and must be killed by the semantic checker. A separate synthetic observation calibrates the checker only. + +**Failure preservation.** All holds are released and all started promises are settled before collection/driver cleanup. Driver, collection, Node-directory, and OPFS cleanup failures are captured separately. The Node helper freezes the primary assertion before removing its directory. The browser page freezes the reached observation and semantic violation before OPFS cleanup. A cleanup failure is appended to an existing primary failure and can never relabel a reached checkpoint as setup failure. + +## Deterministic package-path receipts + +### Neutral reach + +```sh +../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ + -t 'reaches cold hydration' \ + --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose +``` + +Result: PASS, one selected test. Three fresh-adapter cold hydrates completed; all six actual public rows exactly matched their independently specified seed rows; persisted-row SELECT reach was observed; cleanup diagnostics were empty. + +### Narrow fixed semantic RED + +```sh +../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ + -t 'completes a pending cold hydrate' \ + --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose +``` + +Fixed history: `persist-0, persist-1, persist-2, hydrate-0`, with two mutations in each persist. + +Result: intended semantic FAIL at `fixed-persist-storm-hydrate-0`. Actual public rows and reach controls passed first. All three persists completed and zero remained pending; the candidate allowed only `persist-0` completed and required at least two pending. Cleanup diagnostics: `[]`. This was neither setup, timeout, typecheck, nor cleanup failure. + +### Generated ordered-history semantic RED and replay + +```sh +../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ + -t 'generated ordered' \ + --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose + +TANSTACK_DB_DRIVER_FAIRNESS_SEED=165905 \ +TANSTACK_DB_DRIVER_FAIRNESS_PATH=0 \ +../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ + -t 'generated ordered' \ + --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose +``` + +Both commands reproduce seed `165905`, path `0`, without shrinking: six hydrates, four persists, three mutations per persist, token history `p0,h3,h0,h4,p1,h1,p2,p3,h2,h5`. The executable kind history is `P,H,H,H,P,H,P,P,H,H`. Actual at the first public hydrate checkpoint: four completed, zero pending. The ordered candidate reference permitted only `persist-0` completed and required at least three pending. All actual public rows matched before the semantic verdict; cleanup diagnostics: `[]`. + +### Hostile persist-first controls + +```sh +../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ + -t 'persist-first' \ + --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose +``` + +Result: PASS, two selected tests. The executable scheduling mutant admitted four persists then one cold hydrate through the real browser-driver/core/persisted/preload fixture. The checkpoint observed all four persists complete, zero pending, exact public rows, and no cleanup failure; the checker killed it. The separate synthetic calibration was also rejected with maximum completed `1` and minimum pending `3`. + +## Real Chrome/OPFS receipt + +The checked-in fixture uses `openBrowserWASQLiteOPFSDatabase`, its dedicated worker, WA-SQLite, and `OPFSCoopSyncVFS` in installed Google Chrome. Its Vite config deliberately excludes WA-SQLite from dependency prebundling so the real sibling WASM is served. The Playwright channel defaults to installed Chrome locally, bundled Chromium in CI, and accepts `PLAYWRIGHT_CHANNEL` as an explicit override. + +The existing `browser-single-tab-persisted-collection.e2e.test.ts` uses `createWASQLiteTestDatabase` backed by Node `better-sqlite3`; it earns driver/package evidence only and is not counted as OPFS evidence. + +The permanent checked-in package script was executed as: + +```sh +npm run test:opfs-fairness +``` + +The repository's global `pnpm` launcher hangs before printing even its version, so the equivalent documented workspace spelling, `pnpm --filter @tanstack/browser-db-sqlite-persistence test:opfs-fairness`, could not invoke the same script locally. The exact lockfile package was already present and was linked package-locally: `@playwright/test` and CLI version `1.60.0`. The package script, config, real Chrome, loopback Vite server, worker, and OPFS fixture all executed. The launcher defect is retained as external setup evidence and is not semantic RED. + +Neutral real-OPFS result: PASS. Two fresh-adapter cold hydrates returned four exact actual public rows, persisted-row SELECT reach was observed, and driver plus OPFS cleanup diagnostics were empty. + +Fixed real-OPFS storm semantic RED: + +- History: five persists followed by four cold hydrates; two mutations per persist. +- Logical completion order: `persist-0` through `persist-4`, then `hydrate-0` through `hydrate-3`. +- First hydrate checkpoint: all five persists completed, zero pending, raw dequeue count `111`. +- Candidate allowed only `persist-0` completed and required at least four pending. +- Total: `53` actual driver admissions and `113` raw SQL dequeues. +- Driver cleanup diagnostics `[]`; OPFS cleanup diagnostics `[]`. +- Page status reached `complete`; the fixture reported the frozen semantic observation after cleanup. This was not setup, wall-clock, browser, typecheck, or cleanup failure. + +## Static and hygiene receipts + +- Package-local `@playwright/test`: resolved version `1.60.0`; package-local CLI reports `Version 1.60.0`. +- Package TypeScript: PASS, `tsc --noEmit -p tsconfig.json`. +- Prettier: PASS for every new/changed package source/config file. The repository lockfile's existing full-file format differs from current Prettier; the three-line importer edit preserves adjacent lockfile style. +- ESLint: PASS for oracle, Node test, OPFS page/spec, and configs. +- `git diff --check`: PASS. +- Production-source diff under `packages/browser-db-sqlite-persistence/src` and `packages/db-sqlite-persistence-core/src`: empty. + +## Coverage limits + +- The Node fixture uses a real `BrowserWASQLiteDriver` but a `better-sqlite3` database handle. It proves deterministic package scheduling and SQLite results, not browser or OPFS behavior. +- The Chrome receipt proves the same fixed order through a real worker/WA-SQLite/OPFS path. It is one finite browser/origin/topology and does not establish multi-tab OPFS ownership, elapsed performance, every browser, or statement-batching throughput. +- The oracle establishes starvation order and bounded pending work only. It does not choose a batching implementation, permit transaction preemption, weaken atomic commit, or prove that statement batching solves the issue's root cost. +- `preload()` request is the logical hydrate-admission checkpoint and its completion is the public hydration-completion checkpoint. Actual driver admissions and raw dequeues remain separate reach evidence; the gate no longer requires a registry query to overtake the held non-preemptible persist. +- The generated RED campaign stops at its first deterministic counterexample. Its unchanged broad history domain becomes a multi-run GREEN obligation after an approved fix. + +## Production-fix gate + +Production code must not change until all of the following are true: + +1. Maintainer explicitly chooses the fairness quantum/bound (accepts `K = 1` or supplies another finite value) and states whether the scheduled unit is a logical hydrate spanning cold setup or only individual driver reads. +2. If only individual reads receive priority, the oracle is revised before production work so it does not falsely require cold setup writes to inherit an unapproved lane. If logical hydrate priority is chosen, the owning API/lane boundary is recorded before implementation. +3. Independent read-only loss audit reports PASS on this final Phase 1 diff, receipts, replay, failure-preserving cleanup, real-path hostile control, and scope. +4. Maintainer gives explicit Phase 2 production-fix approval after reviewing the decision above. + +After approval, the smallest correctly owned fix must preserve the unchanged fixed/generated/OPFS inputs and checkpoints, keep an already-running transaction non-preemptible, make all three fairness oracles GREEN, and retain transaction atomicity. Focused/full package suites, typecheck/build/lint/format/replay/cleanup, and a second independent loss audit are then required. + +## Frozen content hashes + +The final non-ledger content hashes and combined manifest hash are recorded here after the last test-only edit. The ledger is hashed separately in the independent audit receipt to avoid a self-referential value. + + + +- `f31c08caf480f5b4e639bac0e2d189a2a6922c70dc3477bff2ac4a082e24af34` `package.json` +- `7bc690c092c30d6c2dc15d2aff7254c92ede3080f68f1006a97a6a3edf57df9b` `tsconfig.json` +- `a094af45b23210b6635683abe999814d53d24411f6f76000157cd57e205be7d9` `pnpm-lock.yaml` +- `fd8a9c633966b8d5531f95ed9465ceb95923550aa5f552c05f7864ac1d9bf9a1` `shared-driver-fairness.opfs.html` +- `a0ac7e13416f6164cf134a240f6b044cb661d98c85991ccff6486cac4c07882f` `shared-driver-fairness.opfs.spec.ts` +- `88ef755a14918ecbf289b9d8e0753012fa1d932bec08be940ed81d9f292606d7` `shared-driver-fairness.opfs.ts` +- `0676bc5223bd00c99a0aa77e7199225a3dedecfddfe590953845f40721094140` `playwright.opfs.config.ts` +- `666ac24f0c6e18f0d4d1e71995b5f6046e4c75918292a1b1896a87eb0cb69890` `shared-driver-fairness-oracle.test.ts` +- `f2446d5e7d8ddbf028dd3fa7252db95bde87bee1d54b40238e91ee675cabaf3d` `shared-driver-fairness-oracle.ts` +- `f9e7b499046544834f0d0cbbc7f9af0c9e92858d99b8bd974f6f51587c1e4550` `vite.opfs.config.ts` +- Tracked binary diff hash (package manifest, package tsconfig, lockfile): `a30c60f449a3332fbdd1824540f9b18fa7b47aa2b5799f8e5fe939c3e40622b9` +- Combined ordered SHA-256 manifest hash for all ten non-ledger files above, from the listed paths in order via `shasum -a 256 ... | shasum -a 256`: `a26caffcdfbac67187b59f8c8054702abe086f97290ede212d4d2861b78b89b5` + +## Independent loss-audit status + +`PASS` on baseline/HEAD/origin-main/merge-base `76d766e84afbfcde2900a661233dd59e1decd5c2`. The independent read-only auditor reproduced the fixed/generated/replay semantic RED, executable mutant and checker-control PASS, package typecheck PASS, and real Chrome/OPFS neutral PASS plus semantic RED. It verified no production-source diff, every prior failure-preservation correction, the combined non-ledger manifest hash, tracked diff hash, and pre-receipt ledger hash `c157b60497f982a9b638a01d461a5340f3dddedde6871b88b5e15ac4649aeb0e`. + +No production authorization follows from this PASS. The numeric fairness quantum and logical-hydrate versus individual-read scheduling unit remain maintainer decisions, followed by explicit Phase 2 approval. + +## Phase 2 gate-blocker receipt + +On `2026-09-18`, the maintainer approved `K = 1` with one complete logical cold hydrate, including registry/schema/metadata setup and local row application, as the non-preemptive priority unit. The intended ownership was a driver-shared two-lane scheduler in the SQLite core adapter, an explicit same-adapter scoped hydrate lease in the persisted runtime, and an unchanged generic `SQLiteDriver` FIFO/transaction contract. + +The frozen Phase 1 gate is incompatible with that exclusive unit: + +1. The fixture starts persist `P0` and holds its real `BEGIN IMMEDIATE` inside `BrowserWASQLiteDriver`. +2. A correct logical-operation scheduler leaves `P0` non-preemptible and queues hydrate `H0` as the next complete adapter operation. +3. The unchanged fixture releases `P0` only after the `collection_registry` query from every hydrate (`H0` through `Hn`) has already been admitted to that same driver FIFO. +4. Because an exclusive `H0` cannot begin and admit its registry query until `P0` completes, `P0` waits for `H0` admission while `H0` waits for `P0`: the frozen gate cannot reach its semantic checkpoint. + +Starting every hydrate callback early would satisfy the admission gate only by overlapping/interleaving multiple logical hydrate units. A duplicate probe query, SQL-text priority inference, or hidden driver-wrapper introspection would be test-shaped and would not implement the approved contract. These alternatives were rejected. + +Phase 2 therefore stopped before runtime wiring, verification, commit, push, or prep-PR. All partial scheduling scaffolding was removed with targeted patches. Production-source diff is empty, `git diff --check` passes, and every frozen non-ledger Phase 1 witness hash still matches. Resolution requires either a surgical oracle-gate revision that observes logical hydrate admission without requiring all underlying registry queries before `P0` release, or an explicit maintainer redefinition allowing overlapping hydrate scopes. The former is recommended. + +## Approved circular-gate repair and repeated RED boundary + +On `2026-09-18`, the maintainer approved the recommended surgical gate revision. Only `shared-driver-fairness-oracle.ts` changed: the fixture now observes the logical public `preload()` requests and releases `P0` after they are all pending. The scenario histories, independent reference, `K = 1` verdicts, seed/replay, public row values, completion checkpoints, neutral/hostile controls, cleanup logic, browser fixture, and every other non-ledger witness remain byte-for-byte unchanged. Production-source diff remains empty. + +Repeated receipts at this revised boundary: + +- Neutral Node reach: PASS; three hydrate requests/completions, six exact actual public rows, persisted-row query reach, cleanup `[]`. +- Fixed Node history: semantic RED at `fixed-persist-storm-hydrate-0`; three completed persists, zero pending, versus maximum one and minimum two; exact rows reached first; cleanup `[]`. +- Generated and explicit replay (`seed 165905`, `path 0`): both semantic RED on the unchanged counterexample and first hydrate checkpoint; four completed, zero pending, versus maximum one and minimum three; cleanup `[]`. +- Executable persist-first package-path mutant and synthetic checker calibration: PASS; both were rejected for the intended fairness violation, with exact public rows and cleanup `[]` for the executable control. +- Real installed Chrome, worker WA-SQLite, and `OPFSCoopSyncVFS`: neutral PASS; storm semantic RED with five completed, zero pending, versus maximum one and minimum four. Logical completion order remained all five persists then all four hydrates; `53` driver admissions, `113` raw dequeues, driver cleanup `[]`, OPFS cleanup `[]`. +- Package TypeScript, ESLint, Prettier check, working/staged `git diff --check`: PASS. Playwright result artifacts were removed after their semantic contents were recorded; no package-local result/report directory remains. + +The first unprivileged browser attempt failed before fixture startup because the sandbox denied loopback listen with `EPERM`; it is classified as setup-only. The explicitly permitted rerun reached both real-browser checkpoints above. This setup failure is not semantic RED. + +The revised test-only boundary is now frozen at the hashes above. Production scheduling remains blocked until a fresh independent read-only loss audit reports PASS on this boundary. + +## Revised RED independent loss-audit PASS receipt + +The fresh independent read-only auditor reported `PASS — no blockers` on the +revised logical-request gate before production work resumed. The exact audited +pre-receipt ledger SHA-256 was +`45ad59441ab46cd001557b189db5520ad8786a436f10f24bdd3115bcd2c25132`. + +The auditor independently confirmed: + +- baseline, `HEAD`, `origin/main`, and merge base + `76d766e84afbfcde2900a661233dd59e1decd5c2`; +- an empty production-source diff and no commit, push, PR, WS5A, PR #1487, or + PR #1837 integration at the audited boundary; +- only the approved logical-`preload()` gate changed, while histories, the + `K = 1` reference, checkpoints, values, controls, replay, cleanup, and the + browser fixture remained unchanged; +- Node neutral and hostile controls PASS; fixed semantic RED `3 complete / 0 +pending` versus `max 1 / min 2`; generated and replay semantic RED `4 / 0` + versus `max 1 / min 3`; +- fresh installed Chrome, worker, WA-SQLite, and OPFS neutral PASS plus storm + semantic RED `5 / 0` versus `max 1 / min 4`, with `53` driver admissions, + `113` raw dequeues, and both cleanup arrays empty; +- package static checks and workspace cleanup PASS; +- frozen tracked-diff SHA-256 + `a30c60f449a3332fbdd1824540f9b18fa7b47aa2b5799f8e5fe939c3e40622b9` + and combined non-ledger manifest SHA-256 + `a26caffcdfbac67187b59f8c8054702abe086f97290ede212d4d2861b78b89b5`. + +This durable receipt opens the already-approved WS5B production gate. It does +not claim GREEN or authorize prep-pr, commit, push, PR, or merge work. + +## Phase 2 GREEN implementation boundary + +The approved production implementation keeps `SQLiteDriver` transaction FIFO +and non-preemption semantics intact while adding an opt-in, driver-shared K=1 +logical scheduler in the SQLite core adapter. A branded promise emitted by the +browser WA-SQLite driver allows transparent driver wrappers to retain the +capability; the hostile global-FIFO wrapper intentionally does not. The core +adapter owns one hydrate queue and one regular queue for all adapters sharing +that driver. A complete scoped hydrate is the scheduling unit. When both lanes +remain queued, one regular operation is admitted between completed hydrates. + +The persisted runtime threads a deliberately unscheduled scoped adapter through +cold startup metadata, index bootstrap, row hydration, buffered transaction +flushes, gap recovery, and reloads. Browser and Electron leader-local +coordinator paths accept that same scoped adapter. Index lifecycle work outside +a hydrate and ordinary persistence continue through the regular lane. + +### Lock-order correction + +An initial full browser-conformance run found six deterministic timeouts. A +temporary diagnostic probe isolated a real lock inversion: an on-demand +`loadSubset` held the hydrate scheduler while waiting for `applyMutex`, while an +update held `applyMutex` while waiting for a regular scheduled persistence +operation. The final code consistently acquires `applyMutex` before entering a +hydrate scope for startup, resume-baseline hydration, `loadSubset`, and +`forceReloadSubset`. Applied-receipt waiting remains outside the scheduler +scope, and `hydrateBaseline` no longer reacquires the mutex. All temporary +`WS5B-PROBE`/`WS5B-RUNTIME` diagnostics were removed before final verification. + +### Final GREEN receipts + +- Exact formerly deadlocked browser case, `should maintain query state during +data changes`: `1/1` PASS. +- Full browser persisted conformance: `113/113` PASS. +- Unchanged fixed, generated, neutral, executable-hostile, and synthetic + fairness oracle: `5/5` PASS. +- Explicit unchanged replay, seed `165905`, path `0`: `1/1` PASS without + shrinking. +- Index bootstrap plus both hydration-buffer replay controls: `3/3` PASS. +- Full runtime packages with package typecheck disabled in Vitest: SQLite core + `96/96`, Browser `39/39`, Electron `27/27` PASS. +- Direct package TypeScript for SQLite core, Browser, and Electron: PASS. +- Dependency-order builds for db-ivm, db, SQLite core, Browser, and Electron: + PASS. +- Fresh installed Chrome, worker, WA-SQLite, and OPFS verification: neutral and + storm `2/2` PASS. Both fixtures reached their semantic checkpoints and empty + cleanup diagnostics. +- Focused ESLint: zero errors; one unchanged `require-await` warning remains on + `applyTargetedInvalidationUnsafe`. +- Prettier check, `git diff --check`, temporary-probe scan, and generated + result/coverage cleanup: PASS. + +The repository's global `pnpm` launcher still hangs during workspace discovery, +including for `pnpm --version`. Final package commands therefore used the exact +installed workspace binaries directly; the real OPFS lane used the checked-in +`npm run test:opfs-fairness` script. This is launcher/setup evidence only and +does not weaken any semantic result. + +### Final candidate content hashes + +The frozen RED/oracle files retain their recorded hashes. The final GREEN +candidate contains the following fifteen non-ledger files in this exact order: + +- `f31c08caf480f5b4e639bac0e2d189a2a6922c70dc3477bff2ac4a082e24af34` `packages/browser-db-sqlite-persistence/package.json` +- `d054c227abf7e5a87ef64779be5edea11a339d3d10967ca037243f038e36c3e5` `packages/browser-db-sqlite-persistence/src/browser-coordinator.ts` +- `747614816fc80b16bf5547c7ab571834568fc4786310f6f9ddfcf266b0ffd088` `packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts` +- `7bc690c092c30d6c2dc15d2aff7254c92ede3080f68f1006a97a6a3edf57df9b` `packages/browser-db-sqlite-persistence/tsconfig.json` +- `22a2b48d0a9a52f010b4913521e0d7a5f923cd4706d5d2a915630d9772caf69b` `packages/db-sqlite-persistence-core/src/persisted.ts` +- `86d58d2f0f7da96eda80d0b760721cefb8afb3334872310ed5c61da7016a5bf4` `packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts` +- `9f6d09f27a0e0e0ccd53b5421b7f0df33fe860baaf42a8e47c34dfd8f52125be` `packages/electron-db-sqlite-persistence/src/electron-coordinator.ts` +- `a094af45b23210b6635683abe999814d53d24411f6f76000157cd57e205be7d9` `pnpm-lock.yaml` +- `fd8a9c633966b8d5531f95ed9465ceb95923550aa5f552c05f7864ac1d9bf9a1` `packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html` +- `a0ac7e13416f6164cf134a240f6b044cb661d98c85991ccff6486cac4c07882f` `packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts` +- `88ef755a14918ecbf289b9d8e0753012fa1d932bec08be940ed81d9f292606d7` `packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts` +- `0676bc5223bd00c99a0aa77e7199225a3dedecfddfe590953845f40721094140` `packages/browser-db-sqlite-persistence/playwright.opfs.config.ts` +- `666ac24f0c6e18f0d4d1e71995b5f6046e4c75918292a1b1896a87eb0cb69890` `packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts` +- `f2446d5e7d8ddbf028dd3fa7252db95bde87bee1d54b40238e91ee675cabaf3d` `packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts` +- `f9e7b499046544834f0d0cbbc7f9af0c9e92858d99b8bd974f6f51587c1e4550` `packages/browser-db-sqlite-persistence/vite.opfs.config.ts` + +- Ordered combined non-ledger content-manifest SHA-256: + `b1b151dfd40cade3da4c30c2418642787bbf2a37a6a4f08106d8ea4f1fe9e9be`. +- Tracked binary diff SHA-256 for the eight modified tracked files: + `516ef26f49204b7e36f437dba92667193dd2221ab366473e343ae8f5fc8c5573`. + +`HEAD` remains the frozen baseline +`76d766e84afbfcde2900a661233dd59e1decd5c2`. Nothing has been committed, +pushed, or prepared as a PR. A fresh independent read-only final loss audit is +required before this GREEN boundary may advance to prep-pr. + +## Final independent GREEN loss-audit PASS receipt + +The fresh independent read-only auditor reported `PASS` on the frozen GREEN +candidate. The exact audited pre-receipt hashes were: + +- candidate content manifest: + `b1b151dfd40cade3da4c30c2418642787bbf2a37a6a4f08106d8ea4f1fe9e9be`; +- tracked binary diff: + `516ef26f49204b7e36f437dba92667193dd2221ab366473e343ae8f5fc8c5573`; +- ledger: + `bbf71465e6e4cf5e877c8e180810ed93b51f9deeda2395c416eaf65bf7e09d27`. + +The auditor independently confirmed every frozen RED/oracle per-file hash and +reproduced fairness `5/5`, replay seed `165905` path `0` `1/1`, browser +conformance `113/113`, SQLite core/Browser/Electron `96/96 + 39/39 + 27/27`, +index/bootstrap-buffer controls `3/3`, all three package typechecks, and real +installed Chrome/worker/WA-SQLite/OPFS `2/2` including cleanup assertions. The +initial sandboxed OPFS attempt failed to bind loopback with `EPERM`; the +permitted rerun passed, so that first attempt remains setup-only evidence. + +Independent semantic inspection confirmed K=1 alternation, non-preemptible +running transactions, the unscheduled scoped hydration adapter preventing +recursive scheduling, capability preservation through transparent promise +wrappers while the hostile FIFO wrapper remains unbranded, mutex-before- +scheduler lock ordering, scoped index/buffer/gap-recovery paths, and Browser and +Electron leader-local adapter propagation. ESLint had zero errors, Prettier and +diff checks passed, and only the existing `require-await` warning remained. + +Workspace status was exactly eight modified tracked files plus the expected +eight untracked ledger/oracle files, with nothing staged and no package-local +test-result contamination. Two `.last-run.json` receipts existed only under +`/private/tmp`. The auditor did not rerun builds in its read-only pass; the +coordinator's final dependency-order five-build PASS above remains the build +receipt. + +Residual coverage limits remain the single installed-Chrome/single-origin +topology. During the audit, local `github/main` had advanced to +`dffb17f34a5e0448b7d72f85d11e35ac8a0d4265`; the candidate `HEAD`, +`origin/main`, and merge base remained the frozen baseline +`76d766e84afbfcde2900a661233dd59e1decd5c2`. Any later prep-pr phase must fetch +and reconcile current main under its own approval gate. This PASS freezes WS5B +GREEN and does not authorize prep-pr, commit, push, PR creation, or merge. diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts new file mode 100644 index 0000000000..c0a857ba2c --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts @@ -0,0 +1,286 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' +import { + SHARED_DRIVER_FAIRNESS_BOUND, + createPersistFirstFaultObservation, + findSharedDriverFairnessViolation, + observeSharedDriverFairness, +} from './shared-driver-fairness-oracle' +import { createWASQLiteTestDatabase } from './helpers/wa-sqlite-test-db' +import type { + SharedDriverFairnessObservation, + SharedDriverFairnessOptions, + SharedDriverFairnessScenario, + SharedDriverFairnessWork, +} from './shared-driver-fairness-oracle' + +const DEFAULT_SEED = 165_905 +const replaySeed = Number( + process.env.TANSTACK_DB_DRIVER_FAIRNESS_SEED ?? DEFAULT_SEED, +) +const replayPath = process.env.TANSTACK_DB_DRIVER_FAIRNESS_PATH + +type WorkKind = SharedDriverFairnessWork[`kind`] + +function createScenario( + id: string, + orderedKinds: ReadonlyArray, + mutationsPerPersist = 1, +): SharedDriverFairnessScenario { + let hydrateIndex = 0 + let persistIndex = 0 + return { + id, + work: orderedKinds.map((kind) => { + if (kind === `persist`) { + const index = persistIndex++ + return { + kind, + id: `persist-${index}`, + mutationsPerPersist, + } + } + const index = hydrateIndex++ + return { + kind, + id: `hydrate-${index}`, + seededRows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + } + }), + } +} + +function expectedHydratedCollections(scenario: SharedDriverFairnessScenario) { + return scenario.work.flatMap((work) => + work.kind === `hydrate` + ? [ + { + collectionId: `${scenario.id}-${work.id}`, + rows: work.seededRows.map((row) => ({ ...row })), + }, + ] + : [], + ) +} + +async function withNodeScenario( + scenario: SharedDriverFairnessScenario, + assertion: ( + observation: SharedDriverFairnessObservation, + ) => void | Promise, + options: SharedDriverFairnessOptions = {}, +): Promise { + const directory = mkdtempSync(join(tmpdir(), `db-driver-fairness-`)) + let primaryFailure: unknown + try { + const observation = await observeSharedDriverFairness( + () => + createWASQLiteTestDatabase({ + filename: join(directory, `state.sqlite`), + }), + scenario, + options, + ) + await assertion(observation) + } catch (error) { + primaryFailure = error + } + + let nodeCleanupFailure: unknown + try { + rmSync(directory, { recursive: true, force: true }) + } catch (error) { + nodeCleanupFailure = error + } + + if (primaryFailure !== undefined) { + if (nodeCleanupFailure !== undefined) { + const primaryMessage = + primaryFailure instanceof Error + ? primaryFailure.message + : String(primaryFailure) + const cleanupMessage = + nodeCleanupFailure instanceof Error + ? nodeCleanupFailure.message + : String(nodeCleanupFailure) + throw new Error( + `${primaryMessage}; node cleanup diagnostics: ${cleanupMessage}`, + ) + } + throw primaryFailure + } + if (nodeCleanupFailure !== undefined) throw nodeCleanupFailure +} + +function expectObservationReach( + observation: SharedDriverFairnessObservation, +): void { + const expectedHydrates = expectedHydratedCollections(observation.scenario) + expect(observation.admittedHydrateIds).toEqual( + expectedHydrates.map(({ collectionId }) => collectionId), + ) + expect(observation.hydrationCompletions).toHaveLength(expectedHydrates.length) + // Actual rows come from the public Collection after preload. The expected + // rows above are derived independently from the generated seed history. + expect(observation.hydratedCollections).toEqual(expectedHydrates) +} + +function expectFairObservation( + observation: SharedDriverFairnessObservation, +): void { + expectObservationReach(observation) + const violation = findSharedDriverFairnessViolation( + observation, + SHARED_DRIVER_FAIRNESS_BOUND, + ) + if (violation) { + throw new Error( + `shared-driver fairness mismatch at ${violation.checkpoint.collectionId}: ` + + `${violation.checkpoint.completedPersistIds.length} persists completed ` + + `(maximum ${violation.expectedMaximumCompletedPersists}), ` + + `${violation.checkpoint.pendingPersistCount} remained pending ` + + `(minimum ${violation.expectedMinimumPendingPersists}); ` + + `permitted completed ids: ${JSON.stringify(violation.permittedCompletedPersistIds)}; ` + + `cleanup diagnostics: ${JSON.stringify(observation.cleanupFailures)}`, + ) + } + expect(observation.cleanupFailures).toEqual([]) +} + +const generatedHistoryArbitrary = fc + .record({ + hydrateCount: fc.integer({ min: 2, max: 7 }), + persistCount: fc.integer({ min: 2, max: 7 }), + mutationsPerPersist: fc.integer({ min: 1, max: 3 }), + }) + .chain(({ hydrateCount, persistCount, mutationsPerPersist }) => { + const tail = [ + ...Array.from({ length: persistCount - 1 }, (_, index) => ({ + kind: `persist` as const, + token: `p${index + 1}`, + })), + ...Array.from({ length: hydrateCount }, (_, index) => ({ + kind: `hydrate` as const, + token: `h${index}`, + })), + ] + return fc + .shuffledSubarray(tail, { + minLength: tail.length, + maxLength: tail.length, + }) + .map((shuffledTail) => ({ + hydrateCount, + persistCount, + mutationsPerPersist, + orderedKinds: [ + `persist` as const, + ...shuffledTail.map(({ kind }) => kind), + ], + replayHistory: `p0,${shuffledTail.map(({ token }) => token).join(`,`)}`, + })) + }) + +describe(`shared BrowserWASQLiteDriver fairness oracle`, () => { + it(`reaches cold hydration and preserves independently seeded public rows without queued persists`, async () => { + await withNodeScenario( + createScenario(`neutral-reach`, [`hydrate`, `hydrate`, `hydrate`]), + (observation) => { + expectFairObservation(observation) + expect( + observation.rawDequeues.some((entry) => + entry.sql.startsWith( + `SELECT key, value, metadata, row_version FROM`, + ), + ), + ).toBe(true) + }, + ) + }) + + it(`completes a pending cold hydrate before an unrelated persist backlog drains`, async () => { + await withNodeScenario( + createScenario( + `fixed-persist-storm`, + [`persist`, `persist`, `persist`, `hydrate`], + 2, + ), + expectFairObservation, + ) + }) + + it(`bounds persist completions for generated ordered cold-hydrate/persist histories`, async () => { + await fc.assert( + fc.asyncProperty( + generatedHistoryArbitrary, + async ({ + hydrateCount, + persistCount, + mutationsPerPersist, + orderedKinds, + replayHistory, + }) => { + const observationId = + `generated-h${hydrateCount}-p${persistCount}-m${mutationsPerPersist}-` + + replayHistory.replaceAll(`,`, `-`) + await withNodeScenario( + createScenario(observationId, orderedKinds, mutationsPerPersist), + expectFairObservation, + ) + }, + ), + { + seed: replaySeed, + path: replayPath, + numRuns: 12, + endOnFailure: true, + verbose: 2, + }, + ) + }) + + it(`kills a persist-first FIFO scheduling mutant through the real package path`, async () => { + await withNodeScenario( + createScenario( + `persist-first-fixture-fault`, + [`persist`, `persist`, `persist`, `persist`, `hydrate`], + 1, + ), + (observation) => { + expectObservationReach(observation) + const violation = findSharedDriverFairnessViolation(observation) + if (!violation) { + throw new Error( + `persist-first scheduling mutant escaped the fairness checker; ` + + `cleanup diagnostics: ${JSON.stringify(observation.cleanupFailures)}`, + ) + } + expect(violation.checkpoint.completedPersistIds).toHaveLength(4) + expect(violation.checkpoint.pendingPersistCount).toBe(0) + expect(observation.cleanupFailures).toEqual([]) + }, + { schedulingFault: `persist-first-fifo` }, + ) + }) + + it(`calibrates the checker against a synthetic persist-first observation`, () => { + const scenario = createScenario( + `persist-first-checker-calibration`, + [`persist`, `persist`, `persist`, `persist`, `hydrate`], + 1, + ) + const fault = createPersistFirstFaultObservation(scenario) + + expect(findSharedDriverFairnessViolation(fault)).toMatchObject({ + checkpoint: fault.hydrationCompletions[0], + expectedMaximumCompletedPersists: 1, + expectedMinimumPendingPersists: 3, + }) + }) +}) diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts new file mode 100644 index 0000000000..670613377f --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts @@ -0,0 +1,708 @@ +import { createCollection } from '../../db/src/index' +import { persistedCollectionOptions } from '../src/index' +import { BrowserWASQLiteDriver } from '../src/wa-sqlite-driver' +import { + SingleProcessCoordinator, + createSQLiteCorePersistenceAdapter, +} from '../../db-sqlite-persistence-core/src/index' +import type { Collection } from '../../db/src/index' +import type { + PersistedCollectionPersistence, + PersistedTx, + SQLiteDriver, +} from '../../db-sqlite-persistence-core/src/index' +import type { BrowserWASQLiteDatabase } from '../src/index' + +export const SHARED_DRIVER_FAIRNESS_BOUND = 1 + +export type FairnessRow = { + id: string + value: number +} + +export type SharedDriverFairnessWork = + | { + kind: `hydrate` + id: string + seededRows: ReadonlyArray + } + | { + kind: `persist` + id: string + mutationsPerPersist: number + } + +export type SharedDriverFairnessScenario = { + id: string + /** + * Logical admission history. A storm begins with the one persist that is + * already non-preemptibly running; the remaining hydrate and persist work + * may be interleaved in any order. + */ + work: ReadonlyArray +} + +export type RawDriverDequeue = { + ordinal: number + sql: string + params: ReadonlyArray +} + +export type RawDriverAdmission = RawDriverDequeue & { + kind: `exec` | `query` | `run` | `transaction` +} + +export type HydrationCompletionCheckpoint = { + collectionId: string + completionOrdinal: number + completedPersistIds: ReadonlyArray + pendingPersistCount: number + rawDequeueCount: number +} + +export type HydratedCollectionRows = { + collectionId: string + rows: ReadonlyArray +} + +export type SharedDriverFairnessObservation = { + scenario: SharedDriverFairnessScenario + admittedHydrateIds: ReadonlyArray + logicalCompletionOrder: ReadonlyArray + hydrationCompletions: ReadonlyArray + driverAdmissions: ReadonlyArray + rawDequeues: ReadonlyArray + hydratedCollections: ReadonlyArray + cleanupFailures: ReadonlyArray +} + +export type SharedDriverFairnessViolation = { + checkpoint: HydrationCompletionCheckpoint + expectedCollectionId: string + expectedMaximumCompletedPersists: number + expectedMinimumPendingPersists: number + permittedCompletedPersistIds: ReadonlyArray + unexpectedCompletedPersistIds: ReadonlyArray +} + +export type SharedDriverFairnessOptions = { + /** Test-only hostile control that preserves the current global FIFO fault. */ + schedulingFault?: `persist-first-fifo` +} + +type Deferred = { + promise: Promise + resolve: () => void +} + +type CloseableSQLiteDriver = SQLiteDriver & { + transactionWithDriver: ( + fn: (transactionDriver: SQLiteDriver) => Promise, + ) => Promise + close: () => Promise +} + +type FairnessReferenceCheckpoint = { + collectionId: string + expectedMaximumCompletedPersists: number + expectedMinimumPendingPersists: number + permittedCompletedPersistIds: ReadonlyArray +} + +export type BrowserWASQLiteDatabaseFactory = () => + | BrowserWASQLiteDatabase + | Promise + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +function normalizeSql(sql: string): string { + return sql.replace(/\s+/g, ` `).trim() +} + +function failureMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function collectionIdFor( + scenario: SharedDriverFairnessScenario, + work: SharedDriverFairnessWork, +): string { + return `${scenario.id}-${work.id}` +} + +function validateScenario(scenario: SharedDriverFairnessScenario): void { + const hydrateWork = scenario.work.filter((work) => work.kind === `hydrate`) + const persistWork = scenario.work.filter((work) => work.kind === `persist`) + if (hydrateWork.length < 1) { + throw new Error(`work must contain at least one hydrate`) + } + if (persistWork.length > 0 && scenario.work[0]?.kind !== `persist`) { + throw new Error( + `a storm history must begin with its already-running persist`, + ) + } + if ( + new Set(scenario.work.map((work) => work.id)).size !== scenario.work.length + ) { + throw new Error(`work ids must be unique`) + } + for (const work of scenario.work) { + if (work.kind === `persist` && work.mutationsPerPersist < 1) { + throw new Error(`mutationsPerPersist must be at least one`) + } + if (work.kind === `hydrate`) { + if (work.seededRows.length < 1) { + throw new Error(`each hydrate must contain at least one seeded row`) + } + if ( + new Set(work.seededRows.map((row) => row.id)).size !== + work.seededRows.length + ) { + throw new Error(`seeded row ids must be unique within each hydrate`) + } + } + } +} + +class ObservedDatabase implements BrowserWASQLiteDatabase { + readonly rawDequeues: Array = [] + private heldBegin: Deferred | undefined + private beginEntered: Deferred | undefined + + constructor(private readonly database: BrowserWASQLiteDatabase) {} + + holdNextTransactionBegin(): { entered: Promise; release: () => void } { + if (this.heldBegin) { + throw new Error(`a transaction begin is already held`) + } + this.heldBegin = createDeferred() + this.beginEntered = createDeferred() + return { + entered: this.beginEntered.promise, + release: () => this.releaseHeldBegin(), + } + } + + clearTrace(): void { + this.rawDequeues.length = 0 + } + + async execute( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + const normalizedSql = normalizeSql(sql) + this.rawDequeues.push({ + ordinal: this.rawDequeues.length, + sql: normalizedSql, + params: [...params], + }) + + if (normalizedSql === `BEGIN IMMEDIATE` && this.heldBegin) { + const heldBegin = this.heldBegin + this.beginEntered?.resolve() + await heldBegin.promise + } + + return this.database.execute(sql, params) + } + + async close(): Promise { + this.releaseHeldBegin() + await this.database.close?.() + } + + private releaseHeldBegin(): void { + this.heldBegin?.resolve() + this.heldBegin = undefined + this.beginEntered = undefined + } +} + +/** + * A test-only hostile scheduler. It admits calls through the same public/core/ + * BrowserWASQLiteDriver path but serializes them in global admission order, so + * a future fair production driver cannot accidentally make this control pass. + */ +class PersistFirstFIFOFaultDriver implements SQLiteDriver { + private tail = Promise.resolve() + + constructor(private readonly driver: CloseableSQLiteDriver) {} + + exec(sql: string): Promise { + return this.enqueue(() => this.driver.exec(sql)) + } + + query( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + return this.enqueue(() => this.driver.query(sql, params)) + } + + run(sql: string, params: ReadonlyArray = []): Promise { + return this.enqueue(() => this.driver.run(sql, params)) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.enqueue(() => this.driver.transaction(fn)) + } + + transactionWithDriver( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.enqueue(() => this.driver.transactionWithDriver(fn)) + } + + close(): Promise { + return this.enqueue(() => this.driver.close()) + } + + private enqueue(operation: () => Promise): Promise { + const result = this.tail.then(operation) + this.tail = result.then( + () => undefined, + () => undefined, + ) + return result + } +} + +class AdmissionObservedDriver implements SQLiteDriver { + readonly admissions: Array = [] + + constructor( + private readonly driver: CloseableSQLiteDriver, + private readonly onAdmission?: (entry: RawDriverAdmission) => void, + ) {} + + exec(sql: string): Promise { + this.record(`exec`, sql, []) + return this.driver.exec(sql) + } + + query( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + this.record(`query`, sql, params) + return this.driver.query(sql, params) + } + + run(sql: string, params: ReadonlyArray = []): Promise { + this.record(`run`, sql, params) + return this.driver.run(sql, params) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.record(`transaction`, `BEGIN IMMEDIATE`, []) + return this.driver.transaction(fn) + } + + transactionWithDriver( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.record(`transaction`, `BEGIN IMMEDIATE`, []) + return this.driver.transactionWithDriver(fn) + } + + close(): Promise { + return this.driver.close() + } + + clearAdmissions(): void { + this.admissions.length = 0 + } + + private record( + kind: RawDriverAdmission[`kind`], + sql: string, + params: ReadonlyArray, + ): void { + const entry = { + ordinal: this.admissions.length, + kind, + sql: normalizeSql(sql), + params: [...params], + } + this.admissions.push(entry) + this.onAdmission?.(entry) + } +} + +function createSharedPersistence( + database: BrowserWASQLiteDatabase, + onAdmission?: (entry: RawDriverAdmission) => void, + schedulingFault?: SharedDriverFairnessOptions[`schedulingFault`], +): { + driver: AdmissionObservedDriver + persistence: PersistedCollectionPersistence +} { + const productionDriver = new BrowserWASQLiteDriver({ database }) + const scheduledDriver = + schedulingFault === `persist-first-fifo` + ? new PersistFirstFIFOFaultDriver(productionDriver) + : productionDriver + const driver = new AdmissionObservedDriver(scheduledDriver, onAdmission) + const adapter = createSQLiteCorePersistenceAdapter({ + driver, + schemaMismatchPolicy: `sync-absent-error`, + appliedTxPruneMaxRows: 0, + appliedTxPruneMaxAgeSeconds: 0, + }) + return { + driver, + persistence: { + adapter, + coordinator: new SingleProcessCoordinator(), + }, + } +} + +function createPersistedTx( + collectionId: string, + sequence: number, + mutationsPerPersist: number, +): PersistedTx { + return { + txId: `${collectionId}-tx-${sequence}`, + term: 1, + seq: sequence, + rowVersion: sequence, + mutations: Array.from({ length: mutationsPerPersist }, (_, index) => ({ + type: `insert` as const, + key: `${sequence}-${index}`, + value: { + id: `${sequence}-${index}`, + value: sequence * 100 + index, + }, + })), + } +} + +function createFairnessReference( + scenario: SharedDriverFairnessScenario, + fairnessBound: number, +): ReadonlyArray { + const persistIds = scenario.work + .filter((work) => work.kind === `persist`) + .map((work) => collectionIdFor(scenario, work)) + const hydrateIds = scenario.work + .filter((work) => work.kind === `hydrate`) + .map((work) => collectionIdFor(scenario, work)) + + return hydrateIds.map((collectionId, hydrateIndex) => { + // The first persist is the one already executing when hydrate work reaches + // the queue. The ordered reference then permits at most K additional + // persists between successive hydrate completions, preserving FIFO order + // within each logical lane. + const expectedMaximumCompletedPersists = Math.min( + persistIds.length, + persistIds.length === 0 ? 0 : 1 + hydrateIndex * fairnessBound, + ) + return { + collectionId, + expectedMaximumCompletedPersists, + expectedMinimumPendingPersists: + persistIds.length - expectedMaximumCompletedPersists, + permittedCompletedPersistIds: persistIds.slice( + 0, + expectedMaximumCompletedPersists, + ), + } + }) +} + +export function findSharedDriverFairnessViolation( + observation: SharedDriverFairnessObservation, + fairnessBound = SHARED_DRIVER_FAIRNESS_BOUND, +): SharedDriverFairnessViolation | undefined { + const reference = createFairnessReference(observation.scenario, fairnessBound) + for (const [ + hydrateIndex, + checkpoint, + ] of observation.hydrationCompletions.entries()) { + const expected = reference[hydrateIndex] + if (!expected) continue + const permitted = new Set(expected.permittedCompletedPersistIds) + const unexpectedCompletedPersistIds = checkpoint.completedPersistIds.filter( + (id) => !permitted.has(id), + ) + if ( + checkpoint.collectionId !== expected.collectionId || + checkpoint.completedPersistIds.length > + expected.expectedMaximumCompletedPersists || + checkpoint.pendingPersistCount < + expected.expectedMinimumPendingPersists || + unexpectedCompletedPersistIds.length > 0 + ) { + return { + checkpoint, + expectedCollectionId: expected.collectionId, + expectedMaximumCompletedPersists: + expected.expectedMaximumCompletedPersists, + expectedMinimumPendingPersists: expected.expectedMinimumPendingPersists, + permittedCompletedPersistIds: expected.permittedCompletedPersistIds, + unexpectedCompletedPersistIds, + } + } + } + return undefined +} + +/** Checker calibration only; the executable hostile control uses the fixture. */ +export function createPersistFirstFaultObservation( + scenario: SharedDriverFairnessScenario, +): SharedDriverFairnessObservation { + const persistIds = scenario.work + .filter((work) => work.kind === `persist`) + .map((work) => collectionIdFor(scenario, work)) + const firstHydrate = scenario.work.find((work) => work.kind === `hydrate`) + if (!firstHydrate) throw new Error(`fault observation requires a hydrate`) + const firstHydrateId = collectionIdFor(scenario, firstHydrate) + return { + scenario, + admittedHydrateIds: [firstHydrateId], + logicalCompletionOrder: [ + ...persistIds.map((id) => `persist:${id}`), + `hydrate:${firstHydrateId}`, + ], + hydrationCompletions: [ + { + collectionId: firstHydrateId, + completionOrdinal: persistIds.length, + completedPersistIds: persistIds, + pendingPersistCount: 0, + rawDequeueCount: persistIds.length + 1, + }, + ], + rawDequeues: [], + driverAdmissions: [], + hydratedCollections: [], + cleanupFailures: [], + } +} + +/** + * Runs the public persisted-collection startup path over one shared + * BrowserWASQLiteDriver. The expected scheduler is deliberately not imported: + * the oracle observes only logical admissions/completions and raw SQL dequeue. + */ +export async function observeSharedDriverFairness( + openDatabase: BrowserWASQLiteDatabaseFactory, + scenario: SharedDriverFairnessScenario, + options: SharedDriverFairnessOptions = {}, +): Promise { + validateScenario(scenario) + + const hydrateWork = scenario.work.filter((work) => work.kind === `hydrate`) + const persistWork = scenario.work.filter((work) => work.kind === `persist`) + const hydrateIds = hydrateWork.map((work) => collectionIdFor(scenario, work)) + const persistIds = persistWork.map((work) => collectionIdFor(scenario, work)) + const admittedHydrateIds: Array = [] + const allHydratesRequested = createDeferred() + + // A separate connection and adapter own seed/setup. Closing and reopening + // makes every measured hydration cold at the adapter and driver layers. + const seedDatabase = await openDatabase() + const seed = createSharedPersistence(seedDatabase) + let seedPrimaryFailure: unknown + try { + for (const work of hydrateWork) { + const collectionId = collectionIdFor(scenario, work) + await seed.persistence.adapter.applyCommittedTx(collectionId, { + txId: `${collectionId}-seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: work.seededRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: { ...row }, + })), + }) + } + for (const collectionId of persistIds) { + await seed.persistence.adapter.loadSubset(collectionId, {}) + } + } catch (error) { + seedPrimaryFailure = error + } + let seedCleanupFailure: unknown + try { + await seed.driver.close() + } catch (error) { + seedCleanupFailure = error + } + if (seedPrimaryFailure !== undefined) { + if (seedCleanupFailure !== undefined) { + throw new Error( + `${failureMessage(seedPrimaryFailure)}; seed cleanup diagnostics: ${failureMessage(seedCleanupFailure)}`, + ) + } + throw seedPrimaryFailure + } + if (seedCleanupFailure !== undefined) throw seedCleanupFailure + + const observedDatabase = new ObservedDatabase(await openDatabase()) + const { driver, persistence } = createSharedPersistence( + observedDatabase, + undefined, + options.schedulingFault, + ) + + const collections: Array> = [] + const persistPromises: Array> = [] + const preloadPromises: Array> = [] + const logicalCompletionOrder: Array = [] + const completedPersistIds: Array = [] + const hydrationCompletions: Array = [] + const observedRows = new Map>() + const cleanupFailures: Array = [] + let releaseHeldBegin: (() => void) | undefined + let beginEntered: Promise | undefined + let persistSequence = 0 + let primaryFailure: unknown + let observation: SharedDriverFairnessObservation | undefined + + try { + // Cache only the unrelated persist tables. Hydrate tables remain cold. + for (const collectionId of persistIds) { + await persistence.adapter.loadSubset(collectionId, {}) + } + + observedDatabase.clearTrace() + driver.clearAdmissions() + + if (persistIds.length > 0) { + const heldBegin = observedDatabase.holdNextTransactionBegin() + releaseHeldBegin = heldBegin.release + beginEntered = heldBegin.entered + } + + // Admit the explicit legal history while the first persist is held. The + // first item is the non-preemptible transaction; every tail permutation + // is therefore observable at the same deterministic release checkpoint. + for (const work of scenario.work) { + const collectionId = collectionIdFor(scenario, work) + if (work.kind === `persist`) { + persistSequence += 1 + const sequence = persistSequence + const persist = persistence.adapter + .applyCommittedTx( + collectionId, + createPersistedTx(collectionId, sequence, work.mutationsPerPersist), + ) + .then(() => { + completedPersistIds.push(collectionId) + logicalCompletionOrder.push(`persist:${collectionId}`) + }) + persistPromises.push(persist) + continue + } + + const collection = createCollection( + persistedCollectionOptions({ + id: collectionId, + getKey: (row) => row.id, + persistence, + }), + ) + collections.push(collection) + const preloadRequest = collection.preload() + admittedHydrateIds.push(collectionId) + if (admittedHydrateIds.length === hydrateIds.length) { + allHydratesRequested.resolve() + } + const preload = preloadRequest.then(() => { + logicalCompletionOrder.push(`hydrate:${collectionId}`) + observedRows.set( + collectionId, + collection.toArray.map((row) => ({ id: row.id, value: row.value })), + ) + hydrationCompletions.push({ + collectionId, + completionOrdinal: logicalCompletionOrder.length - 1, + completedPersistIds: [...completedPersistIds], + pendingPersistCount: persistIds.length - completedPersistIds.length, + rawDequeueCount: observedDatabase.rawDequeues.length, + }) + }) + preloadPromises.push(preload) + } + + await beginEntered + await allHydratesRequested.promise + // Every public preload request is now pending. Releasing the one + // non-preemptible persist here leaves production responsible for admitting + // and completing each full logical hydrate under the approved scheduler. + releaseHeldBegin?.() + releaseHeldBegin = undefined + + await Promise.all([...persistPromises, ...preloadPromises]) + + observation = { + scenario, + admittedHydrateIds, + logicalCompletionOrder, + hydrationCompletions, + driverAdmissions: driver.admissions.map((entry) => ({ + ...entry, + params: [...entry.params], + })), + rawDequeues: observedDatabase.rawDequeues.map((entry) => ({ + ...entry, + params: [...entry.params], + })), + hydratedCollections: hydrateWork.map((work) => { + const collectionId = collectionIdFor(scenario, work) + return { + collectionId, + rows: + observedRows.get(collectionId)?.map((row) => ({ ...row })) ?? [], + } + }), + cleanupFailures, + } + } catch (error) { + primaryFailure = error + } finally { + releaseHeldBegin?.() + await Promise.allSettled([...persistPromises, ...preloadPromises]) + for (const collection of collections) { + try { + await collection.cleanup() + } catch (error) { + cleanupFailures.push(failureMessage(error)) + } + } + try { + await driver.close() + } catch (error) { + cleanupFailures.push(failureMessage(error)) + } + } + + if (primaryFailure !== undefined) { + if (cleanupFailures.length > 0) { + throw new Error( + `${failureMessage(primaryFailure)}; active cleanup diagnostics: ${JSON.stringify(cleanupFailures)}`, + ) + } + throw primaryFailure + } + if (!observation) { + throw new Error(`shared-driver observation ended without a result`) + } + return observation +} diff --git a/packages/browser-db-sqlite-persistence/tsconfig.json b/packages/browser-db-sqlite-persistence/tsconfig.json index 5b14f299c7..b8bc1c02dc 100644 --- a/packages/browser-db-sqlite-persistence/tsconfig.json +++ b/packages/browser-db-sqlite-persistence/tsconfig.json @@ -19,6 +19,14 @@ ] } }, - "include": ["src", "tests", "e2e", "vite.config.ts", "vitest.e2e.config.ts"], + "include": [ + "src", + "tests", + "e2e", + "vite.config.ts", + "vite.opfs.config.ts", + "vitest.e2e.config.ts", + "playwright.opfs.config.ts" + ], "exclude": ["node_modules", "dist"] } diff --git a/packages/browser-db-sqlite-persistence/vite.opfs.config.ts b/packages/browser-db-sqlite-persistence/vite.opfs.config.ts new file mode 100644 index 0000000000..c66d394740 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/vite.opfs.config.ts @@ -0,0 +1,30 @@ +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vite' + +const packageDirectory = dirname(fileURLToPath(import.meta.url)) + +export default defineConfig({ + base: `./`, + // wa-sqlite locates its sibling WASM file through import.meta.url. Keeping + // the module out of Vite's dependency prebundle preserves that relationship + // for this real-browser fixture. + optimizeDeps: { + exclude: [`@journeyapps/wa-sqlite`], + }, + resolve: { + alias: { + '@tanstack/db': resolve(packageDirectory, `../db/src`), + '@tanstack/db-ivm': resolve(packageDirectory, `../db-ivm/src`), + '@tanstack/db-sqlite-persistence-core': resolve( + packageDirectory, + `../db-sqlite-persistence-core/src`, + ), + }, + }, + server: { + fs: { + allow: [resolve(packageDirectory, `../..`)], + }, + }, +}) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..ea68c1c1b2 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -219,6 +219,21 @@ export type PersistedRowScanOptions = { metadataOnly?: boolean } +export type PersistencePullSinceResult = + | { + latestRowVersion: number + requiresFullReload: true + } + | { + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array + deltas?: Array< + ReplayableTxDelta, string | number> + > + } + export type PersistedTx< T extends object = Record, TKey extends string | number = string | number, @@ -249,6 +264,21 @@ export type PersistedTx< collectionMetadataMutations?: Array } +/** + * Opaque identity shared by every adapter over the same physical SQLite driver. + * The core adapter uses it to serialize non-preemptible logical operations + * while alternating one regular operation between queued hydrations. + * + * Delegating drivers must forward this property before they are passed to a + * SQLite persistence adapter. A driver may also brand each returned Promise + * with the same key for late capability discovery, but a wrapper must return + * that exact Promise: discovering the key after a call cannot retroactively + * schedule the wrapper's first logical operation. + */ +export const SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY = Symbol.for( + `tanstack-db.sqlite-driver-supports-shared-logical-scheduling`, +) + export interface PersistenceAdapter { loadSubset: ( collectionId: string, @@ -280,9 +310,25 @@ export interface PersistenceAdapter { latestSeq: number latestRowVersion: number }> + /** + * Runs one complete logical hydrate as a non-preemptible scheduler unit. + * The callback must use the supplied unscheduled adapter for all nested + * persistence work and must not retain it after the callback settles. + */ + runInHydrationScope?: ( + task: (adapter: HydrationPersistenceAdapter) => Promise, + ) => Promise +} + +export type HydrationPersistenceAdapter = PersistenceAdapter & { + pullSince?: ( + collectionId: string, + fromRowVersion: number, + ) => Promise } export interface SQLiteDriver { + readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]?: object exec: (sql: string) => Promise query: ( sql: string, @@ -297,6 +343,24 @@ export interface SQLiteDriver { ) => Promise } +/** + * Forwards a driver's shared logical scheduling identity to a transparent + * delegating driver before the wrapper is used by a persistence adapter. + */ +export function forwardSQLiteDriverSharedLogicalScheduling< + TDriver extends SQLiteDriver, +>(source: SQLiteDriver, wrapper: TDriver): TDriver { + const key = source[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] + if (key) { + Object.defineProperty( + wrapper, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { value: key }, + ) + } + return wrapper +} + export interface PersistedCollectionCoordinator { getNodeId: () => string subscribe: ( @@ -310,18 +374,22 @@ export interface PersistedCollectionCoordinator { collectionId: string, options: LoadSubsetOptions, ) => Promise + /** The scoped adapter is leader-local and is never serialized to a follower. */ requestEnsurePersistedIndex: ( collectionId: string, signature: string, spec: PersistedIndexSpec, + scopedAdapter?: HydrationPersistenceAdapter, ) => Promise requestApplyLocalMutations?: ( collectionId: string, mutations: Array, ) => Promise + /** The scoped adapter is leader-local and is never serialized to a follower. */ pullSince?: ( collectionId: string, fromRowVersion: number, + scopedAdapter?: HydrationPersistenceAdapter, ) => Promise } @@ -802,7 +870,6 @@ class PersistedCollectionRuntime< truncate: null, metadata: null, } - private started = false private startupMetadataPromise: Promise | null = null private startPromise: Promise | null = null private resumeBaselinePromise: Promise | null = null @@ -899,13 +966,55 @@ class PersistedCollectionRuntime< } } + private runInHydrationScope( + task: (adapter: HydrationPersistenceAdapter) => Promise, + ): Promise { + if (this.persistence.adapter.runInHydrationScope) { + return this.persistence.adapter.runInHydrationScope(task) + } + return Promise.resolve().then(() => task(this.persistence.adapter)) + } + async ensureStarted(): Promise { if (this.startPromise) { return this.startPromise } const lifecycleGeneration = this.lifecycleGeneration - this.startPromise = this.startInternal(lifecycleGeneration) + let resolveStartupMetadata!: () => void + let rejectStartupMetadata!: (error: unknown) => void + this.startupMetadataPromise = new Promise((resolve, reject) => { + resolveStartupMetadata = resolve + rejectStartupMetadata = reject + }) + void this.startupMetadataPromise.catch(() => undefined) + + this.startPromise = (async () => { + const appliedCursor = await this.applyMutex.run(() => + this.runInHydrationScope(async (adapter) => { + if (lifecycleGeneration !== this.lifecycleGeneration) { + resolveStartupMetadata() + return undefined + } + + try { + await this.loadStartupMetadataInternal(lifecycleGeneration, adapter) + resolveStartupMetadata() + } catch (error) { + rejectStartupMetadata(error) + throw error + } + + return this.startInternal(lifecycleGeneration, adapter) + }), + ) + if ( + appliedCursor !== undefined && + lifecycleGeneration === this.lifecycleGeneration + ) { + await this.waitForAppliedReceiptsAfter(appliedCursor) + } + })() return this.startPromise } @@ -920,26 +1029,41 @@ class PersistedCollectionRuntime< if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.syncMode !== `on-demand`) return - await this.hydrateBaseline(lifecycleGeneration) + const appliedCursor = await this.applyMutex.run(() => + this.runInHydrationScope((adapter) => + this.hydrateBaseline(lifecycleGeneration, adapter), + ), + ) + if ( + appliedCursor !== undefined && + lifecycleGeneration === this.lifecycleGeneration + ) { + await this.waitForAppliedReceiptsAfter(appliedCursor) + } })() return this.resumeBaselinePromise } - private async hydrateBaseline(lifecycleGeneration: number): Promise { - if (lifecycleGeneration !== this.lifecycleGeneration) return + private async hydrateBaseline( + lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return undefined const baseline = {} this.activeSubsets.set(this.getSubsetKey(baseline), baseline) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(async () => { - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.hydrateSubsetUnsafe(baseline, { + await this.hydrateSubsetUnsafe( + baseline, + { requestRemoteEnsure: false, lifecycleGeneration, - }) - }) - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.waitForAppliedReceiptsAfter(appliedCursor) + }, + adapter, + ) + return lifecycleGeneration === this.lifecycleGeneration + ? appliedCursor + : undefined } async ensureStartupMetadataLoaded(): Promise { @@ -947,41 +1071,35 @@ class PersistedCollectionRuntime< return this.startupMetadataPromise } - const lifecycleGeneration = this.lifecycleGeneration - this.startupMetadataPromise = - this.loadStartupMetadataInternal(lifecycleGeneration) - return this.startupMetadataPromise + void this.ensureStarted() + return this.startupMetadataPromise! } - private async startInternal(lifecycleGeneration: number): Promise { - if (this.started) { - return - } - - this.started = true - - await this.ensureStartupMetadataLoaded() - if (lifecycleGeneration !== this.lifecycleGeneration) return + private async startInternal( + lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return undefined const indexBootstrapSnapshot = this.collection?.getIndexMetadata() ?? [] this.attachIndexLifecycleListeners() - await this.bootstrapPersistedIndexes(indexBootstrapSnapshot) - if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.bootstrapPersistedIndexes(indexBootstrapSnapshot, adapter) + if (lifecycleGeneration !== this.lifecycleGeneration) return undefined if (this.syncMode !== `on-demand`) { - await this.hydrateBaseline(lifecycleGeneration) + return this.hydrateBaseline(lifecycleGeneration, adapter) } + return undefined } private async loadStartupMetadataInternal( lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, ): Promise { // Restore stream position from the database so that new mutations // don't collide with previously applied transactions. - if (this.persistence.adapter.getStreamPosition) { - const position = await this.persistence.adapter.getStreamPosition( - this.collectionId, - ) + if (adapter.getStreamPosition) { + const position = await adapter.getStreamPosition(this.collectionId) if (lifecycleGeneration !== this.lifecycleGeneration) return this.observeStreamPosition( position.latestTerm, @@ -990,19 +1108,20 @@ class PersistedCollectionRuntime< ) } - const collectionMetadata = await this.loadCollectionMetadataSnapshot() + const collectionMetadata = + await this.loadCollectionMetadataSnapshot(adapter) if (lifecycleGeneration !== this.lifecycleGeneration) return this.replaceCollectionMetadataSnapshot(collectionMetadata) } - private async loadCollectionMetadataSnapshot(): Promise< - Array<{ key: string; value: unknown }> - > { - if (!this.persistence.adapter.loadCollectionMetadata) { + private async loadCollectionMetadataSnapshot( + adapter: HydrationPersistenceAdapter, + ): Promise> { + if (!adapter.loadCollectionMetadata) { return [] } - return this.persistence.adapter.loadCollectionMetadata(this.collectionId) + return adapter.loadCollectionMetadata(this.collectionId) } private replaceCollectionMetadataSnapshot( @@ -1046,13 +1165,19 @@ class PersistedCollectionRuntime< ): Promise { const lifecycleGeneration = this.lifecycleGeneration this.activeSubsets.set(this.getSubsetKey(options), options) - const appliedCursor = this.appliedReceiptSequence + await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, - lifecycleGeneration, - }), + this.runInHydrationScope((adapter) => + this.hydrateSubsetUnsafe( + options, + { + requestRemoteEnsure: this.mode === `sync-present`, + lifecycleGeneration, + }, + adapter, + ), + ), ) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) @@ -1092,10 +1217,16 @@ class PersistedCollectionRuntime< const lifecycleGeneration = this.lifecycleGeneration // A one-shot refresh does not acquire an enduring subscription lease. await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: false, - lifecycleGeneration, - }), + this.runInHydrationScope((adapter) => + this.hydrateSubsetUnsafe( + options, + { + requestRemoteEnsure: false, + lifecycleGeneration, + }, + adapter, + ), + ), ) } @@ -1243,7 +1374,6 @@ class PersistedCollectionRuntime< private advanceLifecycle(): void { this.lifecycleGeneration++ - this.started = false this.startupMetadataPromise = null this.startPromise = null this.resumeBaselinePromise = null @@ -1270,8 +1400,9 @@ class PersistedCollectionRuntime< private loadSubsetRowsUnsafe( options: LoadSubsetOptions, + adapter: HydrationPersistenceAdapter, ): Promise> { - return this.persistence.adapter.loadSubset(this.collectionId, options, { + return adapter.loadSubset(this.collectionId, options, { requiredIndexSignatures: this.getRequiredIndexSignatures(), }) as Promise> } @@ -1301,10 +1432,11 @@ class PersistedCollectionRuntime< requestRemoteEnsure: boolean lifecycleGeneration: number }, + adapter: HydrationPersistenceAdapter, ): Promise { this.hydratingGeneration = config.lifecycleGeneration try { - const rows = await this.loadSubsetRowsUnsafe(options) + const rows = await this.loadSubsetRowsUnsafe(options, adapter) if (config.lifecycleGeneration !== this.lifecycleGeneration) return this.applyRowsToCollection(rows) @@ -1314,8 +1446,8 @@ class PersistedCollectionRuntime< } } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() + await this.flushQueuedHydrationTransactionsUnsafe(adapter) + await this.flushQueuedTxCommittedUnsafe(adapter) if (config.requestRemoteEnsure) { this.queueRemoteSubsetEnsure(options) @@ -1397,14 +1529,16 @@ class PersistedCollectionRuntime< }) } - private async flushQueuedHydrationTransactionsUnsafe(): Promise { + private async flushQueuedHydrationTransactionsUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise { while (this.queuedHydrationTransactions.length > 0) { const transaction = this.queuedHydrationTransactions.shift() if (!transaction) { continue } try { - await this.applyBufferedSyncTransactionUnsafe(transaction) + await this.applyBufferedSyncTransactionUnsafe(transaction, adapter) } catch (error) { transaction.rejectApplied?.(error) for (const abandoned of this.queuedHydrationTransactions) { @@ -1418,6 +1552,7 @@ class PersistedCollectionRuntime< private async applyBufferedSyncTransactionUnsafe( transaction: BufferedSyncTransaction, + adapter: HydrationPersistenceAdapter, ): Promise { if (transaction.signal?.aborted) { transaction.rejectApplied?.(new SyncTransactionAbortedError()) @@ -1431,7 +1566,10 @@ class PersistedCollectionRuntime< } const applyToCollection = (): SyncAppliedReceipt => { - begin() + // Buffered source replay is part of persistence hydration. Apply it + // immediately so it cannot wait for a persisting mutation whose + // persistence is queued behind this hydrate's apply mutex. + begin({ immediate: true }) if (transaction.truncate) { truncate?.() @@ -1480,7 +1618,10 @@ class PersistedCollectionRuntime< } if (!transaction.internal) { - await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) + await this.persistAndBroadcastExternalSyncTransactionUnsafe( + transaction, + adapter, + ) } transaction.resolveApplied?.() } catch (error) { @@ -1491,6 +1632,7 @@ class PersistedCollectionRuntime< private async persistAndBroadcastExternalSyncTransactionUnsafe( transaction: BufferedSyncTransaction, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { if (transaction.internal) { return @@ -1520,7 +1662,7 @@ class PersistedCollectionRuntime< const tx = this.createPersistedTxFromOperations(transaction, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + await adapter.applyCommittedTx(this.collectionId, tx) this.publishTxCommittedEvent( this.createTxCommittedPayload({ term: tx.term, @@ -1961,7 +2103,9 @@ class PersistedCollectionRuntime< } void this.applyMutex - .run(() => this.processCommittedTxUnsafe(payload)) + .run(() => + this.processCommittedTxUnsafe(payload, this.persistence.adapter), + ) .catch((error) => { console.warn(`Failed to process tx:committed message:`, error) }) @@ -1975,25 +2119,28 @@ class PersistedCollectionRuntime< if (isCollectionResetPayload(payload)) { void this.applyMutex - .run(() => this.truncateAndReloadUnsafe()) + .run(() => this.truncateAndReloadUnsafe(this.persistence.adapter)) .catch((error) => { console.warn(`Failed to process collection reset message:`, error) }) } } - private async flushQueuedTxCommittedUnsafe(): Promise { + private async flushQueuedTxCommittedUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise { while (this.queuedTxCommitted.length > 0) { const queued = this.queuedTxCommitted.shift() if (!queued) { continue } - await this.processCommittedTxUnsafe(queued) + await this.processCommittedTxUnsafe(queued, adapter) } } private async processCommittedTxUnsafe( txCommitted: TxCommitted, + adapter: HydrationPersistenceAdapter, ): Promise { if (txCommitted.term < this.latestTerm) { return @@ -2014,7 +2161,7 @@ class PersistedCollectionRuntime< const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms if (hasGap) { - await this.recoverFromSeqGapUnsafe() + await this.recoverFromSeqGapUnsafe(adapter) if ( txCommitted.term < this.latestTerm || (txCommitted.term === this.latestTerm && @@ -2030,15 +2177,18 @@ class PersistedCollectionRuntime< txCommitted.latestRowVersion, ) - await this.invalidateFromCommittedTxUnsafe(txCommitted) + await this.invalidateFromCommittedTxUnsafe(txCommitted, adapter) } - private async recoverFromSeqGapUnsafe(): Promise { + private async recoverFromSeqGapUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise { if (this.persistence.coordinator.pullSince && this.latestRowVersion >= 0) { try { const pullResponse = await this.persistence.coordinator.pullSince( this.collectionId, this.latestRowVersion, + adapter, ) if (pullResponse.ok) { @@ -2048,23 +2198,26 @@ class PersistedCollectionRuntime< pullResponse.latestRowVersion, ) if (pullResponse.requiresFullReload || !pullResponse.deltas) { - await this.reloadActiveSubsetsUnsafe() + await this.reloadActiveSubsetsUnsafe(adapter) return } for (const delta of pullResponse.deltas) { - await this.invalidateFromCommittedTxUnsafe({ - type: `tx:committed`, - term: pullResponse.latestTerm, - seq: pullResponse.latestSeq, - txId: delta.txId, - latestRowVersion: delta.latestRowVersion, - requiresFullReload: false, - changedRows: delta.changedRows, - deletedKeys: delta.deletedKeys, - rowMetadataMutations: delta.rowMetadataMutations, - collectionMetadataMutations: delta.collectionMetadataMutations, - }) + await this.invalidateFromCommittedTxUnsafe( + { + type: `tx:committed`, + term: pullResponse.latestTerm, + seq: pullResponse.latestSeq, + txId: delta.txId, + latestRowVersion: delta.latestRowVersion, + requiresFullReload: false, + changedRows: delta.changedRows, + deletedKeys: delta.deletedKeys, + rowMetadataMutations: delta.rowMetadataMutations, + collectionMetadataMutations: delta.collectionMetadataMutations, + }, + adapter, + ) } return } @@ -2073,7 +2226,7 @@ class PersistedCollectionRuntime< } } - await this.truncateAndReloadUnsafe() + await this.truncateAndReloadUnsafe(adapter) if (this.mode === `sync-present`) { for (const options of this.activeSubsets.values()) { @@ -2082,7 +2235,9 @@ class PersistedCollectionRuntime< } } - private async truncateAndReloadUnsafe(): Promise { + private async truncateAndReloadUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise { if (this.syncControls.begin && this.syncControls.commit) { this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) @@ -2091,21 +2246,22 @@ class PersistedCollectionRuntime< }) } - await this.reloadActiveSubsetsUnsafe() + await this.reloadActiveSubsetsUnsafe(adapter) } private async invalidateFromCommittedTxUnsafe( txCommitted: TxCommitted, + adapter: HydrationPersistenceAdapter, ): Promise { if (txCommitted.requiresFullReload) { - await this.reloadActiveSubsetsUnsafe() + await this.reloadActiveSubsetsUnsafe(adapter) return } const changedKeyCount = txCommitted.changedRows.length + txCommitted.deletedKeys.length if (changedKeyCount > TARGETED_INVALIDATION_KEY_LIMIT) { - await this.reloadActiveSubsetsUnsafe() + await this.reloadActiveSubsetsUnsafe(adapter) return } @@ -2120,7 +2276,7 @@ class PersistedCollectionRuntime< // Has paginated subsets — fall back to full reload. // Targeted invalidation for paginated subsets is deferred to a future iteration. - await this.reloadActiveSubsetsUnsafe() + await this.reloadActiveSubsetsUnsafe(adapter) } private async applyTargetedInvalidationUnsafe( @@ -2179,7 +2335,9 @@ class PersistedCollectionRuntime< }) } - private async reloadActiveSubsetsUnsafe(): Promise { + private async reloadActiveSubsetsUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise { const lifecycleGeneration = this.lifecycleGeneration const activeSubsetOptions = this.activeSubsets.size > 0 @@ -2189,10 +2347,11 @@ class PersistedCollectionRuntime< this.hydratingGeneration = lifecycleGeneration try { const mergedRows = new Map() - const collectionMetadata = await this.loadCollectionMetadataSnapshot() + const collectionMetadata = + await this.loadCollectionMetadataSnapshot(adapter) if (lifecycleGeneration !== this.lifecycleGeneration) return for (const options of activeSubsetOptions) { - const subsetRows = await this.loadSubsetRowsUnsafe(options) + const subsetRows = await this.loadSubsetRowsUnsafe(options, adapter) if (lifecycleGeneration !== this.lifecycleGeneration) return for (const row of subsetRows) { mergedRows.set(row.key, { @@ -2216,8 +2375,8 @@ class PersistedCollectionRuntime< } } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() + await this.flushQueuedHydrationTransactionsUnsafe(adapter) + await this.flushQueuedTxCommittedUnsafe(adapter) } private attachIndexLifecycleListeners(): void { @@ -2242,6 +2401,7 @@ class PersistedCollectionRuntime< private async bootstrapPersistedIndexes( indexMetadataSnapshot?: Array, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { const collection = this.collection if (!collection && !indexMetadataSnapshot) { @@ -2251,7 +2411,7 @@ class PersistedCollectionRuntime< const indexMetadata = indexMetadataSnapshot ?? collection?.getIndexMetadata() ?? [] for (const metadata of indexMetadata) { - await this.ensurePersistedIndex(metadata) + await this.ensurePersistedIndex(metadata, adapter) } } @@ -2270,11 +2430,12 @@ class PersistedCollectionRuntime< private async ensurePersistedIndex( indexMetadata: CollectionIndexMetadata, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { const spec = this.buildPersistedIndexSpec(indexMetadata) try { - await this.persistence.adapter.ensureIndex( + await adapter.ensureIndex( this.collectionId, indexMetadata.signature, spec, @@ -2288,6 +2449,7 @@ class PersistedCollectionRuntime< this.collectionId, indexMetadata.signature, spec, + adapter, ) } catch (error) { console.warn( diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 69f29fc604..f76e1f310d 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -8,12 +8,14 @@ import { InvalidPersistedStorageKeyEncodingError, } from './errors' import { + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, createPersistedTableName, decodePersistedStorageKey, encodePersistedStorageKey, } from './persisted' import type { LoadSubsetOptions } from '@tanstack/db' import type { + HydrationPersistenceAdapter, PersistedIndexSpec, PersistedRowScanOptions, PersistedScannedRow, @@ -71,6 +73,144 @@ export type SQLitePullSinceResult = deltas: Array, TKey>> } +type ScheduledOperationKind = `regular` | `hydrate` + +type ScheduledOperation = { + kind: ScheduledOperationKind + task: () => Promise + resolve: (value: T) => void + reject: (error: unknown) => void +} + +class SharedPersistenceScheduler { + private readonly regularQueue: Array> = [] + private readonly hydrateQueue: Array> = [] + private running = false + private lastCompletedKind: ScheduledOperationKind | undefined + + runRegular(task: () => Promise): Promise { + return this.enqueue(`regular`, task) + } + + runHydrate(task: () => Promise): Promise { + return this.enqueue(`hydrate`, task) + } + + private enqueue( + kind: ScheduledOperationKind, + task: () => Promise, + ): Promise { + const result = new Promise((resolve, reject) => { + const operation: ScheduledOperation = { + kind, + task, + resolve, + reject, + } + const queue = kind === `hydrate` ? this.hydrateQueue : this.regularQueue + queue.push(operation as ScheduledOperation) + }) + this.drain() + return result + } + + private drain(): void { + if (this.running) return + + const operation = this.takeNext() + if (!operation) return + + this.running = true + void this.execute(operation) + } + + private async execute(operation: ScheduledOperation): Promise { + try { + operation.resolve(await operation.task()) + } catch (error) { + operation.reject(error) + } finally { + this.lastCompletedKind = operation.kind + this.running = false + this.drain() + } + } + + private takeNext(): ScheduledOperation | undefined { + // Hydrates get priority after the currently running non-preemptible unit. + // While both lanes remain queued, alternate one regular operation after + // each hydrate (K=1), preserving FIFO order within each lane. + if (this.hydrateQueue.length > 0) { + if ( + this.regularQueue.length > 0 && + this.lastCompletedKind === `hydrate` + ) { + return this.regularQueue.shift() + } + return this.hydrateQueue.shift() + } + return this.regularQueue.shift() + } +} + +const sharedPersistenceSchedulers = new WeakMap< + object, + SharedPersistenceScheduler +>() + +function getSharedPersistenceScheduler( + key: object, +): SharedPersistenceScheduler { + let scheduler = sharedPersistenceSchedulers.get(key) + if (!scheduler) { + scheduler = new SharedPersistenceScheduler() + sharedPersistenceSchedulers.set(key, scheduler) + } + return scheduler +} + +function getSharedLogicalSchedulingKey(value: unknown): object | undefined { + if ((typeof value !== `object` && typeof value !== `function`) || !value) { + return undefined + } + const key = ( + value as { + [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]?: unknown + } + )[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] + return key !== null && (typeof key === `object` || typeof key === `function`) + ? key + : undefined +} + +function observeSharedLogicalSchedulingSupport( + driver: SQLiteDriver, + onSupport: (key: object) => void, +): SQLiteDriver { + const observe = (promise: Promise): Promise => { + const key = getSharedLogicalSchedulingKey(promise) + if (key) onSupport(key) + return promise + } + + return { + exec: (sql) => observe(driver.exec(sql)), + query: (sql: string, params: ReadonlyArray = []) => + observe(driver.query(sql, params)), + run: (sql, params = []) => observe(driver.run(sql, params)), + transaction: (fn: (transactionDriver: SQLiteDriver) => Promise) => + observe(driver.transaction(fn)), + transactionWithDriver: ( + fn: (transactionDriver: SQLiteDriver) => Promise, + ) => + observe( + driver.transactionWithDriver + ? driver.transactionWithDriver(fn) + : driver.transaction(fn), + ), + } +} + const DEFAULT_SCHEMA_VERSION = 1 const DEFAULT_PULL_SINCE_RELOAD_THRESHOLD = 128 @@ -1023,6 +1163,8 @@ function buildIndexName(collectionId: string, signature: string): string { export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { private readonly driver: SQLiteDriver + private scheduler: SharedPersistenceScheduler | undefined + private readonly hydrationAdapter: HydrationPersistenceAdapter private readonly schemaVersion: number private readonly schemaMismatchPolicy: SQLiteCoreAdapterSchemaMismatchPolicy private readonly appliedTxPruneMaxRows: number | undefined @@ -1078,13 +1220,52 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ) } - this.driver = options.driver + const schedulingKey = getSharedLogicalSchedulingKey(options.driver) + this.scheduler = schedulingKey + ? getSharedPersistenceScheduler(schedulingKey) + : undefined + this.driver = schedulingKey + ? options.driver + : observeSharedLogicalSchedulingSupport(options.driver, (key) => { + this.scheduler ??= getSharedPersistenceScheduler(key) + }) this.schemaVersion = schemaVersion this.schemaMismatchPolicy = options.schemaMismatchPolicy ?? `sync-present-reset` this.appliedTxPruneMaxRows = options.appliedTxPruneMaxRows this.appliedTxPruneMaxAgeSeconds = options.appliedTxPruneMaxAgeSeconds this.pullSinceReloadThreshold = pullSinceReloadThreshold + this.hydrationAdapter = { + loadSubset: (collectionId, loadOptions, context) => + this.loadSubsetUnscheduled(collectionId, loadOptions, context), + applyCommittedTx: (collectionId, tx) => + this.applyCommittedTxUnscheduled(collectionId, tx), + loadCollectionMetadata: (collectionId) => + this.loadCollectionMetadataUnscheduled(collectionId), + scanRows: (collectionId, scanOptions) => + this.scanRowsUnscheduled(collectionId, scanOptions), + ensureIndex: (collectionId, signature, spec) => + this.ensureIndexUnscheduled(collectionId, signature, spec), + markIndexRemoved: (collectionId, signature) => + this.markIndexRemovedUnscheduled(collectionId, signature), + getStreamPosition: (collectionId) => + this.getStreamPositionUnscheduled(collectionId), + pullSince: (collectionId, fromRowVersion) => + this.pullSinceUnscheduled(collectionId, fromRowVersion), + runInHydrationScope: async (task) => task(this.hydrationAdapter), + } + } + + runInHydrationScope( + task: (adapter: HydrationPersistenceAdapter) => Promise, + ): Promise { + return this.scheduler + ? this.scheduler.runHydrate(() => task(this.hydrationAdapter)) + : Promise.resolve().then(() => task(this.hydrationAdapter)) + } + + private runRegular(task: () => Promise): Promise { + return this.scheduler ? this.scheduler.runRegular(task) : task() } private runInTransaction( @@ -1097,7 +1278,23 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { return this.driver.transaction(fn) } - async loadSubset( + loadSubset( + collectionId: string, + options: LoadSubsetOptions, + ctx?: { requiredIndexSignatures?: ReadonlyArray }, + ): Promise< + Array<{ + key: string | number + value: Record + metadata?: unknown + }> + > { + return this.runRegular(() => + this.loadSubsetUnscheduled(collectionId, options, ctx), + ) + } + + private async loadSubsetUnscheduled( collectionId: string, options: LoadSubsetOptions, ctx?: { requiredIndexSignatures?: ReadonlyArray }, @@ -1159,7 +1356,16 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { })) } - async applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + return this.runRegular(() => + this.applyCommittedTxUnscheduled(collectionId, tx), + ) + } + + private async applyCommittedTxUnscheduled( + collectionId: string, + tx: PersistedTx, + ): Promise { const tableMapping = await this.ensureCollectionReady(collectionId) const collectionTableSql = quoteIdentifier(tableMapping.tableName) const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) @@ -1379,7 +1585,15 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async loadCollectionMetadata( + loadCollectionMetadata( + collectionId: string, + ): Promise> { + return this.runRegular(() => + this.loadCollectionMetadataUnscheduled(collectionId), + ) + } + + private async loadCollectionMetadataUnscheduled( collectionId: string, ): Promise> { const rows = await this.driver.query<{ key: string; value: string }>( @@ -1395,7 +1609,16 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { })) } - async scanRows( + scanRows( + collectionId: string, + options?: PersistedRowScanOptions, + ): Promise> { + return this.runRegular(() => + this.scanRowsUnscheduled(collectionId, options), + ) + } + + private async scanRowsUnscheduled( collectionId: string, options?: PersistedRowScanOptions, ): Promise> { @@ -1418,7 +1641,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { })) } - async ensureIndex( + ensureIndex( + collectionId: string, + signature: string, + spec: PersistedIndexSpec, + ): Promise { + return this.runRegular(() => + this.ensureIndexUnscheduled(collectionId, signature, spec), + ) + } + + private async ensureIndexUnscheduled( collectionId: string, signature: string, spec: PersistedIndexSpec, @@ -1478,7 +1711,13 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async markIndexRemoved( + markIndexRemoved(collectionId: string, signature: string): Promise { + return this.runRegular(() => + this.markIndexRemovedUnscheduled(collectionId, signature), + ) + } + + private async markIndexRemovedUnscheduled( collectionId: string, signature: string, ): Promise { @@ -1508,7 +1747,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } } - async getStreamPosition(collectionId: string): Promise<{ + getStreamPosition(collectionId: string): Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> { + return this.runRegular(() => + this.getStreamPositionUnscheduled(collectionId), + ) + } + + private async getStreamPositionUnscheduled(collectionId: string): Promise<{ latestTerm: number latestSeq: number latestRowVersion: number @@ -1547,7 +1796,16 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } } - async pullSince( + pullSince( + collectionId: string, + fromRowVersion: number, + ): Promise> { + return this.runRegular(() => + this.pullSinceUnscheduled(collectionId, fromRowVersion), + ) + } + + private async pullSinceUnscheduled( collectionId: string, fromRowVersion: number, ): Promise> { diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0895e52375..096becc9d1 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -258,6 +258,17 @@ async function flushAsyncWork(delayMs: number = 0): Promise { await new Promise((resolve) => setTimeout(resolve, delayMs)) } +function createDeferred(): { + promise: Promise + resolve: (value: T) => void +} { + let resolve!: (value: T) => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + describe(`persistedCollectionOptions`, () => { it(`provides a sync-absent loopback configuration with persisted utils`, async () => { const adapter = createRecordingAdapter() @@ -987,6 +998,199 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`propagates an operation-owned receipt rejection that settles before the hydration waiter snapshots`, async () => { + const adapter = createRecordingAdapter() + const hydrateLoadEntered = createDeferred() + const allowHydrateLoad = createDeferred() + let gateHydrationLoad = true + adapter.loadSubset = async () => { + if (gateHydrationLoad) { + gateHydrationLoad = false + hydrateLoadEntered.resolve() + await allowHydrateLoad.promise + } + return [] + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-settled-receipt-boundary`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + let load: Promise | undefined + let receipt: Promise | undefined + const abortController = new AbortController() + + try { + collection.startSyncImmediate() + await collection.stateWhenReady() + load = Promise.resolve(collection._sync.loadSubset({ limit: 1 })) + await hydrateLoadEntered.promise + + let keyReads = 0 + const establishingRow = { + get id() { + keyReads++ + if (keyReads === 2) abortController.abort() + return `establishing` + }, + title: `Abort during buffered replay`, + } + remoteBegin?.() + remoteWrite?.({ type: `insert`, value: establishingRow }) + const applied = remoteCommit?.(abortController.signal) + if (!(applied instanceof Promise)) { + throw new Error(`expected a buffered establishing receipt`) + } + receipt = applied + void receipt.catch(() => undefined) + + allowHydrateLoad.resolve() + const [loadResult, receiptResult] = await Promise.allSettled([ + load, + receipt, + ]) + + expect(keyReads).toBeGreaterThanOrEqual(2) + expect(abortController.signal.aborted).toBe(true) + expect(receiptResult.status).toBe(`rejected`) + expect(loadResult.status).toBe(`rejected`) + if ( + loadResult.status === `rejected` && + receiptResult.status === `rejected` + ) { + expect(loadResult.reason).toBe(receiptResult.reason) + } + } finally { + abortController.abort() + allowHydrateLoad.resolve() + await receipt?.catch(() => undefined) + await load?.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`does not adopt an unrelated source receipt created after hydration work returns`, async () => { + const adapter = createRecordingAdapter() + const mutationEntered = createDeferred() + const releaseMutation = createDeferred() + const unrelatedStarted = createDeferred<{ + abortController: AbortController + receipt: Promise + }>() + const trace: Array = [] + let probeActive = false + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + + adapter.runInHydrationScope = async (task) => { + if (!probeActive) return task(adapter) + probeActive = false + + const result = await task(adapter) + trace.push(`hydrate-task-returned`) + + const abortController = new AbortController() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `unrelated`, title: `Outside hydrate boundary` }, + }) + const receipt = remoteCommit?.(abortController.signal) + if (!(receipt instanceof Promise)) { + throw new Error(`expected a pending unrelated receipt`) + } + trace.push(`unrelated-receipt-created`) + unrelatedStarted.resolve({ abortController, receipt }) + return result + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-unrelated-receipt-boundary`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + onInsert: async () => { + mutationEntered.resolve() + await releaseMutation.promise + }, + }), + ) + let mutation: ReturnType | undefined + let load: Promise | undefined + let unrelated: + | { abortController: AbortController; receipt: Promise } + | undefined + + try { + collection.startSyncImmediate() + await collection.stateWhenReady() + mutation = collection.insert({ id: `local`, title: `Persisting gate` }) + await mutationEntered.promise + + probeActive = true + load = Promise.resolve(collection._sync.loadSubset({ limit: 1 })) + unrelated = await unrelatedStarted.promise + expect(trace).toEqual([ + `hydrate-task-returned`, + `unrelated-receipt-created`, + ]) + + // Give the public load continuation the opportunity to snapshot receipts. + await flushAsyncWork() + unrelated.abortController.abort() + await unrelated.receipt.catch(() => undefined) + + await expect(load).resolves.toBeUndefined() + } finally { + unrelated?.abortController.abort() + releaseMutation.resolve() + await mutation?.isPersisted.promise.catch(() => undefined) + await load?.catch(() => undefined) + await collection.cleanup() + } + }) + it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { const adapter = createRecordingAdapter() const ownership = { queryCollection: { owners: [`gc:q1`] } } @@ -1268,6 +1472,122 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`replays a buffered source receipt without blocking its persisting predecessor`, async () => { + const adapter = createRecordingAdapter() + const hydrateLoadEntered = createDeferred() + const allowHydrateLoad = createDeferred() + let gateHydrationLoad = true + const replayState = { persisted: false } + + adapter.loadSubset = async () => { + if (gateHydrationLoad) { + gateHydrationLoad = false + hydrateLoadEntered.resolve() + await allowHydrateLoad.promise + } + return [] + } + const applyCommittedTx = adapter.applyCommittedTx + adapter.applyCommittedTx = async (...args) => { + replayState.persisted = true + await applyCommittedTx(...args) + } + adapter.runInHydrationScope = (task) => task(adapter) + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `update`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const sourceReady = createDeferred() + const bufferedCommitReturned = createDeferred<{ + receipt: Promise + }>() + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-buffered-causal-replay`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `update` + value: Todo + }) => void + remoteCommit = commit + sourceReady.resolve() + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + onInsert: async () => { + if (!remoteBegin || !remoteWrite || !remoteCommit) { + throw new Error(`source sync is not ready`) + } + remoteBegin() + remoteWrite({ + type: `update`, + value: { id: `source-row`, title: `Buffered during hydrate` }, + }) + const applied = remoteCommit() + if (applied === true) { + throw new Error(`source commit was not buffered during hydration`) + } + bufferedCommitReturned.resolve({ receipt: applied }) + await applied + }, + }), + ) + + const preload = Promise.resolve(collection.preload()) + void preload.catch(() => undefined) + let mutationPersisted: Promise | undefined + let bufferedReceipt: Promise | undefined + + try { + await hydrateLoadEntered.promise + await sourceReady.promise + + const mutation = collection.insert({ id: `local`, title: `Pending` }) + mutationPersisted = mutation.isPersisted.promise + void mutationPersisted.catch(() => undefined) + const bufferedCommit = await bufferedCommitReturned.promise + bufferedReceipt = bufferedCommit.receipt + void bufferedReceipt.catch(() => undefined) + + allowHydrateLoad.resolve() + + let causalCycleObserved = false + for ( + let attempt = 0; + attempt < 100 && !replayState.persisted; + attempt++ + ) { + causalCycleObserved = collection._state.pendingSyncedTransactions.some( + (transaction) => + transaction.committed && transaction.applied.isPending(), + ) + if (causalCycleObserved) break + await Promise.resolve() + } + + expect(causalCycleObserved).toBe(false) + expect(replayState.persisted).toBe(true) + await expect(bufferedReceipt).resolves.toBeUndefined() + await expect(mutationPersisted).resolves.toBeDefined() + await expect(preload).resolves.toBeUndefined() + expect(stripVirtualProps(collection.get(`source-row`))).toEqual({ + id: `source-row`, + title: `Buffered during hydrate`, + }) + } finally { + allowHydrateLoad.resolve() + await collection.cleanup() + } + }) + it(`discards a hydration-buffered transaction aborted before replay`, async () => { const adapter = createRecordingAdapter() let resolveLoadSubset: (() => void) | undefined @@ -1748,6 +2068,91 @@ describe(`persistedCollectionOptions`, () => { expect(collection.get(`2`)).toBeUndefined() }) + it(`does not let stale startup install index work on a rebound lifecycle`, async () => { + const adapter = createRecordingAdapter() + const g0MetadataEntered = createDeferred() + const allowG0Metadata = createDeferred() + const g1MetadataEntered = createDeferred() + const allowG1Metadata = createDeferred() + let metadataCalls = 0 + adapter.loadCollectionMetadata = async () => { + metadataCalls++ + if (metadataCalls === 1) { + g0MetadataEntered.resolve() + await allowG0Metadata.promise + } else if (metadataCalls === 2) { + g1MetadataEntered.resolve() + await allowG1Metadata.promise + } + return [] + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-startup-index-generation`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const stalePreload = Promise.resolve(collection.preload()) + void stalePreload.catch(() => undefined) + let freshReady: Promise | undefined + + try { + await g0MetadataEntered.promise + await collection.cleanup() + + const reboundIndex = collection.createIndex((row) => row.title, { + name: `rebound-bootstrap`, + }) + const reboundSignature = collection + .getIndexMetadata() + .find((metadata) => metadata.indexId === reboundIndex.id)?.signature + expect(reboundSignature).toBeDefined() + freshReady = collection.stateWhenReady() + + allowG0Metadata.resolve() + await g1MetadataEntered.promise + + const staleBootstrapCalls = adapter.ensureIndexCalls.filter( + (call) => call.signature === reboundSignature, + ) + const listenerIndex = collection.createIndex((row) => row.id, { + name: `rebound-listener`, + }) + const listenerSignature = collection + .getIndexMetadata() + .find((metadata) => metadata.indexId === listenerIndex.id)?.signature + expect(listenerSignature).toBeDefined() + const staleListenerCalls = adapter.ensureIndexCalls.filter( + (call) => call.signature === listenerSignature, + ) + + expect({ + staleBootstrapCalls: staleBootstrapCalls.length, + staleListenerCalls: staleListenerCalls.length, + }).toEqual({ + staleBootstrapCalls: 0, + staleListenerCalls: 0, + }) + + allowG1Metadata.resolve() + await freshReady + } finally { + allowG0Metadata.resolve() + allowG1Metadata.resolve() + await stalePreload.catch(() => undefined) + await freshReady?.catch(() => undefined) + await collection.cleanup() + } + }) + it(`does not let a stale invalidation reload overwrite a restarted lifecycle`, async () => { const adapter = createRecordingAdapter([{ id: `1`, title: `Initial` }]) const coordinator = createCoordinatorHarness() diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts new file mode 100644 index 0000000000..ac8fdc28a8 --- /dev/null +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest' +import { + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + createSQLiteCorePersistenceAdapter, + forwardSQLiteDriverSharedLogicalScheduling, +} from '../src' +import type { SQLiteDriver } from '../src' + +type Deferred = { + promise: Promise + resolve: () => void +} + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +class FirstQueryGatedDriver implements SQLiteDriver { + readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]: object + readonly admissions: Array = [] + readonly firstQueryEntered = createDeferred() + private readonly firstQueryGate = createDeferred() + private holdFirstQuery = true + + constructor(sharedLogicalSchedulingKey: object = {}) { + this[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] = + sharedLogicalSchedulingKey + } + + exec(): Promise { + this.admissions.push(`exec`) + return Promise.resolve() + } + + async query(): Promise> { + this.admissions.push(`query`) + if (this.holdFirstQuery) { + this.holdFirstQuery = false + this.firstQueryEntered.resolve() + await this.firstQueryGate.promise + } + return [] + } + + run(): Promise { + this.admissions.push(`run`) + return Promise.resolve() + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.admissions.push(`transaction`) + return fn(this) + } + + releaseFirstQuery(): void { + this.firstQueryGate.resolve() + } +} + +function createTransparentWrapper(driver: SQLiteDriver): SQLiteDriver { + return forwardSQLiteDriverSharedLogicalScheduling(driver, { + exec: (sql) => driver.exec(sql), + query: (sql: string, params: ReadonlyArray = []) => + driver.query(sql, params), + run: (sql, params = []) => driver.run(sql, params), + transaction: (fn: (transactionDriver: SQLiteDriver) => Promise) => + driver.transaction(fn), + transactionWithDriver: ( + fn: (transactionDriver: SQLiteDriver) => Promise, + ) => + driver.transactionWithDriver + ? driver.transactionWithDriver(fn) + : driver.transaction(fn), + }) +} + +describe(`shared logical scheduling`, () => { + it.each([ + { + name: `direct driver`, + key: {}, + wrap: (driver: SQLiteDriver) => driver, + }, + { + name: `transparent delegating driver`, + key: {}, + wrap: createTransparentWrapper, + }, + { + name: `function-valued key`, + key: () => undefined, + wrap: (driver: SQLiteDriver) => driver, + }, + ])( + `shares a scheduler across two fresh adapters through a $name`, + async ({ key, wrap }) => { + const underlying = new FirstQueryGatedDriver(key) + const driver = wrap(underlying) + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const regularAdapter = createSQLiteCorePersistenceAdapter({ driver }) + + const hydrate = hydrateAdapter.runInHydrationScope!(async (scoped) => { + await scoped.loadCollectionMetadata!(`hydrate`) + }) + await underlying.firstQueryEntered.promise + + const regular = regularAdapter.loadCollectionMetadata!(`regular`) + await Promise.resolve() + await Promise.resolve() + + expect(underlying.admissions).toEqual([`query`]) + + underlying.releaseFirstQuery() + await Promise.all([hydrate, regular]) + expect(underlying.admissions.length).toBeGreaterThan(1) + }, + ) +}) diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index a4c6bb7fe8..efd242dfdc 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -1,6 +1,7 @@ import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' import type { ApplyLocalMutationsResponse, + HydrationPersistenceAdapter, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, @@ -221,9 +222,15 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin collectionId: string, signature: string, spec: PersistedIndexSpec, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + // A scoped adapter is a leader-local capability and never crosses RPC. + await (scopedAdapter ?? this.requireAdapter()).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -270,13 +277,19 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin async pullSince( collectionId: string, fromRowVersion: number, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { if (this.isLeader(collectionId)) { - return this.handlePullSince(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) + // A scoped adapter is a leader-local capability and never crosses RPC. + return this.handlePullSince( + collectionId, + { + type: `rpc:pullSince:req`, + rpcId: safeRandomUUID(), + fromRowVersion, + }, + scopedAdapter, + ) } return this.sendRPC(collectionId, { @@ -733,10 +746,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin rpcId: string fromRowVersion: number }, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = scopedAdapter ?? this.requireAdapter() if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 328d4fe5ad..450ddd4bfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1072,6 +1072,9 @@ importers: '@journeyapps/wa-sqlite': specifier: ^1.4.1 version: 1.5.0 + '@playwright/test': + specifier: ^1.60.0 + version: 1.60.0 '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 From b4a12572962e49c4db72dd60f8206ae74acd8c15 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 00:16:11 +0100 Subject: [PATCH 02/14] fix(persistence): preserve unscheduled restart startup --- .../src/persisted.ts | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index ea68c1c1b2..ecd335a94d 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -990,24 +990,41 @@ class PersistedCollectionRuntime< void this.startupMetadataPromise.catch(() => undefined) this.startPromise = (async () => { - const appliedCursor = await this.applyMutex.run(() => - this.runInHydrationScope(async (adapter) => { - if (lifecycleGeneration !== this.lifecycleGeneration) { - resolveStartupMetadata() - return undefined - } + const loadStartupMetadata = async ( + adapter: HydrationPersistenceAdapter, + ) => { + if (lifecycleGeneration !== this.lifecycleGeneration) { + resolveStartupMetadata() + return false + } - try { - await this.loadStartupMetadataInternal(lifecycleGeneration, adapter) - resolveStartupMetadata() - } catch (error) { - rejectStartupMetadata(error) - throw error - } + try { + await this.loadStartupMetadataInternal(lifecycleGeneration, adapter) + resolveStartupMetadata() + return lifecycleGeneration === this.lifecycleGeneration + } catch (error) { + rejectStartupMetadata(error) + throw error + } + } - return this.startInternal(lifecycleGeneration, adapter) - }), - ) + let appliedCursor: number | undefined + if (this.persistence.adapter.runInHydrationScope) { + appliedCursor = await this.applyMutex.run(() => + this.runInHydrationScope(async (adapter) => { + if (!(await loadStartupMetadata(adapter))) return undefined + return this.startInternal(lifecycleGeneration, adapter) + }), + ) + } else { + // Preserve the existing unscheduled-adapter lifecycle contract: a + // replacement upstream may start while stale hydration is settling. + if (await loadStartupMetadata(this.persistence.adapter)) { + appliedCursor = await this.applyMutex.run(() => + this.startInternal(lifecycleGeneration, this.persistence.adapter), + ) + } + } if ( appliedCursor !== undefined && lifecycleGeneration === this.lifecycleGeneration From 7b999281601aa1f954be56beb2e74f1e91c14cc8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 00:27:13 +0100 Subject: [PATCH 03/14] fix(persistence): scope collection reset reloads --- .../src/persisted.ts | 6 +- .../tests/persisted.test.ts | 66 ++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index ecd335a94d..3488f4d5ba 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -2136,7 +2136,11 @@ class PersistedCollectionRuntime< if (isCollectionResetPayload(payload)) { void this.applyMutex - .run(() => this.truncateAndReloadUnsafe(this.persistence.adapter)) + .run(() => + this.runInHydrationScope((adapter) => + this.truncateAndReloadUnsafe(adapter), + ), + ) .catch((error) => { console.warn(`Failed to process collection reset message:`, error) }) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 096becc9d1..b1ac45f44f 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -19,6 +19,7 @@ import { persistedCollectionOptions, } from '../src' import type { + CollectionReset, PersistedCollectionCoordinator, PersistedCollectionPersistence, PersistedSyncWrappedOptions, @@ -184,7 +185,7 @@ function createNoopAdapter(): PersistenceAdapter { } type CoordinatorHarness = PersistedCollectionCoordinator & { - emit: (payload: TxCommitted, senderId?: string) => void + emit: (payload: TxCommitted | CollectionReset, senderId?: string) => void pullSinceCalls: number setPullSinceResponse: (response: PullSinceResponse) => void } @@ -458,6 +459,69 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`keeps a collection-reset reload inside one hydration scope`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) + adapter.collectionMetadata.set(`snapshot`, `initial`) + const coordinator = createCoordinatorHarness() + const loadSubset = adapter.loadSubset.bind(adapter) + let inHydrationScope = false + let interleaveArmed = false + let interleaveRan = false + + const runInterleavedWrite = () => { + interleaveRan = true + adapter.collectionMetadata.set(`snapshot`, `v2`) + adapter.rows.set(`1`, { id: `1`, title: `v2 row` }) + } + + adapter.loadSubset = async (...args) => { + if (interleaveArmed && !inHydrationScope && !interleaveRan) { + runInterleavedWrite() + } + return loadSubset(...args) + } + adapter.runInHydrationScope = async (task) => { + inHydrationScope = true + try { + return await task(adapter) + } finally { + inHydrationScope = false + if (interleaveArmed && !interleaveRan) runInterleavedWrite() + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + adapter.collectionMetadata.set(`snapshot`, `v1`) + adapter.rows.set(`1`, { id: `1`, title: `v1 row` }) + interleaveArmed = true + + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + + await vi.waitFor(() => expect(interleaveRan).toBe(true)) + expect(collection._state.syncedCollectionMetadata.get(`snapshot`)).toBe( + `v1`, + ) + expect(collection.get(`1`)?.title).toBe(`v1 row`) + await collection.cleanup() + }) + it(`persists metadata-only wrapped sync transactions`, async () => { const adapter = createRecordingAdapter() From 6cd13a8a15280980374c4c36b2cd27277e8b45ad Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 00:49:53 +0100 Subject: [PATCH 04/14] fix(persistence): scope sequence-gap recovery --- .../src/persisted.ts | 11 ++- .../tests/persisted.test.ts | 77 +++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 3488f4d5ba..453a2850f3 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -2155,13 +2155,14 @@ class PersistedCollectionRuntime< if (!queued) { continue } - await this.processCommittedTxUnsafe(queued, adapter) + await this.processCommittedTxUnsafe(queued, adapter, true) } } private async processCommittedTxUnsafe( txCommitted: TxCommitted, adapter: HydrationPersistenceAdapter, + gapRecoveryAlreadyScoped = false, ): Promise { if (txCommitted.term < this.latestTerm) { return @@ -2182,7 +2183,13 @@ class PersistedCollectionRuntime< const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms if (hasGap) { - await this.recoverFromSeqGapUnsafe(adapter) + if (gapRecoveryAlreadyScoped) { + await this.recoverFromSeqGapUnsafe(adapter) + } else { + await this.runInHydrationScope((scopedAdapter) => + this.recoverFromSeqGapUnsafe(scopedAdapter), + ) + } if ( txCommitted.term < this.latestTerm || (txCommitted.term === this.latestTerm && diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index b1ac45f44f..e6da325b91 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -2083,6 +2083,83 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`keeps sequence-gap recovery inside one hydration scope`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) + adapter.collectionMetadata.set(`snapshot`, `initial`) + const coordinator = createCoordinatorHarness() + coordinator.setPullSinceResponse({ + type: `rpc:pullSince:res`, + rpcId: `pull-gap-scope`, + ok: true, + latestTerm: 1, + latestSeq: 1, + latestRowVersion: 1, + requiresFullReload: true, + }) + const loadSubset = adapter.loadSubset.bind(adapter) + let inHydrationScope = false + let interleaveArmed = false + let interleaveRan = false + + const runInterleavedWrite = () => { + interleaveRan = true + adapter.collectionMetadata.set(`snapshot`, `v2`) + adapter.rows.set(`1`, { id: `1`, title: `v2 row` }) + } + + adapter.loadSubset = async (...args) => { + if (interleaveArmed && !inHydrationScope && !interleaveRan) { + runInterleavedWrite() + } + return loadSubset(...args) + } + adapter.runInHydrationScope = async (task) => { + inHydrationScope = true + try { + return await task(adapter) + } finally { + inHydrationScope = false + if (interleaveArmed && !interleaveRan) runInterleavedWrite() + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + adapter.collectionMetadata.set(`snapshot`, `v1`) + adapter.rows.set(`1`, { id: `1`, title: `v1 row` }) + interleaveArmed = true + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `tx-gap-scope`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + + await vi.waitFor(() => expect(interleaveRan).toBe(true)) + expect(collection._state.syncedCollectionMetadata.get(`snapshot`)).toBe( + `v1`, + ) + expect(collection.get(`1`)?.title).toBe(`v1 row`) + await collection.cleanup() + }) + it(`removes deleted rows after tx:committed invalidation reload`, async () => { const adapter = createRecordingAdapter([ { id: `1`, title: `Keep` }, From e2aa680dea0ab7aa811208d8bbe567aa0b85e4bc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 01:10:10 +0100 Subject: [PATCH 05/14] fix(persistence): scope committed reloads --- .../src/persisted.ts | 25 +++++-- .../tests/persisted.test.ts | 66 +++++++++++++++++++ 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 453a2850f3..8cbcf7d811 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -2162,7 +2162,7 @@ class PersistedCollectionRuntime< private async processCommittedTxUnsafe( txCommitted: TxCommitted, adapter: HydrationPersistenceAdapter, - gapRecoveryAlreadyScoped = false, + hydrationScopeAlreadyActive = false, ): Promise { if (txCommitted.term < this.latestTerm) { return @@ -2183,7 +2183,7 @@ class PersistedCollectionRuntime< const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms if (hasGap) { - if (gapRecoveryAlreadyScoped) { + if (hydrationScopeAlreadyActive) { await this.recoverFromSeqGapUnsafe(adapter) } else { await this.runInHydrationScope((scopedAdapter) => @@ -2205,7 +2205,11 @@ class PersistedCollectionRuntime< txCommitted.latestRowVersion, ) - await this.invalidateFromCommittedTxUnsafe(txCommitted, adapter) + await this.invalidateFromCommittedTxUnsafe( + txCommitted, + adapter, + hydrationScopeAlreadyActive, + ) } private async recoverFromSeqGapUnsafe( @@ -2245,6 +2249,7 @@ class PersistedCollectionRuntime< collectionMetadataMutations: delta.collectionMetadataMutations, }, adapter, + true, ) } return @@ -2280,16 +2285,24 @@ class PersistedCollectionRuntime< private async invalidateFromCommittedTxUnsafe( txCommitted: TxCommitted, adapter: HydrationPersistenceAdapter, + hydrationScopeAlreadyActive = false, ): Promise { + const reloadActiveSubsets = () => + hydrationScopeAlreadyActive + ? this.reloadActiveSubsetsUnsafe(adapter) + : this.runInHydrationScope((scopedAdapter) => + this.reloadActiveSubsetsUnsafe(scopedAdapter), + ) + if (txCommitted.requiresFullReload) { - await this.reloadActiveSubsetsUnsafe(adapter) + await reloadActiveSubsets() return } const changedKeyCount = txCommitted.changedRows.length + txCommitted.deletedKeys.length if (changedKeyCount > TARGETED_INVALIDATION_KEY_LIMIT) { - await this.reloadActiveSubsetsUnsafe(adapter) + await reloadActiveSubsets() return } @@ -2304,7 +2317,7 @@ class PersistedCollectionRuntime< // Has paginated subsets — fall back to full reload. // Targeted invalidation for paginated subsets is deferred to a future iteration. - await this.reloadActiveSubsetsUnsafe(adapter) + await reloadActiveSubsets() } private async applyTargetedInvalidationUnsafe( diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index e6da325b91..8046507ac0 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -2160,6 +2160,72 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`keeps contiguous committed reload inside one hydration scope`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) + adapter.collectionMetadata.set(`snapshot`, `initial`) + const coordinator = createCoordinatorHarness() + const loadSubset = adapter.loadSubset.bind(adapter) + let inHydrationScope = false + let interleaveArmed = false + let interleaveRan = false + + const runInterleavedWrite = () => { + interleaveRan = true + adapter.collectionMetadata.set(`snapshot`, `v2`) + adapter.rows.set(`1`, { id: `1`, title: `v2 row` }) + } + + adapter.loadSubset = async (...args) => { + if (interleaveArmed && !inHydrationScope && !interleaveRan) { + runInterleavedWrite() + } + return loadSubset(...args) + } + adapter.runInHydrationScope = async (task) => { + inHydrationScope = true + try { + return await task(adapter) + } finally { + inHydrationScope = false + if (interleaveArmed && !interleaveRan) runInterleavedWrite() + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + adapter.collectionMetadata.set(`snapshot`, `v1`) + adapter.rows.set(`1`, { id: `1`, title: `v1 row` }) + interleaveArmed = true + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `tx-contiguous-reload-scope`, + latestRowVersion: 1, + requiresFullReload: true, + }) + + await vi.waitFor(() => expect(interleaveRan).toBe(true)) + expect(collection._state.syncedCollectionMetadata.get(`snapshot`)).toBe( + `v1`, + ) + expect(collection.get(`1`)?.title).toBe(`v1 row`) + await collection.cleanup() + }) + it(`removes deleted rows after tx:committed invalidation reload`, async () => { const adapter = createRecordingAdapter([ { id: `1`, title: `Keep` }, From 2ef8235212df2565d868648ce7ce7d1c3688ccc0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 09:52:06 +0100 Subject: [PATCH 06/14] docs(test): explain persistence oracle contracts --- .../e2e/shared-driver-fairness.opfs.spec.ts | 8 ++++ .../e2e/shared-driver-fairness.opfs.ts | 9 +++++ .../shared-driver-fairness-oracle.test.ts | 9 +++++ .../tests/shared-driver-fairness-oracle.ts | 37 +++++++++++++++++++ .../tests/persisted.test.ts | 21 +++++++++++ .../tests/shared-logical-scheduling.test.ts | 29 +++++++++++++++ 6 files changed, 113 insertions(+) diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts index 4b99a1db15..00e0df4855 100644 --- a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts @@ -1,3 +1,11 @@ +/** + * Browser checkpoint assertions for the OPFS refinement. Expected public rows + * are rebuilt from the scenario IDs rather than from production output. The + * neutral case proves cold-query reach; the storm case requires no K=1 + * violation. Failure-before-checkpoint, semantic mismatch, driver cleanup, and + * OPFS cleanup remain distinct outcomes so setup or teardown cannot satisfy the + * scheduling law. + */ import { expect, test } from '@playwright/test' import type { Page } from '@playwright/test' import type { OPFSOracleResult } from './shared-driver-fairness.opfs' diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts index 533155912c..a443f680f2 100644 --- a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts @@ -1,3 +1,12 @@ +/** + * Real-provider refinement of the shared-driver fairness oracle. The page runs + * the same legal neutral and fixed-storm histories through public `preload()`, + * the browser/core adapter boundary, and Chromium's OPFSCoopSyncVFS worker. It + * freezes logical completions, public rows, raw dequeue reach, and the K=1 + * violation result before cleanup, then reports provider cleanup separately. + * This fixture adds real OPFS/worker evidence; it does not claim multi-tab, + * multi-process, non-Chromium, latency, or unbounded-eventuality coverage. + */ import { openBrowserWASQLiteOPFSDatabase } from '../src/index' import { findSharedDriverFairnessViolation, diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts index c0a857ba2c..7fc27c5d67 100644 --- a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts @@ -1,3 +1,12 @@ +/** + * Node campaign for the shared-driver K=1 contract documented in + * `shared-driver-fairness-oracle.ts`. Fixed histories prove neutral reach and a + * persist storm; generated legal histories vary both lane sizes, mutation + * width, and tail order. Public rows and logical completion checkpoints are + * checked independently of production scheduling, and an executable + * persist-first FIFO mutant proves the checker rejects the original fault. + * Failures replay through TANSTACK_DB_DRIVER_FAIRNESS_SEED/PATH. + */ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts index 670613377f..639ab85275 100644 --- a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts @@ -1,3 +1,40 @@ +/** + * # When does a cold hydrate get a turn on a shared SQLite driver? + * + * Contract and source: RFC #1659 accepts K=1 complete-logical-cold-hydrate + * scheduling. The persist already executing when the storm begins is + * non-preemptible. After that, at most one additional persist may complete + * between consecutive hydrate completions, with FIFO identity preserved + * inside the hydrate and persist lanes. + * + * History grammar and domain: a legal storm has unique work IDs, starts with + * that already-running persist when any persist exists, and then permutes + * complete hydrate and persist requests. Hydrates contain nonempty unique + * seeded rows; persists contain one or more mutations. The generated campaign + * varies 2..7 hydrates, 2..7 persists, 1..3 mutations per persist, and sampled + * tail permutations. Neutral histories contain only hydrates. + * + * Independent model and production boundary: `createFairnessReference` + * computes permitted completed persist IDs from the ordered history and K; it + * does not import or simulate the production scheduler. The driver exercises + * public `Collection.preload()` through persisted collection options, the core + * adapter, and one real `BrowserWASQLiteDriver`. Each preload completion is a + * checkpoint after the complete logical hydrate, not after an individual SQL + * statement. + * + * Observed public facts: admitted and completed logical IDs, completed and + * pending persists at every hydrate checkpoint, independently seeded public + * collection rows, raw SQL dequeue reach, and cleanup diagnostics. The oracle + * does not establish elapsed-time latency, unbounded eventuality, multi-process + * coordination, or a browser matrix; the Chromium OPFS fixture separately + * refines the provider boundary. + * + * Challenge and replay: the executable persist-first FIFO driver must violate + * the same K=1 checker while using the public/core/driver path. Re-run a + * generated failure with TANSTACK_DB_DRIVER_FAIRNESS_SEED and + * TANSTACK_DB_DRIVER_FAIRNESS_PATH. Cleanup preserves the primary failure and + * reports secondary resource-release diagnostics separately. + */ import { createCollection } from '../../db/src/index' import { persistedCollectionOptions } from '../src/index' import { BrowserWASQLiteDriver } from '../src/wa-sqlite-driver' diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 8046507ac0..365a2c7b35 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -459,6 +459,10 @@ describe(`persistedCollectionOptions`, () => { }) }) + // Focused collection-reset refinement: metadata and rows must come from one + // hydration scope. The adapter makes an interleaved v2 write possible only + // outside that scope, so the public v1 metadata and row are the independent + // coherence checkpoint. This fixed history does not model arbitrary resets. it(`keeps a collection-reset reload inside one hydration scope`, async () => { const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) adapter.collectionMetadata.set(`snapshot`, `initial`) @@ -1062,6 +1066,11 @@ describe(`persistedCollectionOptions`, () => { } }) + // Focused receipt-ownership refinements. A source receipt created by the + // hydration operation belongs to its waiter even if it rejects before the + // waiter snapshots; a receipt created after hydration work returns does not. + // The public load result and exact rejection identity distinguish those two + // boundaries without treating every pending source receipt as related. it(`propagates an operation-owned receipt rejection that settles before the hydration waiter snapshots`, async () => { const adapter = createRecordingAdapter() const hydrateLoadEntered = createDeferred() @@ -1536,6 +1545,10 @@ describe(`persistedCollectionOptions`, () => { }) }) + // Focused R7 causal-replay witness: after hydration releases its buffer, the + // source receipt must replay without awaiting the persisting operation whose + // callback is itself awaiting that receipt. Persistence reach, both public + // settlements, and the final source row expose the otherwise hidden cycle. it(`replays a buffered source receipt without blocking its persisting predecessor`, async () => { const adapter = createRecordingAdapter() const hydrateLoadEntered = createDeferred() @@ -2083,6 +2096,10 @@ describe(`persistedCollectionOptions`, () => { }) }) + // Focused invalidation-reload refinements. Whether recovery follows a + // sequence gap or a contiguous committed notification, metadata and rows + // must be read inside one hydration scope. The adapter schedules v2 only + // outside the scope; coherent public v1 state is the checkpoint. it(`keeps sequence-gap recovery inside one hydration scope`, async () => { const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) adapter.collectionMetadata.set(`snapshot`, `initial`) @@ -2275,6 +2292,10 @@ describe(`persistedCollectionOptions`, () => { expect(collection.get(`2`)).toBeUndefined() }) + // Focused lifecycle-fencing witness: generation-zero startup is held across + // cleanup and rebound, then released while generation one is still loading. + // Zero ensure-index calls for the rebound signatures prove stale bootstrap + // and listener work did not cross the public lifecycle boundary. it(`does not let stale startup install index work on a rebound lifecycle`, async () => { const adapter = createRecordingAdapter() const g0MetadataEntered = createDeferred() diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts index ac8fdc28a8..2daa08e563 100644 --- a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -1,3 +1,32 @@ +/** + * # Which adapters share one logical scheduling boundary? + * + * Contract and source: the RFC #1659 driver protocol keys scheduling by the + * exact shared driver identity. Two fresh core adapters over that identity + * must not interleave a complete hydration scope with regular adapter work. + * Transparent wrappers must forward the identity unchanged, including when + * the identity itself is a function object. + * + * Independent relation and legal domain: a two-adapter history starts one + * hydration metadata query, holds it at the driver boundary, then requests one + * regular metadata query. Before release, exactly the first query may be + * admitted; after release, both operations must finish. The three legal driver + * forms are direct, transparently wrapped, and direct with a function-valued + * key. This relation records admissions without copying the production + * scheduler or invoking the opaque key. + * + * Production boundary and checkpoint: both operations use + * `createSQLiteCorePersistenceAdapter`; the hydration operation enters through + * `runInHydrationScope`. The first checkpoint is after the regular request has + * had microtasks to reach the gated driver, while the first query remains + * held. Losing wrapper identity or treating a function key as a getter admits + * the second query and fails the exact admission assertion. + * + * This focused contract test does not establish K=1 lane fairness, SQL result + * correctness, eventual progress under arbitrary I/O, or cross-process + * coordination. Those belong to the shared-driver oracle and provider + * refinements. + */ import { describe, expect, it } from 'vitest' import { SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, From 7e1c09047145749a9b8111020cbf72e5b93c1029 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 14:00:03 +0100 Subject: [PATCH 07/14] test(browser-sqlite): surface OPFS oracle startup errors --- .../e2e/shared-driver-fairness.opfs.spec.ts | 31 +- .../tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md | 355 ------------------ 2 files changed, 26 insertions(+), 360 deletions(-) delete mode 100644 packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts index 00e0df4855..849de99ccc 100644 --- a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts @@ -14,11 +14,32 @@ async function readOracleResult( page: Page, mode: `neutral` | `storm`, ): Promise { - await page.goto(`/e2e/shared-driver-fairness.opfs.html?mode=${mode}`) - await page.waitForFunction( - () => window.__tanstackDriverFairnessOracle !== undefined, - ) - return page.evaluate(() => window.__tanstackDriverFairnessOracle!) + let rejectPageError!: (error: Error) => void + const pageError = new Promise((_resolve, reject) => { + rejectPageError = reject + }) + void pageError.catch(() => undefined) + const onPageError = (error: Error) => { + rejectPageError( + new Error( + `OPFS fairness page failed before publishing a result: ${error.message}`, + ), + ) + } + page.on(`pageerror`, onPageError) + + try { + await page.goto(`/e2e/shared-driver-fairness.opfs.html?mode=${mode}`) + await Promise.race([ + page.waitForFunction( + () => window.__tanstackDriverFairnessOracle !== undefined, + ), + pageError, + ]) + return page.evaluate(() => window.__tanstackDriverFairnessOracle!) + } finally { + page.off(`pageerror`, onPageError) + } } function expectedHydratedCollections(scenarioId: string, count: number) { diff --git a/packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md b/packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md deleted file mode 100644 index a04f607d68..0000000000 --- a/packages/browser-db-sqlite-persistence/tests/SHARED_DRIVER_FAIRNESS_RED_AUDIT.md +++ /dev/null @@ -1,355 +0,0 @@ -# RFC #1659 Workstream 5B: RED oracle ledger - -## Frozen baseline and scope - -- Repository/worktree: `powersync-v2-main/.worktrees/rfc-1659-ws5b-driver-fairness-oracle` -- Branch: `rfc-1659-ws5b-driver-fairness-oracle` -- `HEAD`, `origin/main`, merge base, and `github/main`: `76d766e84afbfcde2900a661233dd59e1decd5c2` -- Baseline production driver: `packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts`; its single promise FIFO is unchanged. -- Required guidance read in full before task work: root `AGENTS.md`, `docs/contributing/oracle-tests.md`, and `docs/contributing/oracle-coverage.md`. -- Issue #1752: open, updated `2026-08-19T20:42:53Z`; reports a shared WA-SQLite/OPFS FIFO persist storm delaying hydration. -- RFC #1659: open; Workstream 5 authorizes an explicit bounded-fairness oracle and requires the current persist-first storm as a fault control. -- PR #1837 is evidence-only: open/clean, base `7f6b6438cd3a5b2cfc54ea1d8ad8a2102ea9d699`, head `83fc42f3ef98c11d49c097ec07ca8040259ac147`, updated `2026-09-17T13:36:35Z`. Its files are offline-runtime work, not the browser driver's FIFO. No code or history was borrowed from it or any other candidate. -- Phase 1 plus the explicitly approved circular-gate repair only. No production source, old RFC branch, merge, rebase, or cherry-pick was used. Nothing was committed or pushed. - -## Oracle card - -**Law and source.** RFC #1659 says queued persist work must not starve cold hydration on the shared browser driver. The approved executable law permits the non-preemptible persist already running when hydration is requested, then bounds completed persists at every public hydration-completion checkpoint. The maintainer-approved bound is `K = 1` additional persist between successive complete logical cold-hydrate units. - -**Authority limit.** The maintainer chose `K = 1` and one complete logical cold hydrate, including setup operations, as the priority unit. This does not authorize priority for all reads, a bidirectional policy, transaction preemption, or changes to generic `SQLiteDriver` FIFO/transaction semantics. A query-only priority queue remains insufficient because genuinely cold startup performs registry/schema/metadata/index setup as well as row loading and local application. - -**Legal ordered histories.** The generated input is an explicit sequence of logical `{ kind: hydrate | persist, id, payload }` work, not a tuple of counts. A storm's first item is the persist already non-preemptibly running; the remaining two to seven cold hydrates and one to six additional persists are randomly interleaved. Persist transactions contain one to three legal insert mutations. Every hydrate carries one independently generated collection identity and two independently specified seed rows. A seed connection closes before a fresh adapter/driver admits the generated history. Only unrelated persist tables are prewarmed. - -**Independent ordered reference.** The reference consumes only the explicit history. It preserves FIFO identity order within the persist and hydrate lanes. At hydrate checkpoint `i`, its permitted completed-persist prefix is the already-running persist plus at most `i*K` subsequent persist identities from the history. It also derives the exact expected hydrate identity and minimum pending-persist count. It does not inspect or call the production FIFO. Public row expectations are derived independently from the history's seed payload and compared exactly, including collection identity, missing/extra rows, row order, IDs, and values. - -**Production path and reach.** The fixture uses the real `BrowserWASQLiteDriver`, `createSQLiteCorePersistenceAdapter`, `persistedCollectionOptions`, `createCollection`, and public `preload()` path over one shared database. The circular gate was surgically revised after explicit approval: it records each logical hydrate only after the public `preload()` request returns its Promise, and releases the held `BEGIN IMMEDIATE` after all such requests are pending. It no longer requires underlying registry queries to overtake the held non-preemptible persist. A test-only delegating `SQLiteDriver` still records every actual driver method admission, and the database wrapper separately records raw SQL dequeue order; neither schedules production observations. - -**Checkpoints.** Each public hydration completion records its logical completion ordinal, exact completed persist identities, bounded pending count, and raw dequeue count. Actual hydrated rows are captured from `collection.toArray` after public `preload()` completion; expected rows are never returned as observations. Wall-clock duration is never a verdict. - -**Neutral and hostile controls.** The neutral lane uses a fresh adapter with no queued persists and proves exact public rows plus a real persisted-row query. The hostile lane wraps the real `BrowserWASQLiteDriver` in a test-only global-FIFO scheduler, then traverses the same core adapter, persisted collection, and `preload()` fixture. This preserves the current persist-first fault even after production becomes fair and must be killed by the semantic checker. A separate synthetic observation calibrates the checker only. - -**Failure preservation.** All holds are released and all started promises are settled before collection/driver cleanup. Driver, collection, Node-directory, and OPFS cleanup failures are captured separately. The Node helper freezes the primary assertion before removing its directory. The browser page freezes the reached observation and semantic violation before OPFS cleanup. A cleanup failure is appended to an existing primary failure and can never relabel a reached checkpoint as setup failure. - -## Deterministic package-path receipts - -### Neutral reach - -```sh -../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ - -t 'reaches cold hydration' \ - --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose -``` - -Result: PASS, one selected test. Three fresh-adapter cold hydrates completed; all six actual public rows exactly matched their independently specified seed rows; persisted-row SELECT reach was observed; cleanup diagnostics were empty. - -### Narrow fixed semantic RED - -```sh -../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ - -t 'completes a pending cold hydrate' \ - --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose -``` - -Fixed history: `persist-0, persist-1, persist-2, hydrate-0`, with two mutations in each persist. - -Result: intended semantic FAIL at `fixed-persist-storm-hydrate-0`. Actual public rows and reach controls passed first. All three persists completed and zero remained pending; the candidate allowed only `persist-0` completed and required at least two pending. Cleanup diagnostics: `[]`. This was neither setup, timeout, typecheck, nor cleanup failure. - -### Generated ordered-history semantic RED and replay - -```sh -../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ - -t 'generated ordered' \ - --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose - -TANSTACK_DB_DRIVER_FAIRNESS_SEED=165905 \ -TANSTACK_DB_DRIVER_FAIRNESS_PATH=0 \ -../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ - -t 'generated ordered' \ - --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose -``` - -Both commands reproduce seed `165905`, path `0`, without shrinking: six hydrates, four persists, three mutations per persist, token history `p0,h3,h0,h4,p1,h1,p2,p3,h2,h5`. The executable kind history is `P,H,H,H,P,H,P,P,H,H`. Actual at the first public hydrate checkpoint: four completed, zero pending. The ordered candidate reference permitted only `persist-0` completed and required at least three pending. All actual public rows matched before the semantic verdict; cleanup diagnostics: `[]`. - -### Hostile persist-first controls - -```sh -../../node_modules/.bin/vitest run tests/shared-driver-fairness-oracle.test.ts \ - -t 'persist-first' \ - --coverage.enabled=false --typecheck.enabled=false --maxWorkers=1 --reporter=verbose -``` - -Result: PASS, two selected tests. The executable scheduling mutant admitted four persists then one cold hydrate through the real browser-driver/core/persisted/preload fixture. The checkpoint observed all four persists complete, zero pending, exact public rows, and no cleanup failure; the checker killed it. The separate synthetic calibration was also rejected with maximum completed `1` and minimum pending `3`. - -## Real Chrome/OPFS receipt - -The checked-in fixture uses `openBrowserWASQLiteOPFSDatabase`, its dedicated worker, WA-SQLite, and `OPFSCoopSyncVFS` in installed Google Chrome. Its Vite config deliberately excludes WA-SQLite from dependency prebundling so the real sibling WASM is served. The Playwright channel defaults to installed Chrome locally, bundled Chromium in CI, and accepts `PLAYWRIGHT_CHANNEL` as an explicit override. - -The existing `browser-single-tab-persisted-collection.e2e.test.ts` uses `createWASQLiteTestDatabase` backed by Node `better-sqlite3`; it earns driver/package evidence only and is not counted as OPFS evidence. - -The permanent checked-in package script was executed as: - -```sh -npm run test:opfs-fairness -``` - -The repository's global `pnpm` launcher hangs before printing even its version, so the equivalent documented workspace spelling, `pnpm --filter @tanstack/browser-db-sqlite-persistence test:opfs-fairness`, could not invoke the same script locally. The exact lockfile package was already present and was linked package-locally: `@playwright/test` and CLI version `1.60.0`. The package script, config, real Chrome, loopback Vite server, worker, and OPFS fixture all executed. The launcher defect is retained as external setup evidence and is not semantic RED. - -Neutral real-OPFS result: PASS. Two fresh-adapter cold hydrates returned four exact actual public rows, persisted-row SELECT reach was observed, and driver plus OPFS cleanup diagnostics were empty. - -Fixed real-OPFS storm semantic RED: - -- History: five persists followed by four cold hydrates; two mutations per persist. -- Logical completion order: `persist-0` through `persist-4`, then `hydrate-0` through `hydrate-3`. -- First hydrate checkpoint: all five persists completed, zero pending, raw dequeue count `111`. -- Candidate allowed only `persist-0` completed and required at least four pending. -- Total: `53` actual driver admissions and `113` raw SQL dequeues. -- Driver cleanup diagnostics `[]`; OPFS cleanup diagnostics `[]`. -- Page status reached `complete`; the fixture reported the frozen semantic observation after cleanup. This was not setup, wall-clock, browser, typecheck, or cleanup failure. - -## Static and hygiene receipts - -- Package-local `@playwright/test`: resolved version `1.60.0`; package-local CLI reports `Version 1.60.0`. -- Package TypeScript: PASS, `tsc --noEmit -p tsconfig.json`. -- Prettier: PASS for every new/changed package source/config file. The repository lockfile's existing full-file format differs from current Prettier; the three-line importer edit preserves adjacent lockfile style. -- ESLint: PASS for oracle, Node test, OPFS page/spec, and configs. -- `git diff --check`: PASS. -- Production-source diff under `packages/browser-db-sqlite-persistence/src` and `packages/db-sqlite-persistence-core/src`: empty. - -## Coverage limits - -- The Node fixture uses a real `BrowserWASQLiteDriver` but a `better-sqlite3` database handle. It proves deterministic package scheduling and SQLite results, not browser or OPFS behavior. -- The Chrome receipt proves the same fixed order through a real worker/WA-SQLite/OPFS path. It is one finite browser/origin/topology and does not establish multi-tab OPFS ownership, elapsed performance, every browser, or statement-batching throughput. -- The oracle establishes starvation order and bounded pending work only. It does not choose a batching implementation, permit transaction preemption, weaken atomic commit, or prove that statement batching solves the issue's root cost. -- `preload()` request is the logical hydrate-admission checkpoint and its completion is the public hydration-completion checkpoint. Actual driver admissions and raw dequeues remain separate reach evidence; the gate no longer requires a registry query to overtake the held non-preemptible persist. -- The generated RED campaign stops at its first deterministic counterexample. Its unchanged broad history domain becomes a multi-run GREEN obligation after an approved fix. - -## Production-fix gate - -Production code must not change until all of the following are true: - -1. Maintainer explicitly chooses the fairness quantum/bound (accepts `K = 1` or supplies another finite value) and states whether the scheduled unit is a logical hydrate spanning cold setup or only individual driver reads. -2. If only individual reads receive priority, the oracle is revised before production work so it does not falsely require cold setup writes to inherit an unapproved lane. If logical hydrate priority is chosen, the owning API/lane boundary is recorded before implementation. -3. Independent read-only loss audit reports PASS on this final Phase 1 diff, receipts, replay, failure-preserving cleanup, real-path hostile control, and scope. -4. Maintainer gives explicit Phase 2 production-fix approval after reviewing the decision above. - -After approval, the smallest correctly owned fix must preserve the unchanged fixed/generated/OPFS inputs and checkpoints, keep an already-running transaction non-preemptible, make all three fairness oracles GREEN, and retain transaction atomicity. Focused/full package suites, typecheck/build/lint/format/replay/cleanup, and a second independent loss audit are then required. - -## Frozen content hashes - -The final non-ledger content hashes and combined manifest hash are recorded here after the last test-only edit. The ledger is hashed separately in the independent audit receipt to avoid a self-referential value. - - - -- `f31c08caf480f5b4e639bac0e2d189a2a6922c70dc3477bff2ac4a082e24af34` `package.json` -- `7bc690c092c30d6c2dc15d2aff7254c92ede3080f68f1006a97a6a3edf57df9b` `tsconfig.json` -- `a094af45b23210b6635683abe999814d53d24411f6f76000157cd57e205be7d9` `pnpm-lock.yaml` -- `fd8a9c633966b8d5531f95ed9465ceb95923550aa5f552c05f7864ac1d9bf9a1` `shared-driver-fairness.opfs.html` -- `a0ac7e13416f6164cf134a240f6b044cb661d98c85991ccff6486cac4c07882f` `shared-driver-fairness.opfs.spec.ts` -- `88ef755a14918ecbf289b9d8e0753012fa1d932bec08be940ed81d9f292606d7` `shared-driver-fairness.opfs.ts` -- `0676bc5223bd00c99a0aa77e7199225a3dedecfddfe590953845f40721094140` `playwright.opfs.config.ts` -- `666ac24f0c6e18f0d4d1e71995b5f6046e4c75918292a1b1896a87eb0cb69890` `shared-driver-fairness-oracle.test.ts` -- `f2446d5e7d8ddbf028dd3fa7252db95bde87bee1d54b40238e91ee675cabaf3d` `shared-driver-fairness-oracle.ts` -- `f9e7b499046544834f0d0cbbc7f9af0c9e92858d99b8bd974f6f51587c1e4550` `vite.opfs.config.ts` -- Tracked binary diff hash (package manifest, package tsconfig, lockfile): `a30c60f449a3332fbdd1824540f9b18fa7b47aa2b5799f8e5fe939c3e40622b9` -- Combined ordered SHA-256 manifest hash for all ten non-ledger files above, from the listed paths in order via `shasum -a 256 ... | shasum -a 256`: `a26caffcdfbac67187b59f8c8054702abe086f97290ede212d4d2861b78b89b5` - -## Independent loss-audit status - -`PASS` on baseline/HEAD/origin-main/merge-base `76d766e84afbfcde2900a661233dd59e1decd5c2`. The independent read-only auditor reproduced the fixed/generated/replay semantic RED, executable mutant and checker-control PASS, package typecheck PASS, and real Chrome/OPFS neutral PASS plus semantic RED. It verified no production-source diff, every prior failure-preservation correction, the combined non-ledger manifest hash, tracked diff hash, and pre-receipt ledger hash `c157b60497f982a9b638a01d461a5340f3dddedde6871b88b5e15ac4649aeb0e`. - -No production authorization follows from this PASS. The numeric fairness quantum and logical-hydrate versus individual-read scheduling unit remain maintainer decisions, followed by explicit Phase 2 approval. - -## Phase 2 gate-blocker receipt - -On `2026-09-18`, the maintainer approved `K = 1` with one complete logical cold hydrate, including registry/schema/metadata setup and local row application, as the non-preemptive priority unit. The intended ownership was a driver-shared two-lane scheduler in the SQLite core adapter, an explicit same-adapter scoped hydrate lease in the persisted runtime, and an unchanged generic `SQLiteDriver` FIFO/transaction contract. - -The frozen Phase 1 gate is incompatible with that exclusive unit: - -1. The fixture starts persist `P0` and holds its real `BEGIN IMMEDIATE` inside `BrowserWASQLiteDriver`. -2. A correct logical-operation scheduler leaves `P0` non-preemptible and queues hydrate `H0` as the next complete adapter operation. -3. The unchanged fixture releases `P0` only after the `collection_registry` query from every hydrate (`H0` through `Hn`) has already been admitted to that same driver FIFO. -4. Because an exclusive `H0` cannot begin and admit its registry query until `P0` completes, `P0` waits for `H0` admission while `H0` waits for `P0`: the frozen gate cannot reach its semantic checkpoint. - -Starting every hydrate callback early would satisfy the admission gate only by overlapping/interleaving multiple logical hydrate units. A duplicate probe query, SQL-text priority inference, or hidden driver-wrapper introspection would be test-shaped and would not implement the approved contract. These alternatives were rejected. - -Phase 2 therefore stopped before runtime wiring, verification, commit, push, or prep-PR. All partial scheduling scaffolding was removed with targeted patches. Production-source diff is empty, `git diff --check` passes, and every frozen non-ledger Phase 1 witness hash still matches. Resolution requires either a surgical oracle-gate revision that observes logical hydrate admission without requiring all underlying registry queries before `P0` release, or an explicit maintainer redefinition allowing overlapping hydrate scopes. The former is recommended. - -## Approved circular-gate repair and repeated RED boundary - -On `2026-09-18`, the maintainer approved the recommended surgical gate revision. Only `shared-driver-fairness-oracle.ts` changed: the fixture now observes the logical public `preload()` requests and releases `P0` after they are all pending. The scenario histories, independent reference, `K = 1` verdicts, seed/replay, public row values, completion checkpoints, neutral/hostile controls, cleanup logic, browser fixture, and every other non-ledger witness remain byte-for-byte unchanged. Production-source diff remains empty. - -Repeated receipts at this revised boundary: - -- Neutral Node reach: PASS; three hydrate requests/completions, six exact actual public rows, persisted-row query reach, cleanup `[]`. -- Fixed Node history: semantic RED at `fixed-persist-storm-hydrate-0`; three completed persists, zero pending, versus maximum one and minimum two; exact rows reached first; cleanup `[]`. -- Generated and explicit replay (`seed 165905`, `path 0`): both semantic RED on the unchanged counterexample and first hydrate checkpoint; four completed, zero pending, versus maximum one and minimum three; cleanup `[]`. -- Executable persist-first package-path mutant and synthetic checker calibration: PASS; both were rejected for the intended fairness violation, with exact public rows and cleanup `[]` for the executable control. -- Real installed Chrome, worker WA-SQLite, and `OPFSCoopSyncVFS`: neutral PASS; storm semantic RED with five completed, zero pending, versus maximum one and minimum four. Logical completion order remained all five persists then all four hydrates; `53` driver admissions, `113` raw dequeues, driver cleanup `[]`, OPFS cleanup `[]`. -- Package TypeScript, ESLint, Prettier check, working/staged `git diff --check`: PASS. Playwright result artifacts were removed after their semantic contents were recorded; no package-local result/report directory remains. - -The first unprivileged browser attempt failed before fixture startup because the sandbox denied loopback listen with `EPERM`; it is classified as setup-only. The explicitly permitted rerun reached both real-browser checkpoints above. This setup failure is not semantic RED. - -The revised test-only boundary is now frozen at the hashes above. Production scheduling remains blocked until a fresh independent read-only loss audit reports PASS on this boundary. - -## Revised RED independent loss-audit PASS receipt - -The fresh independent read-only auditor reported `PASS — no blockers` on the -revised logical-request gate before production work resumed. The exact audited -pre-receipt ledger SHA-256 was -`45ad59441ab46cd001557b189db5520ad8786a436f10f24bdd3115bcd2c25132`. - -The auditor independently confirmed: - -- baseline, `HEAD`, `origin/main`, and merge base - `76d766e84afbfcde2900a661233dd59e1decd5c2`; -- an empty production-source diff and no commit, push, PR, WS5A, PR #1487, or - PR #1837 integration at the audited boundary; -- only the approved logical-`preload()` gate changed, while histories, the - `K = 1` reference, checkpoints, values, controls, replay, cleanup, and the - browser fixture remained unchanged; -- Node neutral and hostile controls PASS; fixed semantic RED `3 complete / 0 -pending` versus `max 1 / min 2`; generated and replay semantic RED `4 / 0` - versus `max 1 / min 3`; -- fresh installed Chrome, worker, WA-SQLite, and OPFS neutral PASS plus storm - semantic RED `5 / 0` versus `max 1 / min 4`, with `53` driver admissions, - `113` raw dequeues, and both cleanup arrays empty; -- package static checks and workspace cleanup PASS; -- frozen tracked-diff SHA-256 - `a30c60f449a3332fbdd1824540f9b18fa7b47aa2b5799f8e5fe939c3e40622b9` - and combined non-ledger manifest SHA-256 - `a26caffcdfbac67187b59f8c8054702abe086f97290ede212d4d2861b78b89b5`. - -This durable receipt opens the already-approved WS5B production gate. It does -not claim GREEN or authorize prep-pr, commit, push, PR, or merge work. - -## Phase 2 GREEN implementation boundary - -The approved production implementation keeps `SQLiteDriver` transaction FIFO -and non-preemption semantics intact while adding an opt-in, driver-shared K=1 -logical scheduler in the SQLite core adapter. A branded promise emitted by the -browser WA-SQLite driver allows transparent driver wrappers to retain the -capability; the hostile global-FIFO wrapper intentionally does not. The core -adapter owns one hydrate queue and one regular queue for all adapters sharing -that driver. A complete scoped hydrate is the scheduling unit. When both lanes -remain queued, one regular operation is admitted between completed hydrates. - -The persisted runtime threads a deliberately unscheduled scoped adapter through -cold startup metadata, index bootstrap, row hydration, buffered transaction -flushes, gap recovery, and reloads. Browser and Electron leader-local -coordinator paths accept that same scoped adapter. Index lifecycle work outside -a hydrate and ordinary persistence continue through the regular lane. - -### Lock-order correction - -An initial full browser-conformance run found six deterministic timeouts. A -temporary diagnostic probe isolated a real lock inversion: an on-demand -`loadSubset` held the hydrate scheduler while waiting for `applyMutex`, while an -update held `applyMutex` while waiting for a regular scheduled persistence -operation. The final code consistently acquires `applyMutex` before entering a -hydrate scope for startup, resume-baseline hydration, `loadSubset`, and -`forceReloadSubset`. Applied-receipt waiting remains outside the scheduler -scope, and `hydrateBaseline` no longer reacquires the mutex. All temporary -`WS5B-PROBE`/`WS5B-RUNTIME` diagnostics were removed before final verification. - -### Final GREEN receipts - -- Exact formerly deadlocked browser case, `should maintain query state during -data changes`: `1/1` PASS. -- Full browser persisted conformance: `113/113` PASS. -- Unchanged fixed, generated, neutral, executable-hostile, and synthetic - fairness oracle: `5/5` PASS. -- Explicit unchanged replay, seed `165905`, path `0`: `1/1` PASS without - shrinking. -- Index bootstrap plus both hydration-buffer replay controls: `3/3` PASS. -- Full runtime packages with package typecheck disabled in Vitest: SQLite core - `96/96`, Browser `39/39`, Electron `27/27` PASS. -- Direct package TypeScript for SQLite core, Browser, and Electron: PASS. -- Dependency-order builds for db-ivm, db, SQLite core, Browser, and Electron: - PASS. -- Fresh installed Chrome, worker, WA-SQLite, and OPFS verification: neutral and - storm `2/2` PASS. Both fixtures reached their semantic checkpoints and empty - cleanup diagnostics. -- Focused ESLint: zero errors; one unchanged `require-await` warning remains on - `applyTargetedInvalidationUnsafe`. -- Prettier check, `git diff --check`, temporary-probe scan, and generated - result/coverage cleanup: PASS. - -The repository's global `pnpm` launcher still hangs during workspace discovery, -including for `pnpm --version`. Final package commands therefore used the exact -installed workspace binaries directly; the real OPFS lane used the checked-in -`npm run test:opfs-fairness` script. This is launcher/setup evidence only and -does not weaken any semantic result. - -### Final candidate content hashes - -The frozen RED/oracle files retain their recorded hashes. The final GREEN -candidate contains the following fifteen non-ledger files in this exact order: - -- `f31c08caf480f5b4e639bac0e2d189a2a6922c70dc3477bff2ac4a082e24af34` `packages/browser-db-sqlite-persistence/package.json` -- `d054c227abf7e5a87ef64779be5edea11a339d3d10967ca037243f038e36c3e5` `packages/browser-db-sqlite-persistence/src/browser-coordinator.ts` -- `747614816fc80b16bf5547c7ab571834568fc4786310f6f9ddfcf266b0ffd088` `packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts` -- `7bc690c092c30d6c2dc15d2aff7254c92ede3080f68f1006a97a6a3edf57df9b` `packages/browser-db-sqlite-persistence/tsconfig.json` -- `22a2b48d0a9a52f010b4913521e0d7a5f923cd4706d5d2a915630d9772caf69b` `packages/db-sqlite-persistence-core/src/persisted.ts` -- `86d58d2f0f7da96eda80d0b760721cefb8afb3334872310ed5c61da7016a5bf4` `packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts` -- `9f6d09f27a0e0e0ccd53b5421b7f0df33fe860baaf42a8e47c34dfd8f52125be` `packages/electron-db-sqlite-persistence/src/electron-coordinator.ts` -- `a094af45b23210b6635683abe999814d53d24411f6f76000157cd57e205be7d9` `pnpm-lock.yaml` -- `fd8a9c633966b8d5531f95ed9465ceb95923550aa5f552c05f7864ac1d9bf9a1` `packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html` -- `a0ac7e13416f6164cf134a240f6b044cb661d98c85991ccff6486cac4c07882f` `packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts` -- `88ef755a14918ecbf289b9d8e0753012fa1d932bec08be940ed81d9f292606d7` `packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts` -- `0676bc5223bd00c99a0aa77e7199225a3dedecfddfe590953845f40721094140` `packages/browser-db-sqlite-persistence/playwright.opfs.config.ts` -- `666ac24f0c6e18f0d4d1e71995b5f6046e4c75918292a1b1896a87eb0cb69890` `packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts` -- `f2446d5e7d8ddbf028dd3fa7252db95bde87bee1d54b40238e91ee675cabaf3d` `packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts` -- `f9e7b499046544834f0d0cbbc7f9af0c9e92858d99b8bd974f6f51587c1e4550` `packages/browser-db-sqlite-persistence/vite.opfs.config.ts` - -- Ordered combined non-ledger content-manifest SHA-256: - `b1b151dfd40cade3da4c30c2418642787bbf2a37a6a4f08106d8ea4f1fe9e9be`. -- Tracked binary diff SHA-256 for the eight modified tracked files: - `516ef26f49204b7e36f437dba92667193dd2221ab366473e343ae8f5fc8c5573`. - -`HEAD` remains the frozen baseline -`76d766e84afbfcde2900a661233dd59e1decd5c2`. Nothing has been committed, -pushed, or prepared as a PR. A fresh independent read-only final loss audit is -required before this GREEN boundary may advance to prep-pr. - -## Final independent GREEN loss-audit PASS receipt - -The fresh independent read-only auditor reported `PASS` on the frozen GREEN -candidate. The exact audited pre-receipt hashes were: - -- candidate content manifest: - `b1b151dfd40cade3da4c30c2418642787bbf2a37a6a4f08106d8ea4f1fe9e9be`; -- tracked binary diff: - `516ef26f49204b7e36f437dba92667193dd2221ab366473e343ae8f5fc8c5573`; -- ledger: - `bbf71465e6e4cf5e877c8e180810ed93b51f9deeda2395c416eaf65bf7e09d27`. - -The auditor independently confirmed every frozen RED/oracle per-file hash and -reproduced fairness `5/5`, replay seed `165905` path `0` `1/1`, browser -conformance `113/113`, SQLite core/Browser/Electron `96/96 + 39/39 + 27/27`, -index/bootstrap-buffer controls `3/3`, all three package typechecks, and real -installed Chrome/worker/WA-SQLite/OPFS `2/2` including cleanup assertions. The -initial sandboxed OPFS attempt failed to bind loopback with `EPERM`; the -permitted rerun passed, so that first attempt remains setup-only evidence. - -Independent semantic inspection confirmed K=1 alternation, non-preemptible -running transactions, the unscheduled scoped hydration adapter preventing -recursive scheduling, capability preservation through transparent promise -wrappers while the hostile FIFO wrapper remains unbranded, mutex-before- -scheduler lock ordering, scoped index/buffer/gap-recovery paths, and Browser and -Electron leader-local adapter propagation. ESLint had zero errors, Prettier and -diff checks passed, and only the existing `require-await` warning remained. - -Workspace status was exactly eight modified tracked files plus the expected -eight untracked ledger/oracle files, with nothing staged and no package-local -test-result contamination. Two `.last-run.json` receipts existed only under -`/private/tmp`. The auditor did not rerun builds in its read-only pass; the -coordinator's final dependency-order five-build PASS above remains the build -receipt. - -Residual coverage limits remain the single installed-Chrome/single-origin -topology. During the audit, local `github/main` had advanced to -`dffb17f34a5e0448b7d72f85d11e35ac8a0d4265`; the candidate `HEAD`, -`origin/main`, and merge base remained the frozen baseline -`76d766e84afbfcde2900a661233dd59e1decd5c2`. Any later prep-pr phase must fetch -and reconcile current main under its own approval gate. This PASS freezes WS5B -GREEN and does not authorize prep-pr, commit, push, PR creation, or merge. From 71668d92d41ba818ab915e6f5989d3c1379125cc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 16:49:41 +0100 Subject: [PATCH 08/14] test(persistence): clarify synchronous admission checkpoint --- .../tests/shared-logical-scheduling.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts index 2daa08e563..1c0e1a0438 100644 --- a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -17,9 +17,9 @@ * * Production boundary and checkpoint: both operations use * `createSQLiteCorePersistenceAdapter`; the hydration operation enters through - * `runInHydrationScope`. The first checkpoint is after the regular request has - * had microtasks to reach the gated driver, while the first query remains - * held. Losing wrapper identity or treating a function key as a getter admits + * `runInHydrationScope`. The first checkpoint is immediately after the regular + * request enters the adapter, while the first query remains held. Losing + * wrapper identity or treating a function key as a getter synchronously admits * the second query and fails the exact admission assertion. * * This focused contract test does not establish K=1 lane fairness, SQL result @@ -140,8 +140,6 @@ describe(`shared logical scheduling`, () => { await underlying.firstQueryEntered.promise const regular = regularAdapter.loadCollectionMetadata!(`regular`) - await Promise.resolve() - await Promise.resolve() expect(underlying.admissions).toEqual([`query`]) From 4b6289375d2e2920902e7642cf85cc7b564bcd09 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 17:43:06 +0100 Subject: [PATCH 09/14] docs(oracles): label fairness coverage limits --- .../tests/shared-driver-fairness-oracle.ts | 8 ++++---- .../db-sqlite-persistence-core/tests/persisted.test.ts | 6 +++--- .../tests/shared-logical-scheduling.test.ts | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts index 639ab85275..a2cb29b5d9 100644 --- a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts @@ -24,10 +24,10 @@ * * Observed public facts: admitted and completed logical IDs, completed and * pending persists at every hydrate checkpoint, independently seeded public - * collection rows, raw SQL dequeue reach, and cleanup diagnostics. The oracle - * does not establish elapsed-time latency, unbounded eventuality, multi-process - * coordination, or a browser matrix; the Chromium OPFS fixture separately - * refines the provider boundary. + * collection rows, raw SQL dequeue reach, and cleanup diagnostics. Known + * omissions: the oracle does not establish elapsed-time latency, unbounded + * eventuality, multi-process coordination, or a browser matrix; the Chromium + * OPFS fixture separately refines the provider boundary. * * Challenge and replay: the executable persist-first FIFO driver must violate * the same K=1 checker while using the public/core/driver path. Re-run a diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 862a3f2225..50ddd16833 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -46,9 +46,9 @@ import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' * cleanup, and restart. They compare durable state, public rows, metadata, * request options, sequence evidence, errors, and late-work fencing. * - * Driver SQL behavior, browser page ownership, native runtimes, and the shared - * conformance portfolio have separate owners. This file models persistence - * protocol state, not a particular SQLite engine. + * Known omissions: driver SQL behavior, browser page ownership, native + * runtimes, and the shared conformance portfolio have separate owners. This + * file models persistence protocol state, not a particular SQLite engine. */ type Todo = { diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts index 1c0e1a0438..65ef56c241 100644 --- a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -22,10 +22,10 @@ * wrapper identity or treating a function key as a getter synchronously admits * the second query and fails the exact admission assertion. * - * This focused contract test does not establish K=1 lane fairness, SQL result - * correctness, eventual progress under arbitrary I/O, or cross-process - * coordination. Those belong to the shared-driver oracle and provider - * refinements. + * Known omissions: this focused contract test does not establish K=1 lane + * fairness, SQL result correctness, eventual progress under arbitrary I/O, or + * cross-process coordination. Those belong to the shared-driver oracle and + * provider refinements. */ import { describe, expect, it } from 'vitest' import { From 2000990dca4abde8cbf7503c616cb36e9d5117e8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 11:29:06 +0100 Subject: [PATCH 10/14] fix(sqlite-persistence): preserve coordinator progress --- .../fix-shared-sqlite-hydration-fairness.md | 2 +- .../src/browser-coordinator.ts | 104 ++- .../src/browser-persistence.ts | 28 +- .../tests/browser-coordinator.test.ts | 596 ++++++++++++++++++ .../src/persisted.ts | 277 +++++--- .../src/sqlite-core-adapter.ts | 62 +- .../persisted-real-adapter-lifecycle.test.ts | 236 +++++++ .../tests/persisted.test.ts | 417 ++++++++++++ .../tests/shared-logical-scheduling.test.ts | 172 ++++- .../tests/sqlite-core-adapter.test.ts | 2 +- .../src/electron-coordinator.ts | 104 ++- .../src/renderer.ts | 28 +- .../tests/electron-coordinator.test.ts | 377 +++++++++++ 13 files changed, 2211 insertions(+), 194 deletions(-) create mode 100644 packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts create mode 100644 packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts diff --git a/.changeset/fix-shared-sqlite-hydration-fairness.md b/.changeset/fix-shared-sqlite-hydration-fairness.md index d2055d929a..cd5b79d342 100644 --- a/.changeset/fix-shared-sqlite-hydration-fairness.md +++ b/.changeset/fix-shared-sqlite-hydration-fairness.md @@ -4,4 +4,4 @@ '@tanstack/electron-db-sqlite-persistence': patch --- -Prevent cold SQLite hydrations from being starved by unrelated queued writes when collections share a browser driver. Schedule each complete hydrate fairly while preserving transaction atomicity and leader-local persistence coordination. +Schedule complete SQLite hydration units fairly without holding coordinator work inside the local hydration scope. Preserve per-Collection leader adapter routing, mutation results across transport retries, terminal coordinator disposal, real-adapter restart order, and promise-discovered shared scheduling. diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 995a63ac94..9ae2fc9d2f 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -82,6 +82,11 @@ type CollectionState = { subscribers: Set<(message: ProtocolEnvelope) => void> } +type AppliedEnvelope = { + appliedAt: number + response: ApplyLocalMutationsResponse +} + // Adapter with pullSince support type AdapterWithPullSince = PersistenceAdapter & { pullSince?: ( @@ -123,10 +128,13 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private readonly nodeId = safeRandomUUID() private readonly dbName: string private adapter: AdapterWithPullSince | null + private readonly collectionAdapters = new Map() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopeIds = new Map() + private readonly disposedPromise: Promise + private rejectDisposed: ((error: Error) => void) | null = null private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -134,18 +142,27 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): AdapterWithPullSince { + let adapter = this.collectionAdapters.get(collectionId) + if (!adapter && this.adapter) { + adapter = this.adapter + this.collectionAdapters.set(collectionId, adapter) + } + if (!adapter) { throw new Error( `BrowserCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: BrowserCollectionCoordinatorOptions) { this.dbName = options.dbName this.adapter = options.adapter ?? null + this.disposedPromise = new Promise((_resolve, reject) => { + this.rejectDisposed = reject + }) + void this.disposedPromise.catch(() => undefined) this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) @@ -161,6 +178,14 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.adapter = adapter } + /** Register the adapter that owns leader-side work for one collection. */ + setAdapterForCollection( + collectionId: string, + adapter: AdapterWithPullSince, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + // ----------------------------------------------------------------------- // PersistedCollectionCoordinator interface // ----------------------------------------------------------------------- @@ -223,10 +248,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina signature: string, spec: PersistedIndexSpec, scopedAdapter?: HydrationPersistenceAdapter, + localEnsureCompleted = false, ): Promise { if (this.isLeader(collectionId)) { - // A scoped adapter is a leader-local capability and never crosses RPC. - await (scopedAdapter ?? this.requireAdapter()).ensureIndex( + if (localEnsureCompleted) return + await (scopedAdapter ?? this.requireAdapter(collectionId)).ensureIndex( collectionId, signature, spec, @@ -304,7 +330,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // ----------------------------------------------------------------------- dispose(): void { + if (this.disposed) return this.disposed = true + const disposedError = new Error(`coordinator disposed`) + this.rejectDisposed?.(disposedError) + this.rejectDisposed = null for (const [collectionId, state] of this.collections) { this.releaseLeadership(collectionId, state) @@ -312,12 +342,13 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina for (const [, pending] of this.pendingRPCs) { clearTimeout(pending.timer) - pending.reject(new Error(`coordinator disposed`)) + pending.reject(disposedError) } this.pendingRPCs.clear() this.channel.close() this.collections.clear() + this.collectionAdapters.clear() } // ----------------------------------------------------------------------- @@ -361,7 +392,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -499,16 +530,22 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId: string, request: RPCRequest, ): Promise { + if (this.isDisposed()) throw new Error(`coordinator disposed`) let lastError: Error | undefined for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { - await sleep(RPC_RETRY_DELAY_MS * attempt) + await Promise.race([ + sleep(RPC_RETRY_DELAY_MS * attempt), + this.disposedPromise, + ]) } + if (this.isDisposed()) throw new Error(`coordinator disposed`) try { return await this.sendRPCOnce(collectionId, request) } catch (error) { + if (this.isDisposed()) throw error lastError = error instanceof Error ? error : new Error(String(error)) } } @@ -623,7 +660,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -646,14 +683,9 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }, ): Promise { // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, - } + const appliedEnvelope = this.appliedEnvelopeIds.get(request.envelopeId) + if (appliedEnvelope) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } } const state = this.collections.get(collectionId) @@ -689,11 +721,25 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), ) - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + + // Retain the completed result so transport retries observe the same + // durable outcome without applying the envelope again. + this.appliedEnvelopeIds.set(request.envelopeId, { + appliedAt: Date.now(), + response, + }) this.pruneAppliedEnvelopeIds() // Broadcast tx:committed to all tabs @@ -728,15 +774,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina subscriber(txCommitted) } - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), - } + return response } private async handlePullSince( @@ -750,7 +788,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ): Promise { const state = this.collections.get(collectionId) - const adapter = scopedAdapter ?? this.requireAdapter() + const adapter = scopedAdapter ?? this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -825,8 +863,8 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { + for (const [id, applied] of this.appliedEnvelopeIds) { + if (applied.appliedAt < cutoff) { this.appliedEnvelopeIds.delete(id) } } diff --git a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts index b36e564900..c9ac081fbf 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts @@ -131,32 +131,36 @@ export function createBrowserWASQLitePersistence( }) adapterCache.set(cacheKey, adapter) - // Wire the adapter into the multi-tab coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, ensureIndex, etc.) - if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { - resolvedCoordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator: resolvedCoordinator, - }) + collectionId?: string, + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if ( + collectionId !== undefined && + resolvedCoordinator instanceof BrowserCollectionCoordinator + ) { + resolvedCoordinator.setAdapterForCollection(collectionId, adapter) + } + return { adapter, coordinator: resolvedCoordinator } + } const defaultPersistence = createCollectionPersistence( `sync-absent`, undefined, ) + if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { + resolvedCoordinator.setAdapter(defaultPersistence.adapter) + } return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(mode, schemaVersion, collectionId), resolvePersistenceForMode: (mode) => createCollectionPersistence(mode, undefined), } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5f554006e2..3f333997eb 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,4 +1,27 @@ +/** + * # Which cross-window coordinator facts survive retries and disposal? + * + * Contract: leader work uses the adapter registered for that Collection. A + * retry after response loss observes the first durable mutation result without + * applying it again. Disposal is terminal for current and future RPC attempts. + * + * Model and history grammar: an independent collection-to-adapter map selects + * one call recorder; a completed envelope retains one result; a disposed flag + * permits no request posts. Generated histories vary Collection registration + * order, RPC kind, mutation count, response delivery, and disposal phase. + * + * Production path and observations: paired real coordinators communicate + * through controlled BroadcastChannel and Web Locks fixtures. The laws record + * exact adapter calls, durable transaction count, returned result, promise + * settlement, and request posts. + * + * Known omissions: these laws do not establish native browser scheduling, + * concurrent delivery of one in-flight envelope, or behavior after the dedup + * retention window. Electron keeps focused parity witnesses for the same + * protocol boundaries. + */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import fc from 'fast-check' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' @@ -12,6 +35,8 @@ const channels: Map< string, Set<{ onmessage: MessageHandler | null }> > = new Map() +let dropNextMessageWhen: ((data: unknown) => boolean) | undefined +let observePostedMessage: ((data: unknown) => void) | undefined class MockBroadcastChannel { readonly name: string @@ -26,6 +51,11 @@ class MockBroadcastChannel { } postMessage(data: unknown): void { + observePostedMessage?.(data) + if (dropNextMessageWhen?.(data)) { + dropNextMessageWhen = undefined + return + } const peers = channels.get(this.name) if (!peers) return // Deliver to all other instances on same channel (simulating cross-tab) @@ -162,6 +192,8 @@ function installGlobals(): void { } function cleanupGlobals(): void { + dropNextMessageWhen = undefined + observePostedMessage = undefined channels.clear() heldLocks.clear() lockQueues.clear() @@ -460,6 +492,167 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + + it(`replays the successful mutation result when its first response is lost`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstSuccessDropped!: () => void + const firstSuccessDroppedPromise = new Promise((resolve) => { + firstSuccessDropped = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(`todos`, () => {}) + + dropNextMessageWhen = (data) => { + const payload = ( + data as { payload?: { type?: string; ok?: boolean } } + ).payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + firstSuccessDropped() + return true + } + return false + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-lost-response`, + type: `insert`, + key: `lost-response`, + value: { id: `lost-response`, title: `Persisted once` }, + }, + ]) + + await firstSuccessDroppedPromise + expect(adapter.appliedTxs).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(10_200) + const response = await responsePromise + expect(response).toMatchObject({ + ok: true, + acceptedMutationIds: [`mut-lost-response`], + }) + expect(adapter.appliedTxs).toHaveLength(1) + } finally { + follower.dispose() + leader.dispose() + vi.useRealTimers() + } + }) + + it(`preserves one durable mutation result across generated response-delivery histories`, async () => { + vi.useFakeTimers() + let run = 0 + try { + await fc.assert( + fc.asyncProperty( + fc.record({ + mutationCount: fc.integer({ min: 1, max: 4 }), + dropFirstSuccess: fc.boolean(), + }), + async ({ mutationCount, dropFirstSuccess }) => { + run++ + const collectionId = `delivery-history-${run}` + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstSuccessDropped!: () => void + const firstSuccessDroppedPromise = new Promise( + (resolve) => { + firstSuccessDropped = resolve + }, + ) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + try { + leader.subscribe(collectionId, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(collectionId, () => {}) + + if (dropFirstSuccess) { + dropNextMessageWhen = (data) => { + const payload = ( + data as { payload?: { type?: string; ok?: boolean } } + ).payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + firstSuccessDropped() + return true + } + return false + } + } + + const mutations = Array.from( + { length: mutationCount }, + (_, index) => ({ + mutationId: `mut-${run}-${index}`, + type: `insert` as const, + key: `${index}`, + value: { id: `${index}`, title: `row ${index}` }, + }), + ) + const responsePromise = follower.requestApplyLocalMutations( + collectionId, + mutations, + ) + if (dropFirstSuccess) { + await firstSuccessDroppedPromise + await vi.advanceTimersByTimeAsync(10_200) + } + + const response = await responsePromise + expect(response).toMatchObject({ + ok: true, + acceptedMutationIds: mutations.map( + (mutation) => mutation.mutationId, + ), + }) + expect(adapter.appliedTxs).toHaveLength(1) + } finally { + dropNextMessageWhen = undefined + follower.dispose() + leader.dispose() + for (let microtask = 0; microtask < 4; microtask++) { + await Promise.resolve() + } + } + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + } finally { + vi.useRealTimers() + } + }) }) describe(`RPC - pullSince`, () => { @@ -528,6 +721,74 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + it(`does not repeat successful leader-local index creation`, async () => { + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coord = createCoordinator(adapter) + try { + coord.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + + const spec = { expressionSql: [`title`] } + await adapter.ensureIndex(`todos`, `idx-once`, spec) + await coord.requestEnsurePersistedIndex( + `todos`, + `idx-once`, + spec, + adapter, + true, + ) + + expect(adapter.ensureIndex).toHaveBeenCalledOnce() + } finally { + coord.dispose() + } + }) + + it(`uses a supplied leader-local adapter when local work is not complete`, async () => { + const registeredAdapter = createStubAdapter() + const scopedAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + registeredAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + registeredAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + scopedAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coord = createCoordinator(registeredAdapter) + try { + coord.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + + await coord.requestEnsurePersistedIndex( + `todos`, + `idx-scoped`, + { expressionSql: [`title`] }, + scopedAdapter, + ) + + expect(scopedAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(registeredAdapter.ensureIndex).not.toHaveBeenCalled() + } finally { + coord.dispose() + } + }) + it(`follower routes ensurePersistedIndex to leader`, async () => { const adapter = createStubAdapter() adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) @@ -548,6 +809,154 @@ describe(`BrowserCollectionCoordinator`, () => { leader.dispose() follower.dispose() }) + + it(`keeps leader RPC work on the adapter registered for its collection`, async () => { + const todosAdapter = createStubAdapter() + const notesAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + todosAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + notesAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + todosAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(todosAdapter) + const follower = createCoordinator(notesAdapter) + + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + expect(leader.isLeader(`todos`)).toBe(true) + follower.subscribe(`todos`, () => {}) + + // Resolving a later collection variant must not replace the adapter + // already owning leader-side work for `todos`. + leader.setAdapter(notesAdapter) + await follower.requestEnsurePersistedIndex(`todos`, `idx-todos`, { + expressionSql: [`title`], + }) + + expect(todosAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(notesAdapter.ensureIndex).not.toHaveBeenCalled() + } finally { + follower.dispose() + leader.dispose() + } + }) + + it(`routes generated collection and RPC histories through their owning adapter`, async () => { + let run = 0 + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 2, max: 4 }).chain((collectionCount) => + fc.record({ + collectionCount: fc.constant(collectionCount), + registrationOrder: fc.shuffledSubarray( + Array.from({ length: collectionCount }, (_, index) => index), + { minLength: collectionCount, maxLength: collectionCount }, + ), + targetIndex: fc.integer({ min: 0, max: collectionCount - 1 }), + rpcKind: fc.constantFrom(`index`, `pull`, `mutation`), + }), + ), + async ({ + collectionCount, + registrationOrder, + targetIndex, + rpcKind, + }) => { + run++ + const collectionIds = Array.from( + { length: collectionCount }, + (_, index) => `routing-${run}-${index}`, + ) + const adapters = collectionIds.map(() => createStubAdapter()) + const ensureCalls = adapters.map(() => vi.fn()) + const pullCalls = adapters.map(() => vi.fn()) + adapters.forEach((adapter, index) => { + adapter.ensureIndex = + ensureCalls[index]!.mockResolvedValue(undefined) + adapter.pullSince = pullCalls[index]!.mockResolvedValue({ + latestRowVersion: index, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + }) + }) + + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + adapters[targetIndex]!.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapters[0]) + const follower = createCoordinator(adapters.at(-1)) + const targetCollectionId = collectionIds[targetIndex]! + try { + for (const index of registrationOrder) { + leader.setAdapterForCollection( + collectionIds[index]!, + adapters[index]!, + ) + } + leader.subscribe(targetCollectionId, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(targetCollectionId, () => {}) + + if (rpcKind === `index`) { + await follower.requestEnsurePersistedIndex( + targetCollectionId, + `idx-${run}`, + { expressionSql: [`title`] }, + ) + } else if (rpcKind === `pull`) { + await follower.pullSince(targetCollectionId, 0) + } else { + const response = await follower.requestApplyLocalMutations( + targetCollectionId, + [ + { + mutationId: `mut-${run}`, + type: `insert`, + key: `${run}`, + value: { id: `${run}`, title: `row ${run}` }, + }, + ], + ) + expect(response.ok).toBe(true) + } + + adapters.forEach((adapter, index) => { + const callCount = + rpcKind === `index` + ? ensureCalls[index]!.mock.calls.length + : rpcKind === `pull` + ? pullCalls[index]!.mock.calls.length + : adapter.appliedTxs.length + expect(callCount).toBe(index === targetIndex ? 1 : 0) + }) + } finally { + follower.dispose() + leader.dispose() + for (let microtask = 0; microtask < 4; microtask++) { + await Promise.resolve() + } + } + }, + ), + { seed: 1868, numRuns: 18, endOnFailure: true }, + ) + }) }) describe(`dispose`, () => { @@ -561,5 +970,192 @@ describe(`BrowserCollectionCoordinator`, () => { // Should not throw after disposal expect(coord.isLeader(`todos`)).toBe(false) }) + + it(`settles an in-flight RPC at dispose without posting retries`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstRequestPosted!: () => void + const firstRequestPostedPromise = new Promise((resolve) => { + firstRequestPosted = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + let requestPosts = 0 + let settled = false + + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(`todos`, () => {}) + + observePostedMessage = (data) => { + const type = (data as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensurePersistedIndex:req`) { + requestPosts++ + if (requestPosts === 1) firstRequestPosted() + } + } + dropNextMessageWhen = (data) => + (data as { payload?: { type?: string } }).payload?.type === + `rpc:ensurePersistedIndex:req` + + const request = follower + .requestEnsurePersistedIndex(`todos`, `idx-dispose`, { + expressionSql: [`title`], + }) + .then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + + await firstRequestPostedPromise + follower.dispose() + for (let microtask = 0; microtask < 8; microtask++) { + await Promise.resolve() + } + const settledAtDispose = settled + + await vi.advanceTimersByTimeAsync(21_000) + await request + + expect({ settledAtDispose, requestPosts }).toEqual({ + settledAtDispose: true, + requestPosts: 1, + }) + } finally { + follower.dispose() + leader.dispose() + vi.useRealTimers() + } + }) + + it(`makes disposal terminal across generated RPC kinds and lifecycle phases`, async () => { + vi.useFakeTimers() + let run = 0 + try { + await fc.assert( + fc.asyncProperty( + fc.record({ + rpcKind: fc.constantFrom(`index`, `pull`, `mutation`), + disposePhase: fc.constantFrom( + `before-request`, + `pending`, + `retry-delay`, + ), + }), + async ({ rpcKind, disposePhase }) => { + run++ + const collectionId = `dispose-history-${run}` + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstRequestPosted!: () => void + const firstRequestPostedPromise = new Promise((resolve) => { + firstRequestPosted = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + let requestPosts = 0 + const requestType = + rpcKind === `index` + ? `rpc:ensurePersistedIndex:req` + : rpcKind === `pull` + ? `rpc:pullSince:req` + : `rpc:applyLocalMutations:req` + + try { + leader.subscribe(collectionId, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(collectionId, () => {}) + + observePostedMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + requestType + ) { + requestPosts++ + if (requestPosts === 1) firstRequestPosted() + } + } + dropNextMessageWhen = (data) => + (data as { payload?: { type?: string } }).payload?.type === + requestType + + if (disposePhase === `before-request`) { + follower.dispose() + } + const request = + rpcKind === `index` + ? follower.requestEnsurePersistedIndex( + collectionId, + `idx-${run}`, + { expressionSql: [`title`] }, + ) + : rpcKind === `pull` + ? follower.pullSince(collectionId, 0) + : follower.requestApplyLocalMutations(collectionId, [ + { + mutationId: `mut-${run}`, + type: `insert`, + key: `${run}`, + value: { id: `${run}`, title: `row ${run}` }, + }, + ]) + const settled = request.then( + () => `resolved` as const, + () => `rejected` as const, + ) + + if (disposePhase !== `before-request`) { + await firstRequestPostedPromise + if (disposePhase === `retry-delay`) { + await vi.advanceTimersByTimeAsync(10_000) + } + follower.dispose() + } + + await expect(settled).resolves.toBe(`rejected`) + expect(requestPosts).toBe( + disposePhase === `before-request` ? 0 : 1, + ) + } finally { + observePostedMessage = undefined + dropNextMessageWhen = undefined + follower.dispose() + leader.dispose() + for (let microtask = 0; microtask < 4; microtask++) { + await Promise.resolve() + } + } + }, + ), + { seed: 1868, numRuns: 18, endOnFailure: true }, + ) + } finally { + vi.useRealTimers() + } + }) }) }) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 8cbcf7d811..940df95177 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -318,6 +318,8 @@ export interface PersistenceAdapter { runInHydrationScope?: ( task: (adapter: HydrationPersistenceAdapter) => Promise, ) => Promise + /** Whether hydration scopes currently enter a shared driver scheduler. */ + isHydrationScopeScheduled?: () => boolean } export type HydrationPersistenceAdapter = PersistenceAdapter & { @@ -374,12 +376,17 @@ export interface PersistedCollectionCoordinator { collectionId: string, options: LoadSubsetOptions, ) => Promise - /** The scoped adapter is leader-local and is never serialized to a follower. */ + /** + * Requests leader-side index creation. The scoped adapter is leader-local + * and is never serialized to a follower. `localEnsureCompleted` lets the + * built-in leader avoid repeating successful local work. + */ requestEnsurePersistedIndex: ( collectionId: string, signature: string, spec: PersistedIndexSpec, scopedAdapter?: HydrationPersistenceAdapter, + localEnsureCompleted?: boolean, ) => Promise requestApplyLocalMutations?: ( collectionId: string, @@ -872,6 +879,7 @@ class PersistedCollectionRuntime< } private startupMetadataPromise: Promise | null = null private startPromise: Promise | null = null + private hasAttemptedStartup = false private resumeBaselinePromise: Promise | null = null private lifecycleGeneration = 0 private internalApplyDepth = 0 @@ -968,11 +976,12 @@ class PersistedCollectionRuntime< private runInHydrationScope( task: (adapter: HydrationPersistenceAdapter) => Promise, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { - if (this.persistence.adapter.runInHydrationScope) { - return this.persistence.adapter.runInHydrationScope(task) + if (adapter.runInHydrationScope) { + return adapter.runInHydrationScope(task) } - return Promise.resolve().then(() => task(this.persistence.adapter)) + return Promise.resolve().then(() => task(adapter)) } async ensureStarted(): Promise { @@ -981,6 +990,8 @@ class PersistedCollectionRuntime< } const lifecycleGeneration = this.lifecycleGeneration + const isRestart = this.hasAttemptedStartup + this.hasAttemptedStartup = true let resolveStartupMetadata!: () => void let rejectStartupMetadata!: (error: unknown) => void this.startupMetadataPromise = new Promise((resolve, reject) => { @@ -1008,28 +1019,59 @@ class PersistedCollectionRuntime< } } - let appliedCursor: number | undefined - if (this.persistence.adapter.runInHydrationScope) { - appliedCursor = await this.applyMutex.run(() => - this.runInHydrationScope(async (adapter) => { + let startup: + | { + appliedCursor: number | undefined + indexBootstrapSnapshot: Array + completedLocalIndexSignatures: Set + } + | undefined + const scheduleStartupAsOneHydrate = + this.persistence.adapter.runInHydrationScope !== undefined && + (!isRestart || + (this.persistence.adapter.isHydrationScopeScheduled?.() ?? true)) + if (scheduleStartupAsOneHydrate) { + startup = await this.applyMutex.run(async () => { + const result = await this.runInHydrationScope(async (adapter) => { if (!(await loadStartupMetadata(adapter))) return undefined return this.startInternal(lifecycleGeneration, adapter) - }), - ) + }) + await this.flushQueuedTxCommittedUnsafe() + return result + }) } else { // Preserve the existing unscheduled-adapter lifecycle contract: a // replacement upstream may start while stale hydration is settling. if (await loadStartupMetadata(this.persistence.adapter)) { - appliedCursor = await this.applyMutex.run(() => - this.startInternal(lifecycleGeneration, this.persistence.adapter), - ) + startup = await this.applyMutex.run(async () => { + const result = + this.persistence.adapter.isHydrationScopeScheduled?.() + ? await this.runInHydrationScope((adapter) => + this.startInternal(lifecycleGeneration, adapter), + ) + : await this.startInternal( + lifecycleGeneration, + this.persistence.adapter, + ) + await this.flushQueuedTxCommittedUnsafe() + return result + }) } } if ( - appliedCursor !== undefined && + startup !== undefined && lifecycleGeneration === this.lifecycleGeneration ) { - await this.waitForAppliedReceiptsAfter(appliedCursor) + await this.requestCoordinatorPersistedIndexes( + startup.indexBootstrapSnapshot, + startup.completedLocalIndexSignatures, + ) + if ( + startup.appliedCursor !== undefined && + lifecycleGeneration === this.lifecycleGeneration + ) { + await this.waitForAppliedReceiptsAfter(startup.appliedCursor) + } } })() return this.startPromise @@ -1046,11 +1088,13 @@ class PersistedCollectionRuntime< if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.syncMode !== `on-demand`) return - const appliedCursor = await this.applyMutex.run(() => - this.runInHydrationScope((adapter) => + const appliedCursor = await this.applyMutex.run(async () => { + const result = await this.runInHydrationScope((adapter) => this.hydrateBaseline(lifecycleGeneration, adapter), - ), - ) + ) + await this.flushQueuedTxCommittedUnsafe() + return result + }) if ( appliedCursor !== undefined && lifecycleGeneration === this.lifecycleGeneration @@ -1095,18 +1139,35 @@ class PersistedCollectionRuntime< private async startInternal( lifecycleGeneration: number, adapter: HydrationPersistenceAdapter, - ): Promise { + ): Promise< + | { + appliedCursor: number | undefined + indexBootstrapSnapshot: Array + completedLocalIndexSignatures: Set + } + | undefined + > { if (lifecycleGeneration !== this.lifecycleGeneration) return undefined const indexBootstrapSnapshot = this.collection?.getIndexMetadata() ?? [] this.attachIndexLifecycleListeners() - await this.bootstrapPersistedIndexes(indexBootstrapSnapshot, adapter) + const completedLocalIndexSignatures = await this.bootstrapPersistedIndexes( + indexBootstrapSnapshot, + adapter, + ) if (lifecycleGeneration !== this.lifecycleGeneration) return undefined - if (this.syncMode !== `on-demand`) { - return this.hydrateBaseline(lifecycleGeneration, adapter) - } - return undefined + const appliedCursor = + this.syncMode !== `on-demand` + ? await this.hydrateBaseline(lifecycleGeneration, adapter) + : undefined + return lifecycleGeneration === this.lifecycleGeneration + ? { + appliedCursor, + indexBootstrapSnapshot, + completedLocalIndexSignatures, + } + : undefined } private async loadStartupMetadataInternal( @@ -1184,8 +1245,8 @@ class PersistedCollectionRuntime< this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(() => - this.runInHydrationScope((adapter) => + await this.applyMutex.run(async () => { + await this.runInHydrationScope((adapter) => this.hydrateSubsetUnsafe( options, { @@ -1194,8 +1255,9 @@ class PersistedCollectionRuntime< }, adapter, ), - ), - ) + ) + await this.flushQueuedTxCommittedUnsafe() + }) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) @@ -1233,8 +1295,8 @@ class PersistedCollectionRuntime< async forceReloadSubset(options: LoadSubsetOptions): Promise { const lifecycleGeneration = this.lifecycleGeneration // A one-shot refresh does not acquire an enduring subscription lease. - await this.applyMutex.run(() => - this.runInHydrationScope((adapter) => + await this.applyMutex.run(async () => { + await this.runInHydrationScope((adapter) => this.hydrateSubsetUnsafe( options, { @@ -1243,8 +1305,9 @@ class PersistedCollectionRuntime< }, adapter, ), - ), - ) + ) + await this.flushQueuedTxCommittedUnsafe() + }) } queueHydrationBufferedTransaction( @@ -1464,7 +1527,6 @@ class PersistedCollectionRuntime< } await this.flushQueuedHydrationTransactionsUnsafe(adapter) - await this.flushQueuedTxCommittedUnsafe(adapter) if (config.requestRemoteEnsure) { this.queueRemoteSubsetEnsure(options) @@ -2120,9 +2182,7 @@ class PersistedCollectionRuntime< } void this.applyMutex - .run(() => - this.processCommittedTxUnsafe(payload, this.persistence.adapter), - ) + .run(() => this.processCommittedTxUnsafe(payload)) .catch((error) => { console.warn(`Failed to process tx:committed message:`, error) }) @@ -2136,33 +2196,30 @@ class PersistedCollectionRuntime< if (isCollectionResetPayload(payload)) { void this.applyMutex - .run(() => - this.runInHydrationScope((adapter) => + .run(async () => { + await this.runInHydrationScope((adapter) => this.truncateAndReloadUnsafe(adapter), - ), - ) + ) + await this.flushQueuedTxCommittedUnsafe() + }) .catch((error) => { console.warn(`Failed to process collection reset message:`, error) }) } } - private async flushQueuedTxCommittedUnsafe( - adapter: HydrationPersistenceAdapter, - ): Promise { + private async flushQueuedTxCommittedUnsafe(): Promise { while (this.queuedTxCommitted.length > 0) { const queued = this.queuedTxCommitted.shift() if (!queued) { continue } - await this.processCommittedTxUnsafe(queued, adapter, true) + await this.processCommittedTxUnsafe(queued) } } private async processCommittedTxUnsafe( txCommitted: TxCommitted, - adapter: HydrationPersistenceAdapter, - hydrationScopeAlreadyActive = false, ): Promise { if (txCommitted.term < this.latestTerm) { return @@ -2183,13 +2240,7 @@ class PersistedCollectionRuntime< const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms if (hasGap) { - if (hydrationScopeAlreadyActive) { - await this.recoverFromSeqGapUnsafe(adapter) - } else { - await this.runInHydrationScope((scopedAdapter) => - this.recoverFromSeqGapUnsafe(scopedAdapter), - ) - } + await this.recoverFromSeqGapUnsafe() if ( txCommitted.term < this.latestTerm || (txCommitted.term === this.latestTerm && @@ -2207,20 +2258,17 @@ class PersistedCollectionRuntime< await this.invalidateFromCommittedTxUnsafe( txCommitted, - adapter, - hydrationScopeAlreadyActive, + this.persistence.adapter, ) + await this.flushQueuedTxCommittedUnsafe() } - private async recoverFromSeqGapUnsafe( - adapter: HydrationPersistenceAdapter, - ): Promise { + private async recoverFromSeqGapUnsafe(): Promise { if (this.persistence.coordinator.pullSince && this.latestRowVersion >= 0) { try { const pullResponse = await this.persistence.coordinator.pullSince( this.collectionId, this.latestRowVersion, - adapter, ) if (pullResponse.ok) { @@ -2229,29 +2277,40 @@ class PersistedCollectionRuntime< pullResponse.latestSeq, pullResponse.latestRowVersion, ) - if (pullResponse.requiresFullReload || !pullResponse.deltas) { - await this.reloadActiveSubsetsUnsafe(adapter) + if (pullResponse.requiresFullReload) { + await this.runInHydrationScope((adapter) => + this.reloadActiveSubsetsUnsafe(adapter), + ) return } - - for (const delta of pullResponse.deltas) { - await this.invalidateFromCommittedTxUnsafe( - { - type: `tx:committed`, - term: pullResponse.latestTerm, - seq: pullResponse.latestSeq, - txId: delta.txId, - latestRowVersion: delta.latestRowVersion, - requiresFullReload: false, - changedRows: delta.changedRows, - deletedKeys: delta.deletedKeys, - rowMetadataMutations: delta.rowMetadataMutations, - collectionMetadataMutations: delta.collectionMetadataMutations, - }, - adapter, - true, + const deltas = pullResponse.deltas + if (!deltas) { + await this.runInHydrationScope((adapter) => + this.reloadActiveSubsetsUnsafe(adapter), ) + return } + + await this.runInHydrationScope(async (adapter) => { + for (const delta of deltas) { + await this.invalidateFromCommittedTxUnsafe( + { + type: `tx:committed`, + term: pullResponse.latestTerm, + seq: pullResponse.latestSeq, + txId: delta.txId, + latestRowVersion: delta.latestRowVersion, + requiresFullReload: false, + changedRows: delta.changedRows, + deletedKeys: delta.deletedKeys, + rowMetadataMutations: delta.rowMetadataMutations, + collectionMetadataMutations: + delta.collectionMetadataMutations, + }, + adapter, + ) + } + }) return } } catch (error) { @@ -2259,7 +2318,9 @@ class PersistedCollectionRuntime< } } - await this.truncateAndReloadUnsafe(adapter) + await this.runInHydrationScope((adapter) => + this.truncateAndReloadUnsafe(adapter), + ) if (this.mode === `sync-present`) { for (const options of this.activeSubsets.values()) { @@ -2285,14 +2346,12 @@ class PersistedCollectionRuntime< private async invalidateFromCommittedTxUnsafe( txCommitted: TxCommitted, adapter: HydrationPersistenceAdapter, - hydrationScopeAlreadyActive = false, ): Promise { const reloadActiveSubsets = () => - hydrationScopeAlreadyActive - ? this.reloadActiveSubsetsUnsafe(adapter) - : this.runInHydrationScope((scopedAdapter) => - this.reloadActiveSubsetsUnsafe(scopedAdapter), - ) + this.runInHydrationScope( + (scopedAdapter) => this.reloadActiveSubsetsUnsafe(scopedAdapter), + adapter, + ) if (txCommitted.requiresFullReload) { await reloadActiveSubsets() @@ -2417,7 +2476,6 @@ class PersistedCollectionRuntime< } await this.flushQueuedHydrationTransactionsUnsafe(adapter) - await this.flushQueuedTxCommittedUnsafe(adapter) } private attachIndexLifecycleListeners(): void { @@ -2443,16 +2501,32 @@ class PersistedCollectionRuntime< private async bootstrapPersistedIndexes( indexMetadataSnapshot?: Array, adapter: HydrationPersistenceAdapter = this.persistence.adapter, - ): Promise { + ): Promise> { const collection = this.collection if (!collection && !indexMetadataSnapshot) { - return + return new Set() } const indexMetadata = indexMetadataSnapshot ?? collection?.getIndexMetadata() ?? [] + const completedLocalIndexSignatures = new Set() + for (const metadata of indexMetadata) { + if (await this.ensureLocalPersistedIndex(metadata, adapter)) { + completedLocalIndexSignatures.add(metadata.signature) + } + } + return completedLocalIndexSignatures + } + + private async requestCoordinatorPersistedIndexes( + indexMetadata: Array, + completedLocalIndexSignatures: ReadonlySet, + ): Promise { for (const metadata of indexMetadata) { - await this.ensurePersistedIndex(metadata, adapter) + await this.requestCoordinatorPersistedIndex( + metadata, + completedLocalIndexSignatures.has(metadata.signature), + ) } } @@ -2473,6 +2547,17 @@ class PersistedCollectionRuntime< indexMetadata: CollectionIndexMetadata, adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { + const completedLocally = await this.ensureLocalPersistedIndex( + indexMetadata, + adapter, + ) + await this.requestCoordinatorPersistedIndex(indexMetadata, completedLocally) + } + + private async ensureLocalPersistedIndex( + indexMetadata: CollectionIndexMetadata, + adapter: HydrationPersistenceAdapter, + ): Promise { const spec = this.buildPersistedIndexSpec(indexMetadata) try { @@ -2481,16 +2566,26 @@ class PersistedCollectionRuntime< indexMetadata.signature, spec, ) + return true } catch (error) { console.warn(`Failed to ensure persisted index in adapter:`, error) + return false } + } + + private async requestCoordinatorPersistedIndex( + indexMetadata: CollectionIndexMetadata, + completedLocally: boolean, + ): Promise { + const spec = this.buildPersistedIndexSpec(indexMetadata) try { await this.persistence.coordinator.requestEnsurePersistedIndex( this.collectionId, indexMetadata.signature, spec, - adapter, + completedLocally ? this.persistence.adapter : undefined, + completedLocally, ) } catch (error) { console.warn( diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index f76e1f310d..075651111a 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -96,6 +96,17 @@ class SharedPersistenceScheduler { return this.enqueue(`hydrate`, task) } + adoptRunningHydrate(completion: Promise): void { + if (this.running) return + this.running = true + const finish = () => { + this.lastCompletedKind = `hydrate` + this.running = false + this.drain() + } + void completion.then(finish, finish) + } + private enqueue( kind: ScheduledOperationKind, task: () => Promise, @@ -157,6 +168,7 @@ const sharedPersistenceSchedulers = new WeakMap< object, SharedPersistenceScheduler >() +const observedDriverSchedulingKeys = new WeakMap() function getSharedPersistenceScheduler( key: object, @@ -187,7 +199,10 @@ function observeSharedLogicalSchedulingSupport( driver: SQLiteDriver, onSupport: (key: object) => void, ): SQLiteDriver { + let observationPending = true const observe = (promise: Promise): Promise => { + if (!observationPending) return promise + observationPending = false const key = getSharedLogicalSchedulingKey(promise) if (key) onSupport(key) return promise @@ -1163,7 +1178,9 @@ function buildIndexName(collectionId: string, signature: string): string { export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { private readonly driver: SQLiteDriver + private readonly schedulingIdentitySource: SQLiteDriver private scheduler: SharedPersistenceScheduler | undefined + private activeUnscheduledHydration: Promise | undefined private readonly hydrationAdapter: HydrationPersistenceAdapter private readonly schemaVersion: number private readonly schemaMismatchPolicy: SQLiteCoreAdapterSchemaMismatchPolicy @@ -1220,14 +1237,22 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ) } - const schedulingKey = getSharedLogicalSchedulingKey(options.driver) + this.schedulingIdentitySource = options.driver + const schedulingKey = + getSharedLogicalSchedulingKey(options.driver) ?? + observedDriverSchedulingKeys.get(options.driver) this.scheduler = schedulingKey ? getSharedPersistenceScheduler(schedulingKey) : undefined this.driver = schedulingKey ? options.driver : observeSharedLogicalSchedulingSupport(options.driver, (key) => { - this.scheduler ??= getSharedPersistenceScheduler(key) + observedDriverSchedulingKeys.set(options.driver, key) + const scheduler = getSharedPersistenceScheduler(key) + this.scheduler ??= scheduler + if (this.activeUnscheduledHydration) { + scheduler.adoptRunningHydrate(this.activeUnscheduledHydration) + } }) this.schemaVersion = schemaVersion this.schemaMismatchPolicy = @@ -1259,13 +1284,38 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { runInHydrationScope( task: (adapter: HydrationPersistenceAdapter) => Promise, ): Promise { - return this.scheduler - ? this.scheduler.runHydrate(() => task(this.hydrationAdapter)) - : Promise.resolve().then(() => task(this.hydrationAdapter)) + const scheduler = this.resolveScheduler() + if (scheduler) { + return scheduler.runHydrate(() => task(this.hydrationAdapter)) + } + + const hydration = Promise.resolve().then(() => task(this.hydrationAdapter)) + this.activeUnscheduledHydration = hydration + const clear = () => { + if (this.activeUnscheduledHydration === hydration) { + this.activeUnscheduledHydration = undefined + } + } + void hydration.then(clear, clear) + return hydration + } + + isHydrationScopeScheduled(): boolean { + return this.resolveScheduler() !== undefined } private runRegular(task: () => Promise): Promise { - return this.scheduler ? this.scheduler.runRegular(task) : task() + const scheduler = this.resolveScheduler() + return scheduler ? scheduler.runRegular(task) : task() + } + + private resolveScheduler(): SharedPersistenceScheduler | undefined { + if (this.scheduler) return this.scheduler + const key = observedDriverSchedulingKeys.get(this.schedulingIdentitySource) + if (key) { + this.scheduler = getSharedPersistenceScheduler(key) + } + return this.scheduler } private runInTransaction( diff --git a/packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts b/packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts new file mode 100644 index 0000000000..0f768887ca --- /dev/null +++ b/packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts @@ -0,0 +1,236 @@ +/** + * # Does real-adapter restart preserve the sync-run start boundary? + * + * A stale lifecycle may finish its already-admitted SQLite read, but it must + * not retain the wrapper mutex that gates startup metadata for the replacement + * sync run. The replacement upstream sync function may start once its metadata + * is loaded, before the stale row read is released. This file uses the real + * core adapter because recording adapters do not expose its public + * hydration-scope method and therefore select a different startup branch. + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it } from 'vitest' +import fc from 'fast-check' +import { createCollection } from '@tanstack/db' +import { + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + createSQLiteCorePersistenceAdapter, + persistedCollectionOptions, +} from '../src' +import { SqliteCliDriver } from './sqlite-core-adapter.test' +import type { SQLiteDriver } from '../src' + +type Deferred = { + promise: Promise + resolve: () => void +} + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +class QueryGateDriver implements SQLiteDriver { + readonly queries: Array = [] + private matcher: ((sql: string) => boolean) | undefined + private gate: Deferred | undefined + private entered: Deferred | undefined + + constructor( + private readonly driver: SQLiteDriver, + schedulingKey?: object, + ) { + if (schedulingKey) { + Object.defineProperty(this, SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, { + value: schedulingKey, + }) + } + } + + holdNextQuery(matcher: (sql: string) => boolean): { + entered: Promise + release: () => void + } { + this.matcher = matcher + this.gate = createDeferred() + this.entered = createDeferred() + return { + entered: this.entered.promise, + release: () => this.release(), + } + } + + exec(sql: string): Promise { + return this.driver.exec(sql) + } + + async query( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + const normalizedSql = sql.replace(/\s+/g, ` `).trim() + this.queries.push(normalizedSql) + if (this.matcher?.(sql)) { + this.matcher = undefined + this.entered?.resolve() + await this.gate?.promise + } + if ( + normalizedSql.includes(`FROM leader_term`) || + normalizedSql.includes(`FROM collection_version`) || + normalizedSql.includes(`FROM applied_tx`) || + normalizedSql.includes(`FROM collection_metadata`) + ) { + return [] + } + return this.driver.query(sql, params) + } + + run(sql: string, params: ReadonlyArray = []): Promise { + return this.driver.run(sql, params) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.driver.transaction(fn) + } + + transactionWithDriver( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.driver.transactionWithDriver + ? this.driver.transactionWithDriver(fn) + : this.driver.transaction(fn) + } + + private release(): void { + this.gate?.resolve() + this.gate = undefined + this.entered = undefined + } +} + +async function observeRestartOrder(options: { + id: string + scheduled: boolean + schemaVersion: number +}): Promise<{ + sourceStartsBeforeRelease: number + sourceStartsAfterRelease: number + streamPositionReadsBeforeRelease: number + collectionMetadataReadsBeforeRelease: number +}> { + const directory = mkdtempSync(join(tmpdir(), `persisted-real-lifecycle-`)) + const driver = new QueryGateDriver( + new SqliteCliDriver(join(directory, `state.sqlite`)), + options.scheduled ? {} : undefined, + ) + const adapter = createSQLiteCorePersistenceAdapter({ + driver, + schemaVersion: options.schemaVersion, + }) + const staleRows = driver.holdNextQuery((sql) => + /SELECT\s+key,\s*value,\s*metadata,\s*row_version\s+FROM/i.test(sql), + ) + let sourceStarts = 0 + + const collection = createCollection( + persistedCollectionOptions<{ id: string }, string>({ + id: `real-adapter-restart-order-${options.id}`, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceStarts++ + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const stalePreload = Promise.resolve(collection.preload()) + void stalePreload.catch(() => undefined) + let freshReady: Promise | undefined + let observation = { + sourceStartsBeforeRelease: -1, + sourceStartsAfterRelease: -1, + streamPositionReadsBeforeRelease: -1, + collectionMetadataReadsBeforeRelease: -1, + } + + try { + await staleRows.entered + expect(sourceStarts).toBe(1) + await collection.cleanup() + + collection.startSyncImmediate() + freshReady = collection.stateWhenReady() + void freshReady.catch(() => undefined) + for (let microtask = 0; microtask < 12; microtask++) { + await Promise.resolve() + } + + observation = { + sourceStartsBeforeRelease: sourceStarts, + sourceStartsAfterRelease: -1, + streamPositionReadsBeforeRelease: driver.queries.filter((sql) => + sql.includes(`SELECT latest_term`), + ).length, + collectionMetadataReadsBeforeRelease: driver.queries.filter((sql) => + sql.includes(`FROM collection_metadata`), + ).length, + } + } finally { + staleRows.release() + await stalePreload.catch(() => undefined) + await freshReady?.catch(() => undefined) + observation.sourceStartsAfterRelease = sourceStarts + await collection.cleanup() + rmSync(directory, { recursive: true, force: true }) + } + + return observation +} + +it(`starts a rebound source before a stale real-adapter row read settles`, async () => { + await expect( + observeRestartOrder({ id: `shrink`, scheduled: false, schemaVersion: 1 }), + ).resolves.toEqual({ + sourceStartsBeforeRelease: 2, + sourceStartsAfterRelease: 2, + streamPositionReadsBeforeRelease: 4, + collectionMetadataReadsBeforeRelease: 2, + }) +}) + +it(`preserves generated restart order across real-adapter scheduler capabilities`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + scheduled: fc.boolean(), + schemaVersion: fc.integer({ min: 0, max: 3 }), + }), + async ({ scheduled, schemaVersion }) => { + const observation = await observeRestartOrder({ + id: `generated-${scheduled}-${schemaVersion}`, + scheduled, + schemaVersion, + }) + expect(observation.sourceStartsBeforeRelease).toBe(scheduled ? 1 : 2) + expect(observation.sourceStartsAfterRelease).toBe(2) + expect(observation.streamPositionReadsBeforeRelease).toBe( + scheduled ? 2 : 4, + ) + expect(observation.collectionMetadataReadsBeforeRelease).toBe( + scheduled ? 1 : 2, + ) + }, + ), + { seed: 1868, numRuns: 6, endOnFailure: true }, + ) +}) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 50ddd16833..7171003731 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import fc from 'fast-check' import { BasicIndex, DbClient, @@ -1483,6 +1484,339 @@ describe(`persistedCollectionOptions`, () => { ).toBe(true) }) + it(`only signals completed leader-local index work after local success`, async () => { + const adapter = createRecordingAdapter() + const coordinator = createCoordinatorHarness() + const localFailure = new Error(`local index creation failed`) + const completedLocalMarkers: Array = [] + const warn = vi.spyOn(console, `warn`).mockImplementation(() => {}) + + adapter.ensureIndex = async (collectionId, signature) => { + adapter.ensureIndexCalls.push({ collectionId, signature }) + throw localFailure + } + coordinator.requestEnsurePersistedIndex = async ( + _collectionId, + _signature, + _spec, + completedLocalAdapter, + localEnsureCompleted, + ) => { + completedLocalMarkers.push( + completedLocalAdapter !== undefined && localEnsureCompleted === true, + ) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `failed-local-index-bootstrap`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { name: `startup-title` }) + + try { + await collection.preload() + expect(completedLocalMarkers).toEqual([false]) + } finally { + await collection.cleanup() + warn.mockRestore() + } + }) + + it(`releases the hydration scope before invoking a coordinator that uses its own adapter`, async () => { + const adapter = createRecordingAdapter() + const coordinatorEntered = createDeferred() + const breakSchedulerCycle = createDeferred() + let hydrationScopeActive = false + let coordinatorEnteredDuringHydration: boolean | undefined + + const publicEnsureIndex = adapter.ensureIndex.bind(adapter) + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: publicEnsureIndex, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + if (hydrationScopeActive) { + // A public core-adapter call queues behind the active hydrate. The + // hydrate cannot release until this coordinator call returns. + await breakSchedulerCycle.promise + } + await publicEnsureIndex(...args) + } + + const coordinator = createCoordinatorHarness() + coordinator.requestEnsurePersistedIndex = async ( + collectionId, + signature, + spec, + ) => { + coordinatorEnteredDuringHydration = hydrationScopeActive + coordinatorEntered.resolve() + // Deliberately ignore the optional scoped adapter, as existing public + // coordinator implementations are allowed to do. + await adapter.ensureIndex(collectionId, signature, spec) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `custom-coordinator-hydration-scope`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { + name: `startup-title`, + }) + const preload = Promise.resolve(collection.preload()) + void preload.catch(() => undefined) + + try { + await coordinatorEntered.promise + expect(coordinatorEnteredDuringHydration).toBe(false) + } finally { + breakSchedulerCycle.resolve() + await preload.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`releases crossed follower hydration scopes before leader index RPC work`, async () => { + const bothLocalIndexesEntered = createDeferred() + const bothCoordinatorRequestsEntered = createDeferred() + let localIndexEntries = 0 + let coordinatorEntries = 0 + + const createTabAdapter = () => { + const adapter = createRecordingAdapter() + const breakSchedulerCycle = createDeferred() + let hydrationScopeActive = false + const remoteScopeObservations: Array = [] + const publicEnsureIndex = adapter.ensureIndex.bind(adapter) + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: async (...args) => { + localIndexEntries++ + if (localIndexEntries === 2) bothLocalIndexesEntered.resolve() + await bothLocalIndexesEntered.promise + await publicEnsureIndex(...args) + }, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + remoteScopeObservations.push(hydrationScopeActive) + coordinatorEntries++ + if (coordinatorEntries === 2) bothCoordinatorRequestsEntered.resolve() + if (hydrationScopeActive) await breakSchedulerCycle.promise + await publicEnsureIndex(...args) + } + return { + adapter, + breakSchedulerCycle, + remoteScopeObservations, + } + } + + const tab1 = createTabAdapter() + const tab2 = createTabAdapter() + const coordinator1 = createCoordinatorHarness() + const coordinator2 = createCoordinatorHarness() + coordinator1.isLeader = () => false + coordinator2.isLeader = () => false + coordinator1.requestEnsurePersistedIndex = ( + collectionId, + signature, + spec, + ) => tab2.adapter.ensureIndex(collectionId, signature, spec) + coordinator2.requestEnsurePersistedIndex = ( + collectionId, + signature, + spec, + ) => tab1.adapter.ensureIndex(collectionId, signature, spec) + + const createFollowerCollection = ( + id: string, + persistence: PersistedCollectionPersistence, + ) => { + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence, + }), + ) + collection.createIndex((row) => row.title, { name: `${id}-title` }) + return collection + } + const followerA = createFollowerCollection(`follower-a`, { + adapter: tab1.adapter, + coordinator: coordinator1, + }) + const followerB = createFollowerCollection(`follower-b`, { + adapter: tab2.adapter, + coordinator: coordinator2, + }) + const preloadA = Promise.resolve(followerA.preload()) + const preloadB = Promise.resolve(followerB.preload()) + void preloadA.catch(() => undefined) + void preloadB.catch(() => undefined) + + try { + await bothCoordinatorRequestsEntered.promise + expect({ + tab1: tab1.remoteScopeObservations, + tab2: tab2.remoteScopeObservations, + }).toEqual({ tab1: [false], tab2: [false] }) + } finally { + tab1.breakSchedulerCycle.resolve() + tab2.breakSchedulerCycle.resolve() + await Promise.all([preloadA, preloadB]).catch(() => undefined) + await Promise.all([followerA.cleanup(), followerB.cleanup()]) + } + }) + + it(`keeps generated crossed-leadership index RPC histories outside local hydration scopes`, async () => { + let run = 0 + await fc.assert( + fc.asyncProperty( + fc.record({ + tabCount: fc.integer({ min: 2, max: 4 }), + indexCount: fc.integer({ min: 1, max: 2 }), + direction: fc.constantFrom(-1, 1), + }), + async ({ tabCount, indexCount, direction }) => { + run++ + const allTabsAtLocalIndex = createDeferred() + let tabsAtLocalIndex = 0 + const tabs = Array.from({ length: tabCount }, () => { + const adapter = createRecordingAdapter() + let hydrationScopeActive = false + let localIndexCalls = 0 + const remoteScopeObservations: Array = [] + const publicEnsureIndex = adapter.ensureIndex.bind(adapter) + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: async (...args) => { + localIndexCalls++ + if (localIndexCalls === 1) { + tabsAtLocalIndex++ + if (tabsAtLocalIndex === tabCount) { + allTabsAtLocalIndex.resolve() + } + await allTabsAtLocalIndex.promise + } + await publicEnsureIndex(...args) + }, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + remoteScopeObservations.push(hydrationScopeActive) + await publicEnsureIndex(...args) + } + return { adapter, remoteScopeObservations } + }) + + const collections = tabs.map((tab, index) => { + const remoteIndex = (index + direction + tabCount) % tabCount + const coordinator = createCoordinatorHarness() + coordinator.isLeader = () => false + coordinator.requestEnsurePersistedIndex = ( + collectionId, + signature, + spec, + ) => + tabs[remoteIndex]!.adapter.ensureIndex( + collectionId, + signature, + spec, + ) + const collection = createCollection( + persistedCollectionOptions({ + id: `generated-crossed-${run}-${index}`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter: tab.adapter, coordinator }, + }), + ) + for ( + let indexOrdinal = 0; + indexOrdinal < indexCount; + indexOrdinal++ + ) { + collection.createIndex( + indexOrdinal % 2 === 0 ? (row) => row.title : (row) => row.id, + { name: `idx-${indexOrdinal}` }, + ) + } + return collection + }) + + try { + await Promise.all( + collections.map((collection) => collection.preload()), + ) + for (const tab of tabs) { + expect(tab.remoteScopeObservations).toEqual( + Array.from({ length: indexCount }, () => false), + ) + } + } finally { + await Promise.all( + collections.map((collection) => collection.cleanup()), + ) + } + }, + ), + { seed: 1868, numRuns: 8, endOnFailure: true }, + ) + }) + it(`queues remote sync writes that arrive during hydration`, async () => { const adapter = createRecordingAdapter([ { @@ -2117,6 +2451,89 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`releases the hydration scope before a gap coordinator uses its own adapter`, async () => { + const adapter = createRecordingAdapter() + const coordinator = createCoordinatorHarness() + const coordinatorEntered = createDeferred() + const breakSchedulerCycle = createDeferred() + let hydrationScopeActive = false + let coordinatorEnteredDuringHydration: boolean | undefined + + const pullSince = async () => ({ + latestRowVersion: 0, + requiresFullReload: false as const, + deltas: [], + }) + const publicAdapter = adapter as RecordingAdapter & { + pullSince: typeof pullSince + } + const scopedAdapter = { + ...adapter, + pullSince, + } + publicAdapter.pullSince = async () => { + if (hydrationScopeActive) await breakSchedulerCycle.promise + return pullSince() + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + coordinator.pullSince = async (_collectionId, _fromRowVersion) => { + coordinatorEnteredDuringHydration = hydrationScopeActive + coordinatorEntered.resolve() + const result = await publicAdapter.pullSince() + return { + type: `rpc:pullSince:res`, + rpcId: `legacy-gap-coordinator`, + ok: true, + latestTerm: 1, + latestSeq: 2, + latestRowVersion: result.latestRowVersion, + requiresFullReload: result.requiresFullReload, + deltas: result.deltas, + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + await collection.preload() + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `tx-gap-legacy-coordinator`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + + try { + await coordinatorEntered.promise + expect(coordinatorEnteredDuringHydration).toBe(false) + } finally { + breakSchedulerCycle.resolve() + await flushAsyncWork() + await collection.cleanup() + } + }) + // Focused invalidation-reload refinements. Whether recovery follows a // sequence gap or a contiguous committed notification, metadata and rows // must be read inside one hydration scope. The adapter schedules v2 only diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts index 65ef56c241..9062ba7f6e 100644 --- a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -13,7 +13,9 @@ * admitted; after release, both operations must finish. The three legal driver * forms are direct, transparently wrapped, and direct with a function-valued * key. This relation records admissions without copying the production - * scheduler or invoking the opaque key. + * scheduler or invoking the opaque key. Promise-only discovery is a fourth + * form: the first operation reveals the shared identity and its complete + * hydration scope becomes the scheduler's already-running unit. * * Production boundary and checkpoint: both operations use * `createSQLiteCorePersistenceAdapter`; the hydration operation enters through @@ -22,12 +24,15 @@ * wrapper identity or treating a function key as a getter synchronously admits * the second query and fails the exact admission assertion. * - * Known omissions: this focused contract test does not establish K=1 lane - * fairness, SQL result correctness, eventual progress under arbitrary I/O, or - * cross-process coordination. Those belong to the shared-driver oracle and - * provider refinements. + * Known omissions: promise-only discovery coordinates adapters over the same + * wrapper identity. Distinct unbranded wrappers must forward the shared key + * before adapter construction. This focused contract test does not establish + * K=1 lane fairness, SQL result correctness, eventual progress under arbitrary + * I/O, or cross-process coordination. Those belong to the shared-driver oracle + * and provider refinements. */ import { describe, expect, it } from 'vitest' +import fc from 'fast-check' import { SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, createSQLiteCorePersistenceAdapter, @@ -92,6 +97,95 @@ class FirstQueryGatedDriver implements SQLiteDriver { } } +class PromiseBrandedFirstQueryGatedDriver implements SQLiteDriver { + readonly admissions: Array = [] + readonly firstQueryEntered = createDeferred() + private readonly firstQueryGate = createDeferred() + private holdFirstQuery = true + + constructor(private readonly schedulingKey: object = {}) {} + + exec(): Promise { + this.admissions.push(`exec`) + return this.brand(Promise.resolve()) + } + + query(): Promise> { + this.admissions.push(`query`) + const result = (async () => { + if (this.holdFirstQuery) { + this.holdFirstQuery = false + this.firstQueryEntered.resolve() + await this.firstQueryGate.promise + } + return [] as ReadonlyArray + })() + return this.brand(result) + } + + run(): Promise { + this.admissions.push(`run`) + return this.brand(Promise.resolve()) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.admissions.push(`transaction`) + return this.brand(fn(this)) + } + + releaseFirstQuery(): void { + this.firstQueryGate.resolve() + } + + private brand(promise: Promise): Promise { + Object.defineProperty( + promise, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { value: this.schedulingKey }, + ) + return promise + } +} + +class UnbrandedPromiseLookupDriver implements SQLiteDriver { + schedulingKeyLookups = 0 + + exec(): Promise { + return this.unbranded(Promise.resolve()) + } + + query(): Promise> { + return this.unbranded(Promise.resolve([] as ReadonlyArray)) + } + + run(): Promise { + return this.unbranded(Promise.resolve()) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.unbranded(fn(this)) + } + + private unbranded(promise: Promise): Promise { + Object.defineProperty( + promise, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { + configurable: true, + get: () => { + this.schedulingKeyLookups++ + return undefined + }, + }, + ) + return promise + } +} + function createTransparentWrapper(driver: SQLiteDriver): SQLiteDriver { return forwardSQLiteDriverSharedLogicalScheduling(driver, { exec: (sql) => driver.exec(sql), @@ -148,4 +242,72 @@ describe(`shared logical scheduling`, () => { expect(underlying.admissions.length).toBeGreaterThan(1) }, ) + + it(`adopts a late-discovered promise identity before admitting peer work`, async () => { + const driver = new PromiseBrandedFirstQueryGatedDriver() + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const regularAdapter = createSQLiteCorePersistenceAdapter({ driver }) + + const hydrate = hydrateAdapter.runInHydrationScope!(async (scoped) => { + await scoped.loadCollectionMetadata!(`hydrate`) + }) + await driver.firstQueryEntered.promise + + const regular = regularAdapter.loadCollectionMetadata!(`regular`) + + expect(driver.admissions).toEqual([`query`]) + + driver.releaseFirstQuery() + await Promise.all([hydrate, regular]) + expect(driver.admissions).toEqual([`query`, `query`]) + }) + + it(`keeps generated promise-discovered hydrate units non-preemptible`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + hydrateQueries: fc.integer({ min: 1, max: 4 }), + peerQueries: fc.integer({ min: 1, max: 3 }), + }), + async ({ hydrateQueries, peerQueries }) => { + const driver = new PromiseBrandedFirstQueryGatedDriver() + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const peerAdapters = Array.from({ length: peerQueries }, () => + createSQLiteCorePersistenceAdapter({ driver }), + ) + + const hydrate = hydrateAdapter.runInHydrationScope!( + async (scoped) => { + for (let index = 0; index < hydrateQueries; index++) { + await scoped.loadCollectionMetadata!(`hydrate-${index}`) + } + }, + ) + await driver.firstQueryEntered.promise + + const peers = peerAdapters.map((adapter, index) => + adapter.loadCollectionMetadata!(`peer-${index}`), + ) + expect(driver.admissions).toEqual([`query`]) + + driver.releaseFirstQuery() + await Promise.all([hydrate, ...peers]) + expect(driver.admissions).toEqual( + Array.from({ length: hydrateQueries + peerQueries }, () => `query`), + ) + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + }) + + it(`stops probing after the first returned promise lacks scheduling support`, async () => { + const driver = new UnbrandedPromiseLookupDriver() + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + + await adapter.loadCollectionMetadata!(`first`) + await adapter.loadCollectionMetadata!(`second`) + + expect(driver.schedulingKeyLookups).toBe(1) + }) }) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index de10c93e6c..43eec12757 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -61,7 +61,7 @@ function interpolateSql(sql: string, params: ReadonlyArray): string { return renderedSql } -class SqliteCliDriver implements SQLiteDriver { +export class SqliteCliDriver implements SQLiteDriver { private readonly transactionDbPath = new AsyncLocalStorage() private queue: Promise = Promise.resolve() diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index efd242dfdc..6cf0f7f06c 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -82,6 +82,11 @@ type CollectionState = { subscribers: Set<(message: ProtocolEnvelope) => void> } +type AppliedEnvelope = { + appliedAt: number + response: ApplyLocalMutationsResponse +} + // Adapter with pullSince support type AdapterWithPullSince = PersistenceAdapter & { pullSince?: ( @@ -123,10 +128,13 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private readonly nodeId = safeRandomUUID() private readonly dbName: string private adapter: AdapterWithPullSince | null + private readonly collectionAdapters = new Map() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopeIds = new Map() + private readonly disposedPromise: Promise + private rejectDisposed: ((error: Error) => void) | null = null private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -134,18 +142,27 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): AdapterWithPullSince { + let adapter = this.collectionAdapters.get(collectionId) + if (!adapter && this.adapter) { + adapter = this.adapter + this.collectionAdapters.set(collectionId, adapter) + } + if (!adapter) { throw new Error( `ElectronCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: ElectronCollectionCoordinatorOptions) { this.dbName = options.dbName this.adapter = options.adapter ?? null + this.disposedPromise = new Promise((_resolve, reject) => { + this.rejectDisposed = reject + }) + void this.disposedPromise.catch(() => undefined) this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) @@ -161,6 +178,14 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.adapter = adapter } + /** Register the adapter that owns leader-side work for one collection. */ + setAdapterForCollection( + collectionId: string, + adapter: AdapterWithPullSince, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + // ----------------------------------------------------------------------- // PersistedCollectionCoordinator interface // ----------------------------------------------------------------------- @@ -223,10 +248,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin signature: string, spec: PersistedIndexSpec, scopedAdapter?: HydrationPersistenceAdapter, + localEnsureCompleted = false, ): Promise { if (this.isLeader(collectionId)) { - // A scoped adapter is a leader-local capability and never crosses RPC. - await (scopedAdapter ?? this.requireAdapter()).ensureIndex( + if (localEnsureCompleted) return + await (scopedAdapter ?? this.requireAdapter(collectionId)).ensureIndex( collectionId, signature, spec, @@ -304,7 +330,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin // ----------------------------------------------------------------------- dispose(): void { + if (this.disposed) return this.disposed = true + const disposedError = new Error(`coordinator disposed`) + this.rejectDisposed?.(disposedError) + this.rejectDisposed = null for (const [collectionId, state] of this.collections) { this.releaseLeadership(collectionId, state) @@ -312,12 +342,13 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin for (const [, pending] of this.pendingRPCs) { clearTimeout(pending.timer) - pending.reject(new Error(`coordinator disposed`)) + pending.reject(disposedError) } this.pendingRPCs.clear() this.channel.close() this.collections.clear() + this.collectionAdapters.clear() } // ----------------------------------------------------------------------- @@ -361,7 +392,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -499,16 +530,22 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin collectionId: string, request: RPCRequest, ): Promise { + if (this.isDisposed()) throw new Error(`coordinator disposed`) let lastError: Error | undefined for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { - await sleep(RPC_RETRY_DELAY_MS * attempt) + await Promise.race([ + sleep(RPC_RETRY_DELAY_MS * attempt), + this.disposedPromise, + ]) } + if (this.isDisposed()) throw new Error(`coordinator disposed`) try { return await this.sendRPCOnce(collectionId, request) } catch (error) { + if (this.isDisposed()) throw error lastError = error instanceof Error ? error : new Error(String(error)) } } @@ -623,7 +660,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -646,14 +683,9 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }, ): Promise { // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, - } + const appliedEnvelope = this.appliedEnvelopeIds.get(request.envelopeId) + if (appliedEnvelope) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } } const state = this.collections.get(collectionId) @@ -689,11 +721,25 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), ) - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + + // Retain the completed result so transport retries observe the same + // durable outcome without applying the envelope again. + this.appliedEnvelopeIds.set(request.envelopeId, { + appliedAt: Date.now(), + response, + }) this.pruneAppliedEnvelopeIds() // Broadcast tx:committed to all tabs @@ -728,15 +774,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin subscriber(txCommitted) } - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), - } + return response } private async handlePullSince( @@ -750,7 +788,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ): Promise { const state = this.collections.get(collectionId) - const adapter = scopedAdapter ?? this.requireAdapter() + const adapter = scopedAdapter ?? this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -825,8 +863,8 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { + for (const [id, applied] of this.appliedEnvelopeIds) { + if (applied.appliedAt < cutoff) { this.appliedEnvelopeIds.delete(id) } } diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index 2ac7203000..351ef42fe5 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -360,32 +360,36 @@ export function createElectronSQLitePersistence( }) adapterCache.set(cacheKey, adapter) - // Wire the adapter into the coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, getStreamPosition, etc.) - if (coordinator instanceof ElectronCollectionCoordinator) { - coordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator, - }) + collectionId?: string, + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if ( + collectionId !== undefined && + coordinator instanceof ElectronCollectionCoordinator + ) { + coordinator.setAdapterForCollection(collectionId, adapter) + } + return { adapter, coordinator } + } const defaultPersistence = createCollectionPersistence( `sync-absent`, undefined, ) + if (coordinator instanceof ElectronCollectionCoordinator) { + coordinator.setAdapter(defaultPersistence.adapter) + } return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(mode, schemaVersion, collectionId), // Backward compatible fallback for older callers. resolvePersistenceForMode: (mode) => createCollectionPersistence(mode, undefined), diff --git a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts new file mode 100644 index 0000000000..d67ff36701 --- /dev/null +++ b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts @@ -0,0 +1,377 @@ +/** + * Browser and Electron coordinators implement the same cross-window protocol. + * The generated Browser law owns the history dimensions; these shrink + * witnesses keep the Electron implementation on the same adapter-routing, + * retry-result, and disposal boundaries. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ElectronCollectionCoordinator } from '../src/electron-coordinator' +import type { ElectronCollectionCoordinatorOptions } from '../src/electron-coordinator' +import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' + +type MessageHandler = (event: { data: unknown }) => void +const channels = new Map>() +let dropNextMessageWhen: ((data: unknown) => boolean) | undefined +let observePostedMessage: ((data: unknown) => void) | undefined + +class MockBroadcastChannel { + onmessage: MessageHandler | null = null + + constructor(readonly name: string) { + const peers = channels.get(name) ?? new Set() + peers.add(this) + channels.set(name, peers) + } + + postMessage(data: unknown): void { + observePostedMessage?.(data) + if (dropNextMessageWhen?.(data)) { + dropNextMessageWhen = undefined + return + } + for (const peer of channels.get(this.name) ?? []) { + if (peer !== this && peer.onmessage) { + const handler = peer.onmessage + queueMicrotask(() => handler({ data: structuredClone(data) })) + } + } + } + + close(): void { + channels.get(this.name)?.delete(this) + } +} + +type LockCallback = (lock: { name: string }) => Promise +type QueuedLock = { + callback: LockCallback + signal?: AbortSignal + resolve: (value: unknown) => void + reject: (error: Error) => void +} +const heldLocks = new Set() +const lockQueues = new Map>() + +function grantNextLock(name: string): void { + if (heldLocks.has(name)) return + const next = lockQueues.get(name)?.shift() + if (!next) return + if (next.signal?.aborted) { + grantNextLock(name) + return + } + heldLocks.add(name) + void Promise.resolve(next.callback({ name })).then( + (value) => { + heldLocks.delete(name) + next.resolve(value) + grantNextLock(name) + }, + (error: unknown) => { + heldLocks.delete(name) + next.reject(error instanceof Error ? error : new Error(String(error))) + grantNextLock(name) + }, + ) +} + +const mockLocks = { + request: ( + name: string, + optionsOrCallback: { signal?: AbortSignal } | LockCallback, + maybeCallback?: LockCallback, + ): Promise => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + const signal = + typeof optionsOrCallback === `function` + ? undefined + : optionsOrCallback.signal + return new Promise((resolve, reject) => { + const entry = { callback, signal, resolve, reject } + const queue = lockQueues.get(name) ?? [] + queue.push(entry) + lockQueues.set(name, queue) + signal?.addEventListener(`abort`, () => { + const index = queue.indexOf(entry) + if (index >= 0) { + queue.splice(index, 1) + reject(new DOMException(`Lock request aborted`, `AbortError`)) + } + }) + grantNextLock(name) + }) + }, +} + +type StubAdapter = PersistenceAdapter & { + appliedTxs: Array + pullSince: () => Promise<{ + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array + }> + getStreamPosition: () => Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> +} + +function createStubAdapter(): StubAdapter { + const appliedTxs: Array = [] + return { + appliedTxs, + loadSubset: async () => [], + applyCommittedTx: async (_collectionId, tx) => { + appliedTxs.push(tx.txId) + }, + ensureIndex: async () => {}, + pullSince: async () => ({ + latestRowVersion: 0, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + }), + getStreamPosition: async () => ({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }), + } +} + +function createCoordinator( + adapter: StubAdapter, +): ElectronCollectionCoordinator { + const options: ElectronCollectionCoordinatorOptions = { + dbName: `electron-coordinator-law`, + adapter, + } + return new ElectronCollectionCoordinator(options) +} + +async function waitForLeadership( + coordinator: ElectronCollectionCoordinator, + collectionId: string, +): Promise { + for (let microtask = 0; microtask < 12; microtask++) { + if (coordinator.isLeader(collectionId)) return + await Promise.resolve() + } + expect(coordinator.isLeader(collectionId)).toBe(true) +} + +describe(`ElectronCollectionCoordinator parity`, () => { + beforeEach(() => { + ;(globalThis as Record).BroadcastChannel = + MockBroadcastChannel as unknown + Object.defineProperty(globalThis, `navigator`, { + value: { locks: mockLocks }, + configurable: true, + }) + }) + + afterEach(() => { + dropNextMessageWhen = undefined + observePostedMessage = undefined + channels.clear() + heldLocks.clear() + lockQueues.clear() + vi.useRealTimers() + }) + + it(`keeps follower RPC work on the collection adapter`, async () => { + const todosAdapter = createStubAdapter() + const notesAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + todosAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + todosAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + notesAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const leader = createCoordinator(todosAdapter) + const follower = createCoordinator(notesAdapter) + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(leader, `todos`) + follower.subscribe(`todos`, () => {}) + leader.setAdapter(notesAdapter) + + const postedTypes: Array = [] + observePostedMessage = (data) => { + postedTypes.push( + (data as { payload?: { type?: string } }).payload?.type, + ) + } + const request = follower.requestEnsurePersistedIndex(`todos`, `idx`, { + expressionSql: [`title`], + }) + for (let microtask = 0; microtask < 12; microtask++) { + await Promise.resolve() + } + expect(postedTypes).toContain(`rpc:ensurePersistedIndex:res`) + await request + + expect(todosAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(notesAdapter.ensureIndex).not.toHaveBeenCalled() + } finally { + follower.dispose() + leader.dispose() + } + }) + + it(`does not repeat successful leader-local index creation`, async () => { + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coordinator = createCoordinator(adapter) + try { + coordinator.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(coordinator, `todos`) + const spec = { expressionSql: [`title`] } + + await adapter.ensureIndex(`todos`, `idx-once`, spec) + await coordinator.requestEnsurePersistedIndex( + `todos`, + `idx-once`, + spec, + adapter, + true, + ) + + expect(adapter.ensureIndex).toHaveBeenCalledOnce() + } finally { + coordinator.dispose() + } + }) + + it(`replays a successful mutation result after response loss`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstSuccessDropped!: () => void + const firstSuccessDroppedPromise = new Promise((resolve) => { + firstSuccessDropped = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(leader, `todos`) + follower.subscribe(`todos`, () => {}) + dropNextMessageWhen = (data) => { + const payload = (data as { payload?: { type?: string; ok?: boolean } }) + .payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + firstSuccessDropped() + return true + } + return false + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-electron`, + type: `insert`, + key: `1`, + value: { id: `1` }, + }, + ]) + await firstSuccessDroppedPromise + await vi.advanceTimersByTimeAsync(10_200) + + await expect(responsePromise).resolves.toMatchObject({ + ok: true, + acceptedMutationIds: [`mut-electron`], + }) + expect(adapter.appliedTxs).toHaveLength(1) + } finally { + follower.dispose() + leader.dispose() + } + }) + + it(`settles pending work at dispose without retrying`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstRequestPosted!: () => void + const firstRequestPostedPromise = new Promise((resolve) => { + firstRequestPosted = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + let requestPosts = 0 + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(leader, `todos`) + follower.subscribe(`todos`, () => {}) + observePostedMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + `rpc:pullSince:req` + ) { + requestPosts++ + firstRequestPosted() + } + } + dropNextMessageWhen = (data) => + (data as { payload?: { type?: string } }).payload?.type === + `rpc:pullSince:req` + + const settled = follower.pullSince(`todos`, 0).then( + () => `resolved`, + () => `rejected`, + ) + await firstRequestPostedPromise + follower.dispose() + + await expect(settled).resolves.toBe(`rejected`) + await vi.advanceTimersByTimeAsync(21_000) + expect(requestPosts).toBe(1) + } finally { + follower.dispose() + leader.dispose() + } + }) +}) From 0ded4c67ec734cb5aa03e456e95cfdc4f29f5237 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 11:55:46 +0100 Subject: [PATCH 11/14] test(persistence): complete gap recovery fixture --- packages/db-sqlite-persistence-core/tests/persisted.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 7171003731..062072abbe 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -2462,6 +2462,8 @@ describe(`persistedCollectionOptions`, () => { const pullSince = async () => ({ latestRowVersion: 0, requiresFullReload: false as const, + changedKeys: [], + deletedKeys: [], deltas: [], }) const publicAdapter = adapter as RecordingAdapter & { @@ -2495,6 +2497,8 @@ describe(`persistedCollectionOptions`, () => { latestSeq: 2, latestRowVersion: result.latestRowVersion, requiresFullReload: result.requiresFullReload, + changedKeys: result.changedKeys, + deletedKeys: result.deletedKeys, deltas: result.deltas, } } From 06024edd17c47cf584152418b0834e8408b6bb27 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 14:22:31 +0100 Subject: [PATCH 12/14] fix(sqlite-persistence): release coordinator retention state --- .../src/browser-coordinator.ts | 82 +++- .../tests/browser-coordinator.test.ts | 360 +++++++++++++++++- .../src/electron-coordinator.ts | 82 +++- .../tests/electron-coordinator.test.ts | 71 ++++ 4 files changed, 580 insertions(+), 15 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 9ae2fc9d2f..d54d25f1ee 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -21,6 +21,7 @@ const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 +const APPLIED_ENVELOPE_RETENTION_MS = 60_000 // --------------------------------------------------------------------------- // Internal types @@ -83,6 +84,7 @@ type CollectionState = { } type AppliedEnvelope = { + collectionId: string appliedAt: number response: ApplyLocalMutationsResponse } @@ -133,6 +135,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private readonly collections = new Map() private readonly pendingRPCs = new Map() private readonly appliedEnvelopeIds = new Map() + private appliedEnvelopePruneTimer: ReturnType | null = null private readonly disposedPromise: Promise private rejectDisposed: ((error: Error) => void) | null = null private disposed = false @@ -202,6 +205,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.subscribers.add(onMessage) return () => { state.subscribers.delete(onMessage) + this.releaseCollectionIfUnused(collectionId, state) } } @@ -346,6 +350,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } this.pendingRPCs.clear() + if (this.appliedEnvelopePruneTimer !== null) { + clearTimeout(this.appliedEnvelopePruneTimer) + this.appliedEnvelopePruneTimer = null + } + this.appliedEnvelopeIds.clear() + this.channel.close() this.collections.clear() this.collectionAdapters.clear() @@ -388,7 +398,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina lockName, { signal: abortController.signal }, async () => { - if (this.isDisposed()) return + if ( + this.isDisposed() || + this.collections.get(collectionId) !== state + ) { + return + } try { // Restore stream position from DB before claiming leadership @@ -400,6 +415,13 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.latestRowVersion = pos.latestRowVersion } + if ( + this.isDisposed() || + this.collections.get(collectionId) !== state + ) { + return + } + state.latestTerm++ state.isLeader = true @@ -437,7 +459,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } // Re-acquire if not disposed (leadership was released by another means) - if (!this.isDisposed()) { + if (!this.isDisposed() && this.collections.get(collectionId) === state) { void this.acquireLeadership(collectionId, state) } } @@ -457,6 +479,23 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.isLeader = false } + private releaseCollectionIfUnused( + collectionId: string, + state: CollectionState, + ): void { + if ( + state.subscribers.size > 0 || + this.collections.get(collectionId) !== state + ) { + return + } + + this.releaseLeadership(collectionId, state) + this.collections.delete(collectionId) + this.collectionAdapters.delete(collectionId) + this.deleteAppliedEnvelopeIdsForCollection(collectionId) + } + private emitHeartbeat(collectionId: string, state: CollectionState): void { const envelope: ProtocolEnvelope = { v: 1, @@ -683,6 +722,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }, ): Promise { // Dedupe by envelopeId + this.pruneAppliedEnvelopeIds() const appliedEnvelope = this.appliedEnvelopeIds.get(request.envelopeId) if (appliedEnvelope) { return { ...appliedEnvelope.response, rpcId: request.rpcId } @@ -737,10 +777,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // Retain the completed result so transport retries observe the same // durable outcome without applying the envelope again. this.appliedEnvelopeIds.set(request.envelopeId, { + collectionId, appliedAt: Date.now(), response, }) this.pruneAppliedEnvelopeIds() + this.scheduleAppliedEnvelopePrune() // Broadcast tx:committed to all tabs const changedRows = request.mutations @@ -861,13 +903,43 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // ----------------------------------------------------------------------- private pruneAppliedEnvelopeIds(): void { - // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - 60_000 + const cutoff = Date.now() - APPLIED_ENVELOPE_RETENTION_MS + for (const [id, applied] of this.appliedEnvelopeIds) { + if (applied.appliedAt <= cutoff) { + this.appliedEnvelopeIds.delete(id) + } + } + } + + private deleteAppliedEnvelopeIdsForCollection(collectionId: string): void { for (const [id, applied] of this.appliedEnvelopeIds) { - if (applied.appliedAt < cutoff) { + if (applied.collectionId === collectionId) { this.appliedEnvelopeIds.delete(id) } } + this.scheduleAppliedEnvelopePrune() + } + + private scheduleAppliedEnvelopePrune(): void { + if (this.appliedEnvelopePruneTimer !== null) { + clearTimeout(this.appliedEnvelopePruneTimer) + this.appliedEnvelopePruneTimer = null + } + if (this.disposed || this.appliedEnvelopeIds.size === 0) return + + let earliestAppliedAt = Number.POSITIVE_INFINITY + for (const applied of this.appliedEnvelopeIds.values()) { + earliestAppliedAt = Math.min(earliestAppliedAt, applied.appliedAt) + } + const delay = Math.max( + 0, + earliestAppliedAt + APPLIED_ENVELOPE_RETENTION_MS - Date.now(), + ) + this.appliedEnvelopePruneTimer = setTimeout(() => { + this.appliedEnvelopePruneTimer = null + this.pruneAppliedEnvelopeIds() + this.scheduleAppliedEnvelopePrune() + }, delay) } } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 3f333997eb..2fed3f18c9 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -15,16 +15,17 @@ * exact adapter calls, durable transaction count, returned result, promise * settlement, and request posts. * - * Known omissions: these laws do not establish native browser scheduling, - * concurrent delivery of one in-flight envelope, or behavior after the dedup - * retention window. Electron keeps focused parity witnesses for the same - * protocol boundaries. + * Known omissions: these laws do not establish native browser scheduling or + * concurrent delivery of one in-flight envelope. Electron keeps focused + * parity witnesses for the same protocol boundaries. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import fc from 'fast-check' +import { BasicIndex, createCollection } from '@tanstack/db' +import { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' -import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' +import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' // --------------------------------------------------------------------------- // BroadcastChannel mock @@ -260,6 +261,31 @@ async function flush(ms: number = 10): Promise { await new Promise((resolve) => setTimeout(resolve, ms)) } +type Deferred = { + promise: Promise + resolve: () => void +} + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +type CoordinatorInspection = { + collectionAdapters: Map + collections: Map + appliedEnvelopeIds: Map +} + +function inspectCoordinator( + coordinator: BrowserCollectionCoordinator, +): CoordinatorInspection { + return coordinator as unknown as CoordinatorInspection +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -755,6 +781,175 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`keeps production collection bootstrap index work exact-once on the leader`, async () => { + const adapter = createStubAdapter() + adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + const coordinator = createCoordinator(adapter) + const leaderReady = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.collectionId === `leader-bootstrap-index` && + envelope.payload?.type === `leader:heartbeat` + ) { + leaderReady.resolve() + } + } + const releaseLeader = coordinator.subscribe( + `leader-bootstrap-index`, + () => {}, + ) + await leaderReady.promise + observePostedMessage = undefined + + const collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id: `leader-bootstrap-index`, + getKey: (row) => row.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { name: `title` }) + + try { + await collection.preload() + expect(adapter.ensureIndex).toHaveBeenCalledOnce() + } finally { + await collection.cleanup() + releaseLeader() + coordinator.dispose() + } + }) + + it(`reaches crossed production RPC leadership only after both hydration scopes exit`, async () => { + const bothLocalIndexesEntered = createDeferred() + const bothLeaderRPCsEntered = createDeferred() + const releaseSchedulerCycle = createDeferred() + let localIndexEntries = 0 + let leaderRPCEntries = 0 + + const createTab = () => { + const adapter = createStubAdapter() + const baseEnsureIndex = adapter.ensureIndex.bind(adapter) + let hydrationScopeActive = false + const leaderRPCScopeObservations: Array = [] + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: async (...args) => { + localIndexEntries++ + if (localIndexEntries === 2) bothLocalIndexesEntered.resolve() + await bothLocalIndexesEntered.promise + await baseEnsureIndex(...args) + }, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + leaderRPCScopeObservations.push(hydrationScopeActive) + leaderRPCEntries++ + if (leaderRPCEntries === 2) bothLeaderRPCsEntered.resolve() + if (hydrationScopeActive) await releaseSchedulerCycle.promise + await baseEnsureIndex(...args) + } + return { adapter, leaderRPCScopeObservations } + } + + const tab1 = createTab() + const tab2 = createTab() + const coordinator1 = createCoordinator(tab1.adapter) + const coordinator2 = createCoordinator(tab2.adapter) + const leaderCollections = new Set() + const bothLeadersReady = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.payload?.type === `leader:heartbeat` && + (envelope.collectionId === `crossed-a` || + envelope.collectionId === `crossed-b`) + ) { + leaderCollections.add(envelope.collectionId) + if (leaderCollections.size === 2) bothLeadersReady.resolve() + } + } + coordinator1.setAdapterForCollection(`crossed-b`, tab1.adapter) + coordinator2.setAdapterForCollection(`crossed-a`, tab2.adapter) + const releaseLeaderB = coordinator1.subscribe(`crossed-b`, () => {}) + const releaseLeaderA = coordinator2.subscribe(`crossed-a`, () => {}) + await bothLeadersReady.promise + observePostedMessage = undefined + + const createFollowerCollection = ( + id: string, + adapter: PersistenceAdapter, + coordinator: BrowserCollectionCoordinator, + ) => { + const collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id, + getKey: (row) => row.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { name: `${id}-title` }) + return collection + } + const followerA = createFollowerCollection( + `crossed-a`, + tab1.adapter, + coordinator1, + ) + const followerB = createFollowerCollection( + `crossed-b`, + tab2.adapter, + coordinator2, + ) + const preloadA = Promise.resolve(followerA.preload()) + const preloadB = Promise.resolve(followerB.preload()) + void preloadA.catch(() => undefined) + void preloadB.catch(() => undefined) + + try { + await bothLeaderRPCsEntered.promise + expect({ + tab1: tab1.leaderRPCScopeObservations, + tab2: tab2.leaderRPCScopeObservations, + }).toEqual({ tab1: [false], tab2: [false] }) + await Promise.all([preloadA, preloadB]) + } finally { + releaseSchedulerCycle.resolve() + await Promise.all([preloadA, preloadB]).catch(() => undefined) + await Promise.all([followerA.cleanup(), followerB.cleanup()]) + releaseLeaderA() + releaseLeaderB() + coordinator2.dispose() + coordinator1.dispose() + } + }) + it(`uses a supplied leader-local adapter when local work is not complete`, async () => { const registeredAdapter = createStubAdapter() const scopedAdapter = createStubAdapter() @@ -959,6 +1154,161 @@ describe(`BrowserCollectionCoordinator`, () => { }) }) + describe(`collection and retry-result retention`, () => { + it(`releases generated collection-owned state after the last subscriber`, async () => { + let run = 0 + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 1, max: 4 }).chain((collectionCount) => + fc.record({ + collectionCount: fc.constant(collectionCount), + releaseOrder: fc.shuffledSubarray( + Array.from({ length: collectionCount }, (_, index) => index), + { minLength: collectionCount, maxLength: collectionCount }, + ), + }), + ), + async ({ collectionCount, releaseOrder }) => { + run++ + const coordinator = createCoordinator() + const collectionIds = Array.from( + { length: collectionCount }, + (_, index) => `lifecycle-${run}-${index}`, + ) + const readyCollections = new Set() + const allReady = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.payload?.type === `leader:heartbeat` && + envelope.collectionId && + collectionIds.includes(envelope.collectionId) + ) { + readyCollections.add(envelope.collectionId) + if (readyCollections.size === collectionCount) { + allReady.resolve() + } + } + } + for (const collectionId of collectionIds) { + coordinator.setAdapterForCollection( + collectionId, + createStubAdapter(), + ) + } + const releases = collectionIds.map((collectionId) => + coordinator.subscribe(collectionId, () => {}), + ) + + try { + await allReady.promise + observePostedMessage = undefined + expect( + inspectCoordinator(coordinator).collectionAdapters.size, + ).toBe(collectionCount) + + let remaining = collectionCount + for (const releasedIndex of releaseOrder) { + releases[releasedIndex]!() + remaining-- + expect({ + adapters: + inspectCoordinator(coordinator).collectionAdapters.size, + collections: inspectCoordinator(coordinator).collections.size, + }).toEqual({ + adapters: remaining, + collections: remaining, + }) + } + } finally { + observePostedMessage = undefined + for (const release of releases) release() + coordinator.dispose() + } + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + }) + + it(`bounds generated retry-result histories by retention and collection lifecycle`, async () => { + vi.useFakeTimers() + let run = 0 + try { + await fc.assert( + fc.asyncProperty( + fc.record({ + mutationCount: fc.integer({ min: 1, max: 4 }), + terminal: fc.constantFrom(`retention`, `unsubscribe`, `dispose`), + }), + async ({ mutationCount, terminal }) => { + run++ + vi.setSystemTime(0) + const adapter = createStubAdapter() + const coordinator = createCoordinator(adapter) + const collectionId = `envelope-retention-${run}` + const ready = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.collectionId === collectionId && + envelope.payload?.type === `leader:heartbeat` + ) { + ready.resolve() + } + } + const release = coordinator.subscribe(collectionId, () => {}) + + try { + await ready.promise + observePostedMessage = undefined + for (let index = 0; index < mutationCount; index++) { + await coordinator.requestApplyLocalMutations(collectionId, [ + { + mutationId: `mutation-${run}-${index}`, + type: `insert`, + key: `${index}`, + value: { id: `${index}`, title: `row ${index}` }, + }, + ]) + } + expect( + inspectCoordinator(coordinator).appliedEnvelopeIds.size, + ).toBe(mutationCount) + + if (terminal === `retention`) { + await vi.advanceTimersByTimeAsync(60_000) + } else if (terminal === `unsubscribe`) { + release() + } else { + coordinator.dispose() + } + + expect( + inspectCoordinator(coordinator).appliedEnvelopeIds.size, + ).toBe(0) + } finally { + observePostedMessage = undefined + release() + coordinator.dispose() + vi.clearAllTimers() + } + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + } finally { + vi.useRealTimers() + } + }) + }) + describe(`dispose`, () => { it(`cleans up on dispose`, async () => { const coord = createCoordinator() diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 6cf0f7f06c..6843ad5884 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -21,6 +21,7 @@ const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 +const APPLIED_ENVELOPE_RETENTION_MS = 60_000 // --------------------------------------------------------------------------- // Internal types @@ -83,6 +84,7 @@ type CollectionState = { } type AppliedEnvelope = { + collectionId: string appliedAt: number response: ApplyLocalMutationsResponse } @@ -133,6 +135,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private readonly collections = new Map() private readonly pendingRPCs = new Map() private readonly appliedEnvelopeIds = new Map() + private appliedEnvelopePruneTimer: ReturnType | null = null private readonly disposedPromise: Promise private rejectDisposed: ((error: Error) => void) | null = null private disposed = false @@ -202,6 +205,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.subscribers.add(onMessage) return () => { state.subscribers.delete(onMessage) + this.releaseCollectionIfUnused(collectionId, state) } } @@ -346,6 +350,12 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } this.pendingRPCs.clear() + if (this.appliedEnvelopePruneTimer !== null) { + clearTimeout(this.appliedEnvelopePruneTimer) + this.appliedEnvelopePruneTimer = null + } + this.appliedEnvelopeIds.clear() + this.channel.close() this.collections.clear() this.collectionAdapters.clear() @@ -388,7 +398,12 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin lockName, { signal: abortController.signal }, async () => { - if (this.isDisposed()) return + if ( + this.isDisposed() || + this.collections.get(collectionId) !== state + ) { + return + } try { // Restore stream position from DB before claiming leadership @@ -400,6 +415,13 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.latestRowVersion = pos.latestRowVersion } + if ( + this.isDisposed() || + this.collections.get(collectionId) !== state + ) { + return + } + state.latestTerm++ state.isLeader = true @@ -437,7 +459,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } // Re-acquire if not disposed (leadership was released by another means) - if (!this.isDisposed()) { + if (!this.isDisposed() && this.collections.get(collectionId) === state) { void this.acquireLeadership(collectionId, state) } } @@ -457,6 +479,23 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.isLeader = false } + private releaseCollectionIfUnused( + collectionId: string, + state: CollectionState, + ): void { + if ( + state.subscribers.size > 0 || + this.collections.get(collectionId) !== state + ) { + return + } + + this.releaseLeadership(collectionId, state) + this.collections.delete(collectionId) + this.collectionAdapters.delete(collectionId) + this.deleteAppliedEnvelopeIdsForCollection(collectionId) + } + private emitHeartbeat(collectionId: string, state: CollectionState): void { const envelope: ProtocolEnvelope = { v: 1, @@ -683,6 +722,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }, ): Promise { // Dedupe by envelopeId + this.pruneAppliedEnvelopeIds() const appliedEnvelope = this.appliedEnvelopeIds.get(request.envelopeId) if (appliedEnvelope) { return { ...appliedEnvelope.response, rpcId: request.rpcId } @@ -737,10 +777,12 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin // Retain the completed result so transport retries observe the same // durable outcome without applying the envelope again. this.appliedEnvelopeIds.set(request.envelopeId, { + collectionId, appliedAt: Date.now(), response, }) this.pruneAppliedEnvelopeIds() + this.scheduleAppliedEnvelopePrune() // Broadcast tx:committed to all tabs const changedRows = request.mutations @@ -861,13 +903,43 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin // ----------------------------------------------------------------------- private pruneAppliedEnvelopeIds(): void { - // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - 60_000 + const cutoff = Date.now() - APPLIED_ENVELOPE_RETENTION_MS + for (const [id, applied] of this.appliedEnvelopeIds) { + if (applied.appliedAt <= cutoff) { + this.appliedEnvelopeIds.delete(id) + } + } + } + + private deleteAppliedEnvelopeIdsForCollection(collectionId: string): void { for (const [id, applied] of this.appliedEnvelopeIds) { - if (applied.appliedAt < cutoff) { + if (applied.collectionId === collectionId) { this.appliedEnvelopeIds.delete(id) } } + this.scheduleAppliedEnvelopePrune() + } + + private scheduleAppliedEnvelopePrune(): void { + if (this.appliedEnvelopePruneTimer !== null) { + clearTimeout(this.appliedEnvelopePruneTimer) + this.appliedEnvelopePruneTimer = null + } + if (this.disposed || this.appliedEnvelopeIds.size === 0) return + + let earliestAppliedAt = Number.POSITIVE_INFINITY + for (const applied of this.appliedEnvelopeIds.values()) { + earliestAppliedAt = Math.min(earliestAppliedAt, applied.appliedAt) + } + const delay = Math.max( + 0, + earliestAppliedAt + APPLIED_ENVELOPE_RETENTION_MS - Date.now(), + ) + this.appliedEnvelopePruneTimer = setTimeout(() => { + this.appliedEnvelopePruneTimer = null + this.pruneAppliedEnvelopeIds() + this.scheduleAppliedEnvelopePrune() + }, delay) } } diff --git a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts index d67ff36701..168ecba1b8 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts @@ -165,6 +165,18 @@ async function waitForLeadership( expect(coordinator.isLeader(collectionId)).toBe(true) } +type CoordinatorInspection = { + collectionAdapters: Map + collections: Map + appliedEnvelopeIds: Map +} + +function inspectCoordinator( + coordinator: ElectronCollectionCoordinator, +): CoordinatorInspection { + return coordinator as unknown as CoordinatorInspection +} + describe(`ElectronCollectionCoordinator parity`, () => { beforeEach(() => { ;(globalThis as Record).BroadcastChannel = @@ -322,6 +334,65 @@ describe(`ElectronCollectionCoordinator parity`, () => { } }) + it(`releases collection-owned state and retry results with the last subscriber`, async () => { + const adapter = createStubAdapter() + const coordinator = createCoordinator(adapter) + const release = coordinator.subscribe(`todos`, () => {}) + try { + await waitForLeadership(coordinator, `todos`) + await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-release`, + type: `insert`, + key: `1`, + value: { id: `1` }, + }, + ]) + expect({ + adapters: inspectCoordinator(coordinator).collectionAdapters.size, + collections: inspectCoordinator(coordinator).collections.size, + envelopes: inspectCoordinator(coordinator).appliedEnvelopeIds.size, + }).toEqual({ adapters: 1, collections: 1, envelopes: 1 }) + + release() + expect({ + adapters: inspectCoordinator(coordinator).collectionAdapters.size, + collections: inspectCoordinator(coordinator).collections.size, + envelopes: inspectCoordinator(coordinator).appliedEnvelopeIds.size, + }).toEqual({ adapters: 0, collections: 0, envelopes: 0 }) + } finally { + release() + coordinator.dispose() + } + }) + + it(`expires retry results without later mutation traffic`, async () => { + vi.useFakeTimers() + vi.setSystemTime(0) + const adapter = createStubAdapter() + const coordinator = createCoordinator(adapter) + const release = coordinator.subscribe(`todos`, () => {}) + try { + await waitForLeadership(coordinator, `todos`) + await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-expire`, + type: `insert`, + key: `1`, + value: { id: `1` }, + }, + ]) + expect(inspectCoordinator(coordinator).appliedEnvelopeIds.size).toBe(1) + + await vi.advanceTimersByTimeAsync(60_000) + expect(inspectCoordinator(coordinator).appliedEnvelopeIds.size).toBe(0) + } finally { + release() + coordinator.dispose() + vi.useRealTimers() + } + }) + it(`settles pending work at dispose without retrying`, async () => { vi.useFakeTimers() const adapter = createStubAdapter() From ca57897482e15e844b8acd6c4ef561d7773b70ae Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 15:11:34 +0100 Subject: [PATCH 13/14] test(sqlite-persistence): strengthen fairness oracle campaigns --- docs/contributing/oracle-coverage.md | 2 +- .../shared-driver-fairness-oracle.test.ts | 394 +++++++++++++++--- .../tests/shared-driver-fairness-oracle.ts | 22 +- 3 files changed, 343 insertions(+), 75 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 06eb175c2a..039f032032 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -110,7 +110,7 @@ comment and the current API/architecture contract before extending its model. | Opaque backend pagination | [window oracle](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, exact driver results, controlled page/worker ownership, and diagnostic-cause retention. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [shared-driver fairness](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, exact driver results, K=1 complete-logical-hydrate fairness over one shared Browser SQLite driver, controlled page/worker ownership, and diagnostic-cause retention. The fairness owner runs one identical generated property in retained fixed-seed and seedless-random lanes, or only a requested checked seed+shrink-path replay. Its grammar controls reconstruct the known witness, exercise bounded marginals, reject an invalid storm boundary, and ablate tail order; a named persist-first FIFO wrong answer calibrates the checker. It does not establish elapsed-time latency, unbounded eventuality, multi-process coordination, or a browser matrix; Chromium OPFS separately refines the provider boundary. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, and compiled output bags. Hash collision freedom is not promised. Unsupported composite cursors reject. | diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts index 7fc27c5d67..297c59e576 100644 --- a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts @@ -4,8 +4,14 @@ * persist storm; generated legal histories vary both lane sizes, mutation * width, and tail order. Public rows and logical completion checkpoints are * checked independently of production scheduling, and an executable - * persist-first FIFO mutant proves the checker rejects the original fault. - * Failures replay through TANSTACK_DB_DRIVER_FAIRNESS_SEED/PATH. + * named persist-first FIFO wrong answer proves the checker rejects the original + * fault. Grammar controls reconstruct that witness, exercise the bounded + * marginals, reject a nearby invalid storm, and ablate tail-order variation. + * The identical generated property runs in retained fixed-seed and seedless + * random lanes. Supplying both TANSTACK_DB_DRIVER_FAIRNESS_SEED and + * TANSTACK_DB_DRIVER_FAIRNESS_PATH replaces those lanes with one checked + * seed+shrink-path replay. An axis-removal calibration proves that removing + * tail-order variation loses hydrate-before-later-persist histories. */ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' @@ -26,14 +32,94 @@ import type { SharedDriverFairnessWork, } from './shared-driver-fairness-oracle' -const DEFAULT_SEED = 165_905 -const replaySeed = Number( - process.env.TANSTACK_DB_DRIVER_FAIRNESS_SEED ?? DEFAULT_SEED, +const FIXED_SEED = 165_905 +const GENERATED_RUNS = 12 +const HYDRATE_COUNT_RANGE = { min: 2, max: 7 } as const +const PERSIST_COUNT_RANGE = { min: 3, max: 7 } as const +const MUTATIONS_PER_PERSIST_RANGE = { min: 1, max: 3 } as const + +type FairnessReplayEnvironment = Record + +type FairnessReplayConfig = { + seed: number + path: string +} + +type FairnessPropertyMode = { + label: `fixed-seed` | `seedless-random` | `checked-replay` + seed?: number + path?: string +} + +function readFairnessReplayConfig( + environment: FairnessReplayEnvironment = process.env, +): FairnessReplayConfig | undefined { + const seedText = environment.TANSTACK_DB_DRIVER_FAIRNESS_SEED + const path = environment.TANSTACK_DB_DRIVER_FAIRNESS_PATH + if (seedText === undefined && path === undefined) return undefined + if (seedText === undefined || path === undefined) { + throw new Error( + `TANSTACK_DB_DRIVER_FAIRNESS_SEED and TANSTACK_DB_DRIVER_FAIRNESS_PATH must be supplied together`, + ) + } + + const seed = Number(seedText) + if (seedText.trim() === `` || !Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_DRIVER_FAIRNESS_SEED must be an integer`) + } + if (!/^\d+(?::\d+)*$/.test(path)) { + throw new Error( + `TANSTACK_DB_DRIVER_FAIRNESS_PATH must contain colon-separated nonnegative integers`, + ) + } + return { seed, path } +} + +function createFairnessPropertyModes( + replay: FairnessReplayConfig | undefined, +): ReadonlyArray { + if (replay) { + return [{ label: `checked-replay`, ...replay }] + } + return [ + { label: `fixed-seed`, seed: FIXED_SEED }, + { label: `seedless-random` }, + ] +} + +function readFairnessCalibration( + environment: FairnessReplayEnvironment = process.env, +): SharedDriverFairnessOptions { + const fault = environment.TANSTACK_DB_DRIVER_FAIRNESS_CALIBRATION + if (fault === undefined) return {} + if (fault !== `persist-first-fifo`) { + throw new Error( + `TANSTACK_DB_DRIVER_FAIRNESS_CALIBRATION must be persist-first-fifo`, + ) + } + return { schedulingFault: fault } +} + +const fairnessPropertyModes = createFairnessPropertyModes( + readFairnessReplayConfig(), ) -const replayPath = process.env.TANSTACK_DB_DRIVER_FAIRNESS_PATH +const fairnessCalibration = readFairnessCalibration() type WorkKind = SharedDriverFairnessWork[`kind`] +type GeneratedFairnessHistory = { + hydrateCount: number + persistCount: number + mutationsPerPersist: number + orderedKinds: ReadonlyArray + replayHistory: string +} + +type GeneratedTailEntry = { + kind: WorkKind + token: string +} + function createScenario( id: string, orderedKinds: ReadonlyArray, @@ -162,41 +248,217 @@ function expectFairObservation( expect(observation.cleanupFailures).toEqual([]) } -const generatedHistoryArbitrary = fc - .record({ - hydrateCount: fc.integer({ min: 2, max: 7 }), - persistCount: fc.integer({ min: 2, max: 7 }), - mutationsPerPersist: fc.integer({ min: 1, max: 3 }), +function buildGeneratedHistory( + hydrateCount: number, + persistCount: number, + mutationsPerPersist: number, + generatedTail: ReadonlyArray, +): GeneratedFairnessHistory { + return { + hydrateCount, + persistCount, + mutationsPerPersist, + orderedKinds: [ + `persist` as const, + ...generatedTail.map(({ kind }) => kind), + ], + replayHistory: `p0,${generatedTail.map(({ token }) => token).join(`,`)}`, + } +} + +function createGeneratedHistoryArbitrary( + tailOrderAxis: `varied` | `removed`, +): fc.Arbitrary { + return fc + .record({ + hydrateCount: fc.integer(HYDRATE_COUNT_RANGE), + persistCount: fc.integer(PERSIST_COUNT_RANGE), + mutationsPerPersist: fc.integer(MUTATIONS_PER_PERSIST_RANGE), + }) + .chain(({ hydrateCount, persistCount, mutationsPerPersist }) => { + const tail = [ + ...Array.from({ length: persistCount - 1 }, (_, index) => ({ + kind: `persist` as const, + token: `p${index + 1}`, + })), + ...Array.from({ length: hydrateCount }, (_, index) => ({ + kind: `hydrate` as const, + token: `h${index}`, + })), + ] + const orderedTail = + tailOrderAxis === `varied` + ? fc.shuffledSubarray(tail, { + minLength: tail.length, + maxLength: tail.length, + }) + : fc.constant(tail) + return orderedTail.map((generatedTail) => + buildGeneratedHistory( + hydrateCount, + persistCount, + mutationsPerPersist, + generatedTail, + ), + ) + }) +} + +function reachesHydrateBeforeLaterPersist( + history: GeneratedFairnessHistory, +): boolean { + let hydrateSeen = false + for (const kind of history.orderedKinds.slice(1)) { + if (kind === `hydrate`) hydrateSeen = true + if (kind === `persist` && hydrateSeen) return true + } + return false +} + +const generatedHistoryArbitrary = createGeneratedHistoryArbitrary(`varied`) +const tailOrderAblatedArbitrary = createGeneratedHistoryArbitrary(`removed`) +let generatedPropertyExecutions = 0 + +const generatedFairnessProperty = fc.asyncProperty( + generatedHistoryArbitrary, + async ({ + hydrateCount, + persistCount, + mutationsPerPersist, + orderedKinds, + replayHistory, + }) => { + generatedPropertyExecutions++ + const observationId = + `generated-h${hydrateCount}-p${persistCount}-m${mutationsPerPersist}-` + + replayHistory.replaceAll(`,`, `-`) + await withNodeScenario( + createScenario(observationId, orderedKinds, mutationsPerPersist), + expectFairObservation, + fairnessCalibration, + ) + }, +) + +describe(`shared BrowserWASQLiteDriver fairness oracle`, () => { + it(`selects fixed and seedless lanes by default and only replay when requested`, () => { + expect(createFairnessPropertyModes(readFairnessReplayConfig({}))).toEqual([ + { label: `fixed-seed`, seed: FIXED_SEED }, + { label: `seedless-random` }, + ]) + expect( + createFairnessPropertyModes( + readFairnessReplayConfig({ + TANSTACK_DB_DRIVER_FAIRNESS_SEED: `42`, + TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0:1`, + }), + ), + ).toEqual([{ label: `checked-replay`, seed: 42, path: `0:1` }]) + expect(readFairnessCalibration({})).toEqual({}) + expect( + readFairnessCalibration({ + TANSTACK_DB_DRIVER_FAIRNESS_CALIBRATION: `persist-first-fifo`, + }), + ).toEqual({ schedulingFault: `persist-first-fifo` }) + }) + + it.each([ + { + name: `seed without path`, + environment: { TANSTACK_DB_DRIVER_FAIRNESS_SEED: `42` }, + message: `must be supplied together`, + }, + { + name: `path without seed`, + environment: { TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0` }, + message: `must be supplied together`, + }, + { + name: `non-integer seed`, + environment: { + TANSTACK_DB_DRIVER_FAIRNESS_SEED: `4.2`, + TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0`, + }, + message: `must be an integer`, + }, + { + name: `invalid shrink path`, + environment: { + TANSTACK_DB_DRIVER_FAIRNESS_SEED: `42`, + TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0:-1`, + }, + message: `colon-separated nonnegative integers`, + }, + ])(`rejects incomplete or invalid replay coordinates: $name`, (probe) => { + expect(() => readFairnessReplayConfig(probe.environment)).toThrow( + probe.message, + ) }) - .chain(({ hydrateCount, persistCount, mutationsPerPersist }) => { - const tail = [ - ...Array.from({ length: persistCount - 1 }, (_, index) => ({ - kind: `persist` as const, - token: `p${index + 1}`, - })), - ...Array.from({ length: hydrateCount }, (_, index) => ({ - kind: `hydrate` as const, - token: `h${index}`, - })), + + it(`reconstructs the known persist-first witness, covers range marginals, and rejects a nearby invalid storm`, async () => { + const knownWitness = buildGeneratedHistory(2, 4, 1, [ + { kind: `persist`, token: `p1` }, + { kind: `persist`, token: `p2` }, + { kind: `persist`, token: `p3` }, + { kind: `hydrate`, token: `h0` }, + { kind: `hydrate`, token: `h1` }, + ]) + expect(knownWitness).toEqual({ + hydrateCount: 2, + persistCount: 4, + mutationsPerPersist: 1, + orderedKinds: [ + `persist`, + `persist`, + `persist`, + `persist`, + `hydrate`, + `hydrate`, + ], + replayHistory: `p0,p1,p2,p3,h0,h1`, + }) + + const marginalHistories = [ + buildGeneratedHistory(2, 3, 1, [ + { kind: `persist`, token: `p1` }, + { kind: `persist`, token: `p2` }, + { kind: `hydrate`, token: `h0` }, + { kind: `hydrate`, token: `h1` }, + ]), + buildGeneratedHistory(7, 7, 3, [ + ...Array.from({ length: 6 }, (_, index) => ({ + kind: `persist` as const, + token: `p${index + 1}`, + })), + ...Array.from({ length: 7 }, (_, index) => ({ + kind: `hydrate` as const, + token: `h${index}`, + })), + ]), ] - return fc - .shuffledSubarray(tail, { - minLength: tail.length, - maxLength: tail.length, - }) - .map((shuffledTail) => ({ - hydrateCount, - persistCount, - mutationsPerPersist, - orderedKinds: [ - `persist` as const, - ...shuffledTail.map(({ kind }) => kind), - ], - replayHistory: `p0,${shuffledTail.map(({ token }) => token).join(`,`)}`, - })) + expect( + marginalHistories.map((history) => [ + history.hydrateCount, + history.persistCount, + history.mutationsPerPersist, + ]), + ).toEqual([ + [2, 3, 1], + [7, 7, 3], + ]) + + await expect( + withNodeScenario( + createScenario(`invalid-storm-boundary`, [ + `hydrate`, + `persist`, + `hydrate`, + ]), + expectFairObservation, + ), + ).rejects.toThrow(`must begin with its already-running persist`) }) -describe(`shared BrowserWASQLiteDriver fairness oracle`, () => { it(`reaches cold hydration and preserves independently seeded public rows without queued persists`, async () => { await withNodeScenario( createScenario(`neutral-reach`, [`hydrate`, `hydrate`, `hydrate`]), @@ -217,48 +479,46 @@ describe(`shared BrowserWASQLiteDriver fairness oracle`, () => { await withNodeScenario( createScenario( `fixed-persist-storm`, - [`persist`, `persist`, `persist`, `hydrate`], + [`persist`, `persist`, `persist`, `hydrate`, `hydrate`], 2, ), expectFairObservation, ) }) - it(`bounds persist completions for generated ordered cold-hydrate/persist histories`, async () => { - await fc.assert( - fc.asyncProperty( - generatedHistoryArbitrary, - async ({ - hydrateCount, - persistCount, - mutationsPerPersist, - orderedKinds, - replayHistory, - }) => { - const observationId = - `generated-h${hydrateCount}-p${persistCount}-m${mutationsPerPersist}-` + - replayHistory.replaceAll(`,`, `-`) - await withNodeScenario( - createScenario(observationId, orderedKinds, mutationsPerPersist), - expectFairObservation, - ) - }, - ), - { - seed: replaySeed, - path: replayPath, - numRuns: 12, - endOnFailure: true, + it.each(fairnessPropertyModes)( + `bounds persist completions for generated ordered cold-hydrate/persist histories in $label mode`, + async (mode) => { + const executionsBefore = generatedPropertyExecutions + await fc.assert(generatedFairnessProperty, { + numRuns: GENERATED_RUNS, verbose: 2, - }, + ...(mode.seed === undefined ? {} : { seed: mode.seed }), + ...(mode.path === undefined ? {} : { path: mode.path }), + }) + expect(generatedPropertyExecutions).toBeGreaterThan(executionsBefore) + }, + ) + + it(`proves tail-order ablation removes a hydrate-before-later-persist history`, () => { + const sampleOptions = { seed: FIXED_SEED, numRuns: GENERATED_RUNS } + const completeGrammar = fc.sample(generatedHistoryArbitrary, sampleOptions) + const tailOrderAblatedGrammar = fc.sample( + tailOrderAblatedArbitrary, + sampleOptions, + ) + + expect(completeGrammar.some(reachesHydrateBeforeLaterPersist)).toBe(true) + expect(tailOrderAblatedGrammar.some(reachesHydrateBeforeLaterPersist)).toBe( + false, ) }) - it(`kills a persist-first FIFO scheduling mutant through the real package path`, async () => { + it(`rejects the named persist-first FIFO wrong answer through the real package path`, async () => { await withNodeScenario( createScenario( `persist-first-fixture-fault`, - [`persist`, `persist`, `persist`, `persist`, `hydrate`], + [`persist`, `persist`, `persist`, `persist`, `hydrate`, `hydrate`], 1, ), (observation) => { @@ -281,7 +541,7 @@ describe(`shared BrowserWASQLiteDriver fairness oracle`, () => { it(`calibrates the checker against a synthetic persist-first observation`, () => { const scenario = createScenario( `persist-first-checker-calibration`, - [`persist`, `persist`, `persist`, `persist`, `hydrate`], + [`persist`, `persist`, `persist`, `persist`, `hydrate`, `hydrate`], 1, ) const fault = createPersistFirstFaultObservation(scenario) diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts index a2cb29b5d9..01666bb94d 100644 --- a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts @@ -11,8 +11,12 @@ * that already-running persist when any persist exists, and then permutes * complete hydrate and persist requests. Hydrates contain nonempty unique * seeded rows; persists contain one or more mutations. The generated campaign - * varies 2..7 hydrates, 2..7 persists, 1..3 mutations per persist, and sampled - * tail permutations. Neutral histories contain only hydrates. + * varies 2..7 hydrates, 3..7 persists, 1..3 mutations per persist, and sampled + * tail permutations. The minimum grammar still has two hydrate checkpoints and + * a persist backlog beyond K=1. Hydrate count supplies repeated checkpoints; + * persist count varies the backlog; mutation width proves K counts logical + * persists rather than their rows; tail order creates overlapping lane work. + * Neutral histories contain only hydrates. * * Independent model and production boundary: `createFairnessReference` * computes permitted completed persist IDs from the ordered history and K; it @@ -29,11 +33,15 @@ * eventuality, multi-process coordination, or a browser matrix; the Chromium * OPFS fixture separately refines the provider boundary. * - * Challenge and replay: the executable persist-first FIFO driver must violate - * the same K=1 checker while using the public/core/driver path. Re-run a - * generated failure with TANSTACK_DB_DRIVER_FAIRNESS_SEED and - * TANSTACK_DB_DRIVER_FAIRNESS_PATH. Cleanup preserves the primary failure and - * reports secondary resource-release diagnostics separately. + * Challenge and replay: the named persist-first FIFO wrong answer must violate + * the same K=1 checker while using the public/core/driver path. The generated + * property has fixed-seed and seedless-random lanes. Supplying both + * TANSTACK_DB_DRIVER_FAIRNESS_SEED and TANSTACK_DB_DRIVER_FAIRNESS_PATH runs + * only their checked oracle replay. Tail-order ablation must lose the legal + * hydrate-before-later-persist cell. The grammar also reconstructs that known + * wrong-answer witness, covers its bounded marginals, and rejects a storm that + * omits the already-running persist boundary. Cleanup preserves the primary + * failure and reports secondary resource-release diagnostics separately. */ import { createCollection } from '../../db/src/index' import { persistedCollectionOptions } from '../src/index' From e5728435c0b1429eb4bea6bf438ad88042795517 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 16:52:22 +0100 Subject: [PATCH 14/14] fix(sqlite-persistence): harden mutation envelope deduplication --- .../src/browser-coordinator.ts | 41 +++- .../tests/browser-coordinator.test.ts | 194 ++++++++++++++++++ .../tests/shared-logical-scheduling.test.ts | 42 ++++ .../src/electron-coordinator.ts | 41 +++- .../tests/electron-coordinator.test.ts | 161 +++++++++++++++ 5 files changed, 473 insertions(+), 6 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index d54d25f1ee..edf19b77af 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -135,6 +135,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private readonly collections = new Map() private readonly pendingRPCs = new Map() private readonly appliedEnvelopeIds = new Map() + private readonly applyingEnvelopeIds = new Map< + string, + Promise + >() private appliedEnvelopePruneTimer: ReturnType | null = null private readonly disposedPromise: Promise private rejectDisposed: ((error: Error) => void) | null = null @@ -355,6 +359,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.appliedEnvelopePruneTimer = null } this.appliedEnvelopeIds.clear() + this.applyingEnvelopeIds.clear() this.channel.close() this.collections.clear() @@ -721,13 +726,43 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina mutations: Array }, ): Promise { - // Dedupe by envelopeId + const envelopeKey = JSON.stringify([collectionId, request.envelopeId]) this.pruneAppliedEnvelopeIds() - const appliedEnvelope = this.appliedEnvelopeIds.get(request.envelopeId) + const appliedEnvelope = this.appliedEnvelopeIds.get(envelopeKey) if (appliedEnvelope) { return { ...appliedEnvelope.response, rpcId: request.rpcId } } + const applyingEnvelope = this.applyingEnvelopeIds.get(envelopeKey) + if (applyingEnvelope) { + return { ...(await applyingEnvelope), rpcId: request.rpcId } + } + + const application = this.applyLocalMutationsOnce( + collectionId, + request, + envelopeKey, + ) + this.applyingEnvelopeIds.set(envelopeKey, application) + try { + return await application + } finally { + if (this.applyingEnvelopeIds.get(envelopeKey) === application) { + this.applyingEnvelopeIds.delete(envelopeKey) + } + } + } + + private async applyLocalMutationsOnce( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array + }, + envelopeKey: string, + ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { return { @@ -776,7 +811,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // Retain the completed result so transport retries observe the same // durable outcome without applying the envelope again. - this.appliedEnvelopeIds.set(request.envelopeId, { + this.appliedEnvelopeIds.set(envelopeKey, { collectionId, appliedAt: Date.now(), response, diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 2fed3f18c9..a73b76cb62 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -519,6 +519,158 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + it(`scopes envelope deduplication by collection`, async () => { + const alphaAdapter = createStubAdapter() + const betaAdapter = createStubAdapter() + const coordinator = createCoordinator(alphaAdapter) + coordinator.setAdapterForCollection(`alpha`, alphaAdapter) + coordinator.setAdapterForCollection(`beta`, betaAdapter) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + await flush(50) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ ok: boolean; rpcId: string }> + } + + try { + const alpha = await internals.handleApplyLocalMutations(`alpha`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `alpha-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `alpha-mutation`, + type: `insert`, + key: `alpha`, + value: { id: `alpha` }, + }, + ], + }) + const beta = await internals.handleApplyLocalMutations(`beta`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `beta-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `beta-mutation`, + type: `insert`, + key: `beta`, + value: { id: `beta` }, + }, + ], + }) + + expect({ + alpha, + alphaApplies: alphaAdapter.appliedTxs, + beta, + betaApplies: betaAdapter.appliedTxs, + }).toMatchObject({ + alpha: { ok: true, rpcId: `alpha-rpc` }, + alphaApplies: [{ collectionId: `alpha` }], + beta: { ok: true, rpcId: `beta-rpc` }, + betaApplies: [{ collectionId: `beta` }], + }) + } finally { + coordinator.dispose() + } + }) + + it(`coalesces an envelope retry while its first write is in flight`, async () => { + const adapter = createStubAdapter() + const firstApplyEntered = createDeferred() + const releaseFirstApply = createDeferred() + let applyCalls = 0 + adapter.applyCommittedTx = async (collectionId, tx) => { + applyCalls++ + if (applyCalls === 1) { + firstApplyEntered.resolve() + await releaseFirstApply.promise + } + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + } + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ + ok: boolean + rpcId: string + term?: number + seq?: number + latestRowVersion?: number + }> + } + const request = { + type: `rpc:applyLocalMutations:req` as const, + envelopeId: `in-flight-envelope`, + mutations: [ + { + mutationId: `mutation`, + type: `insert` as const, + key: `row`, + value: { id: `row` }, + }, + ], + } + + try { + const first = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `first-rpc`, + }) + await firstApplyEntered.promise + const retry = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `retry-rpc`, + }) + await Promise.resolve() + releaseFirstApply.resolve() + + const [firstResponse, retryResponse] = await Promise.all([first, retry]) + expect({ + applyCalls, + first: { ...firstResponse, rpcId: undefined }, + retry: { ...retryResponse, rpcId: undefined }, + }).toEqual({ + applyCalls: 1, + first: { ...retryResponse, rpcId: undefined }, + retry: { ...firstResponse, rpcId: undefined }, + }) + } finally { + releaseFirstApply.resolve() + coordinator.dispose() + } + }) + it(`replays the successful mutation result when its first response is lost`, async () => { vi.useFakeTimers() const adapter = createStubAdapter() @@ -1044,6 +1196,48 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`replaces a cached default adapter with its collection registration`, async () => { + const defaultAdapter = createStubAdapter() + const replacementAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + defaultAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + defaultAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + replacementAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coordinator = createCoordinator(defaultAdapter) + try { + coordinator.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + + await coordinator.requestEnsurePersistedIndex(`todos`, `idx-default`, { + expressionSql: [`title`], + }) + coordinator.setAdapterForCollection(`todos`, replacementAdapter) + await coordinator.requestEnsurePersistedIndex( + `todos`, + `idx-replacement`, + { expressionSql: [`title`] }, + ) + + expect(defaultAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(replacementAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(replacementAdapter.ensureIndex).toHaveBeenCalledWith( + `todos`, + `idx-replacement`, + { expressionSql: [`title`] }, + ) + } finally { + coordinator.dispose() + } + }) + it(`routes generated collection and RPC histories through their owning adapter`, async () => { let run = 0 await fc.assert( diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts index 9062ba7f6e..aa2bd829e0 100644 --- a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -57,6 +57,7 @@ class FirstQueryGatedDriver implements SQLiteDriver { readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]: object readonly admissions: Array = [] readonly firstQueryEntered = createDeferred() + onQuery: (() => void) | undefined private readonly firstQueryGate = createDeferred() private holdFirstQuery = true @@ -72,6 +73,7 @@ class FirstQueryGatedDriver implements SQLiteDriver { async query(): Promise> { this.admissions.push(`query`) + this.onQuery?.() if (this.holdFirstQuery) { this.holdFirstQuery = false this.firstQueryEntered.resolve() @@ -215,6 +217,12 @@ describe(`shared logical scheduling`, () => { key: {}, wrap: createTransparentWrapper, }, + { + name: `twice-transparent delegating driver`, + key: {}, + wrap: (driver: SQLiteDriver) => + createTransparentWrapper(createTransparentWrapper(driver)), + }, { name: `function-valued key`, key: () => undefined, @@ -301,6 +309,40 @@ describe(`shared logical scheduling`, () => { ) }) + it(`counts a rejected hydrate as a completed K=1 lane unit`, async () => { + const driver = new FirstQueryGatedDriver() + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const regularAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const events: Array = [] + + const initialRegular = regularAdapter.loadCollectionMetadata!(`initial`) + await driver.firstQueryEntered.promise + driver.onQuery = () => events.push(`query`) + + const failedHydrate = hydrateAdapter.runInHydrationScope!(() => { + events.push(`failed-hydrate`) + throw new Error(`expected hydrate failure`) + }) + const failedHydrateExpectation = expect(failedHydrate).rejects.toThrow( + `expected hydrate failure`, + ) + const regular = regularAdapter.loadCollectionMetadata!(`regular`) + const nextHydrate = hydrateAdapter.runInHydrationScope!(async (scoped) => { + events.push(`next-hydrate`) + await scoped.loadCollectionMetadata!(`hydrate`) + }) + + driver.releaseFirstQuery() + await Promise.all([ + initialRegular, + failedHydrateExpectation, + regular, + nextHydrate, + ]) + + expect(events).toEqual([`failed-hydrate`, `query`, `next-hydrate`, `query`]) + }) + it(`stops probing after the first returned promise lacks scheduling support`, async () => { const driver = new UnbrandedPromiseLookupDriver() const adapter = createSQLiteCorePersistenceAdapter({ driver }) diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 6843ad5884..d370659e68 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -135,6 +135,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private readonly collections = new Map() private readonly pendingRPCs = new Map() private readonly appliedEnvelopeIds = new Map() + private readonly applyingEnvelopeIds = new Map< + string, + Promise + >() private appliedEnvelopePruneTimer: ReturnType | null = null private readonly disposedPromise: Promise private rejectDisposed: ((error: Error) => void) | null = null @@ -355,6 +359,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.appliedEnvelopePruneTimer = null } this.appliedEnvelopeIds.clear() + this.applyingEnvelopeIds.clear() this.channel.close() this.collections.clear() @@ -721,13 +726,43 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin mutations: Array }, ): Promise { - // Dedupe by envelopeId + const envelopeKey = JSON.stringify([collectionId, request.envelopeId]) this.pruneAppliedEnvelopeIds() - const appliedEnvelope = this.appliedEnvelopeIds.get(request.envelopeId) + const appliedEnvelope = this.appliedEnvelopeIds.get(envelopeKey) if (appliedEnvelope) { return { ...appliedEnvelope.response, rpcId: request.rpcId } } + const applyingEnvelope = this.applyingEnvelopeIds.get(envelopeKey) + if (applyingEnvelope) { + return { ...(await applyingEnvelope), rpcId: request.rpcId } + } + + const application = this.applyLocalMutationsOnce( + collectionId, + request, + envelopeKey, + ) + this.applyingEnvelopeIds.set(envelopeKey, application) + try { + return await application + } finally { + if (this.applyingEnvelopeIds.get(envelopeKey) === application) { + this.applyingEnvelopeIds.delete(envelopeKey) + } + } + } + + private async applyLocalMutationsOnce( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array + }, + envelopeKey: string, + ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { return { @@ -776,7 +811,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin // Retain the completed result so transport retries observe the same // durable outcome without applying the envelope again. - this.appliedEnvelopeIds.set(request.envelopeId, { + this.appliedEnvelopeIds.set(envelopeKey, { collectionId, appliedAt: Date.now(), response, diff --git a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts index 168ecba1b8..00b9791008 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts @@ -334,6 +334,167 @@ describe(`ElectronCollectionCoordinator parity`, () => { } }) + it(`scopes envelope deduplication by collection`, async () => { + const alphaAdapter = createStubAdapter() + const betaAdapter = createStubAdapter() + const coordinator = createCoordinator(alphaAdapter) + coordinator.setAdapterForCollection(`alpha`, alphaAdapter) + coordinator.setAdapterForCollection(`beta`, betaAdapter) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ ok: boolean; rpcId: string }> + } + + try { + await Promise.all([ + waitForLeadership(coordinator, `alpha`), + waitForLeadership(coordinator, `beta`), + ]) + const alpha = await internals.handleApplyLocalMutations(`alpha`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `alpha-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `alpha-mutation`, + type: `insert`, + key: `alpha`, + value: { id: `alpha` }, + }, + ], + }) + const beta = await internals.handleApplyLocalMutations(`beta`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `beta-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `beta-mutation`, + type: `insert`, + key: `beta`, + value: { id: `beta` }, + }, + ], + }) + + expect({ + alpha, + alphaApplies: alphaAdapter.appliedTxs.length, + beta, + betaApplies: betaAdapter.appliedTxs.length, + }).toMatchObject({ + alpha: { ok: true, rpcId: `alpha-rpc` }, + alphaApplies: 1, + beta: { ok: true, rpcId: `beta-rpc` }, + betaApplies: 1, + }) + } finally { + coordinator.dispose() + } + }) + + it(`coalesces an envelope retry while its first write is in flight`, async () => { + const adapter = createStubAdapter() + let enterFirstApply!: () => void + const firstApplyEntered = new Promise((resolve) => { + enterFirstApply = resolve + }) + let releaseFirstApply!: () => void + const firstApplyRelease = new Promise((resolve) => { + releaseFirstApply = resolve + }) + let applyCalls = 0 + adapter.applyCommittedTx = async (_collectionId, tx) => { + applyCalls++ + if (applyCalls === 1) { + enterFirstApply() + await firstApplyRelease + } + adapter.appliedTxs.push(tx.txId) + } + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ + ok: boolean + rpcId: string + term?: number + seq?: number + latestRowVersion?: number + }> + } + const request = { + type: `rpc:applyLocalMutations:req` as const, + envelopeId: `in-flight-envelope`, + mutations: [ + { + mutationId: `mutation`, + type: `insert` as const, + key: `row`, + value: { id: `row` }, + }, + ], + } + + try { + await waitForLeadership(coordinator, `todos`) + const first = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `first-rpc`, + }) + await firstApplyEntered + const retry = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `retry-rpc`, + }) + await Promise.resolve() + releaseFirstApply() + + const [firstResponse, retryResponse] = await Promise.all([first, retry]) + expect({ + applyCalls, + first: { ...firstResponse, rpcId: undefined }, + retry: { ...retryResponse, rpcId: undefined }, + }).toEqual({ + applyCalls: 1, + first: { ...retryResponse, rpcId: undefined }, + retry: { ...firstResponse, rpcId: undefined }, + }) + } finally { + releaseFirstApply() + coordinator.dispose() + } + }) + it(`releases collection-owned state and retry results with the last subscriber`, async () => { const adapter = createStubAdapter() const coordinator = createCoordinator(adapter)