From 066b6424ae3de154cd3f67a081e4b246f65602b3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 06:36:01 -0600 Subject: [PATCH 1/4] fix: enforce crash-only persistence coordination --- docs/contributing/oracle-coverage.md | 19 +- .../browser-db-sqlite-persistence/README.md | 51 +- .../src/browser-coordinator.ts | 1346 +++++- .../src/browser-persistence.ts | 30 +- .../src/index.ts | 9 + .../tests/browser-coordinator.test.ts | 3474 +++++++++++++++- .../per-collection-coordinator-oracle.test.ts | 3666 +++++++++++++++++ packages/db-sqlite-persistence-core/README.md | 133 +- .../db-sqlite-persistence-core/src/errors.ts | 101 + .../db-sqlite-persistence-core/src/index.ts | 1 + .../src/persisted.ts | 456 +- .../src/remote-subset-wire.ts | 899 ++++ .../src/sqlite-core-adapter.ts | 61 +- .../tests/persisted.test-d.ts | 225 +- .../tests/persisted.test.ts | 845 +++- .../db/skills/db-core/persistence/SKILL.md | 27 +- .../tests/electric-recovery-oracle.test.ts | 24 + .../electron-db-sqlite-persistence/README.md | 65 + .../src/electron-coordinator.ts | 1290 +++++- .../src/index.ts | 13 +- .../src/renderer.ts | 26 +- .../tests/electron-ipc.test.ts | 2988 +++++++++++++- 22 files changed, 15445 insertions(+), 304 deletions(-) create mode 100644 packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts create mode 100644 packages/db-sqlite-persistence-core/src/remote-subset-wire.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 1b2aa2f157..d95cf11712 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -35,15 +35,32 @@ comment and the current API/architecture contract before extending its model. | Drafts and native values | [proxy](../../packages/db/tests/proxy.test.ts), [detachment](../../packages/db/tests/proxy-detachment-contract.test.ts), [iteration](../../packages/db/tests/proxy-iteration-contract.test.ts) | Native-operation controls, exact patches and actual stored rows; alias/cycle/adversarial-key histories. General native-mutator and symbol-write support is not established by a plain-object oracle. | | Query DB and observer | [ownership](../../packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts), [load lifecycle](../../packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts), [observer histories](../../packages/db/tests/live-query-observer-history.property.test.ts) | Real QueryClient boundary and a per-listener eligibility ledger, not a duplicate dispatch queue. Check reentry, peer survival, FIFO and disposal independently of final rows. | | Ordered acquisition | [pagination](../../packages/db/tests/query/pagination-oracle.property.test.ts), [ordered work](../../packages/db/tests/query/ordered-work-oracle.property.test.ts), [ordered lifecycle](../../packages/db/tests/query/ordered-lifecycle-oracle.property.test.ts) | Complete finite provider results, pending windows, ties/nulls, ownership and documented repair timing. Request completion is not proof of unrequested source extent. | +| Opaque backend pagination | [window oracle](../../packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](../../packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](../../packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](../../packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](../../packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](../../packages/powersync-db-collection/tests) | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories and exact driver results. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [Browser composed-owner histories](../../packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](../../packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [Electron IPC and composed owner](../../packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The Browser seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](../../packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](../../packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](../../packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](../../packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](../../packages/react-db/tests/conformance.test.tsx), [React pagination](../../packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](../../packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Small structures and test mechanics | [SortedMap](../../packages/db/tests/SortedMap.test.ts), [cleanup queue](../../packages/db/tests/cleanup-queue.property.test.ts), [guarded replay](../../packages/db/tests/oracle-replay.test.ts) | Map/full-sort and appointment-list models; executed target/seed/path checks. Callback-reentrant scheduling is outside the initial cleanup-queue domain. | ## Acceptance map +The post-merge review added three missing domains to existing owners: + +- [Top-K batch contracts](../../packages/db-ivm/tests/operators/topk-batch-contract.test.ts) + cross sparse-array length/holes and RegExp source/flags/position with equal + controls, replacement order, hash consolidation, and actual retained graph + output. Ordinary replacements also run without the global `File` constructor. +- [Leadership replay](../../packages/offline-transactions/tests/leadership-replay.property.test.ts) + holds real storage-read delivery across successful and permanently rejected + durable removals, with bounded scans, concurrent loads, and unfinished peers. + This is distinct from exactly-once execution across independent owners. +- [Accepted-snapshot retention](../../packages/db/tests/collection-state-retention-oracle.property.test.ts) + varies truncate before/during/after an optimistic delete, rejection versus + rollback, post-capture direct insertion, and later ordinary sync/key reuse. A + hidden accepted insert returns after rollback; an uncaptured insert retires, + and neither snapshot is rebased onto synced fields. + | Issue obligation | Implemented evidence | Limit | | --- | --- | --- | | Metamorphic laws | Includes cross-formulation/partition, D2 independent-key commutation, DBSP incremental/full recomputation, pagination provider/UI boundaries, optimistic snapshot stability | Equivalence premises are explicit; not arbitrary query rewrites. | diff --git a/packages/browser-db-sqlite-persistence/README.md b/packages/browser-db-sqlite-persistence/README.md index 84175bf7cd..08a5222935 100644 --- a/packages/browser-db-sqlite-persistence/README.md +++ b/packages/browser-db-sqlite-persistence/README.md @@ -9,6 +9,7 @@ is opt-in by passing a `BrowserCollectionCoordinator`. - `createBrowserWASQLitePersistence(...)` - `openBrowserWASQLiteOPFSDatabase(...)` +- `BrowserCollectionCoordinator` - `persistedCollectionOptions(...)` (re-exported from core) ## Quick start (single-tab) @@ -97,13 +98,61 @@ export const todosCollection = createCollection( See [`examples/react/offline-transactions`](../../examples/react/offline-transactions/src/db/persisted-todos.ts) for a full multi-tab example. +### Committed transaction ownership + +The persisted sync wrapper sends every source transaction with durable effects +through the coordinator's required +`requestApplyCommittedTx(collectionId, tx)` method. +`BrowserCollectionCoordinator` routes the complete transaction to the current +leader for that collection. The leader applies it with the adapter registered +for the same collection id, including that collection's resolved mode and +`schemaVersion`. + +The route preserves truncation, row changes, row metadata, collection metadata, +and stream position as one `PersistedTx`. It does not feature-detect a partial +route or fall back to row-only mutation RPC. A custom coordinator that omits +`requestApplyCommittedTx` is rejected while the collection is configured, +before its sync source can publish rows. + +Single-tab mode uses the same complete transaction contract. Its +`SingleProcessCoordinator` skips election and channel traffic but still routes +the transaction to the resolved adapter for that collection. + +If a mutating RPC loses its response, Browser coordination replays it only +while the requester still knows the same non-null leader id and term. An +unknown initial route or any leader/term change rejects with +`IndeterminateCommitError`; the application must reconcile the outcome. The +coordinator does not retry that mutation against an unknown or replacement +leader. + +### Remote subset requests + +`BrowserCollectionCoordinator.requestEnsureRemoteSubset(...)` validates and +projects the request before it chooses the local leader or `BroadcastChannel` +route. Registered owners receive the exported +`TransportedLoadSubsetOptions` type. It contains the supported +structured-clone wire data and excludes live `signal` and `subscription` +fields. Unsupported nested values fail immediately with +`RemoteSubsetWireValueError` and the exact value path; no owner callback or +channel post occurs. See the core package's remote subset wire contract for the +complete supported domain. + +Each accepted request is an explicit lease. Retries of the same request object +reuse its acquisition identity, while distinct equal request objects remain +independent. Release is routed to the elected collection owner and unloads the +exact acquired options once. Leadership loss unloads the retiring owner's live +leases, and requesters replay still-live acquisitions against the next leader. +Registering a second owner for one collection throws +`DuplicateRemoteSubsetOwnerError`; no adapter fallback replaces the owner. + ## Notes - `openBrowserWASQLiteOPFSDatabase(...)` starts a dedicated Web Worker and routes SQL operations through it. OPFS sync access handle APIs are used in that worker context. - Single-tab mode does not require `BroadcastChannel` or Web Locks for - correctness. + election, but committed transactions still go through the collection's + registered persistence owner. - Multi-tab mode requires `BroadcastChannel` and the Web Locks API; both are available in all modern browsers. - OPFS capability failures are surfaced as `PersistenceUnavailableError`. diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 1babddc5a7..e5c8c419cb 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -1,12 +1,30 @@ -import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' +import { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + safeRandomUUID, + toPersistedCollectionDurabilityError, + toTransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' import type { + ApplyCommittedTxResponse, ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + IndeterminateCommitRequestType, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, + PersistedRowMetadataMutation, + PersistedTx, PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, + ReleaseRemoteSubsetRequest, + ReleaseRemoteSubsetResponse, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, + TxCommitted, } from '@tanstack/db-sqlite-persistence-core' import type { LoadSubsetOptions } from '@tanstack/db' @@ -26,11 +44,8 @@ const WRITER_LOCK_MAX_RETRIES = 20 // --------------------------------------------------------------------------- type RPCRequest = - | { - type: `rpc:ensureRemoteSubset:req` - rpcId: string - options: LoadSubsetOptions - } + | EnsureRemoteSubsetRequest + | ReleaseRemoteSubsetRequest | { type: `rpc:ensurePersistedIndex:req` rpcId: string @@ -43,6 +58,12 @@ type RPCRequest = envelopeId: string mutations: Array } + | { + type: `rpc:applyCommittedTx:req` + rpcId: string + envelopeId: string + tx: PersistedTx + } | { type: `rpc:pullSince:req` rpcId: string @@ -50,12 +71,8 @@ type RPCRequest = } type RPCResponse = - | { - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - } + | EnsureRemoteSubsetResponse + | ReleaseRemoteSubsetResponse | { type: `rpc:ensurePersistedIndex:res` rpcId: string @@ -63,6 +80,7 @@ type RPCResponse = error?: string } | ApplyLocalMutationsResponse + | ApplyCommittedTxResponse | PullSinceResponse type PendingRPC = { @@ -73,6 +91,7 @@ type PendingRPC = { type CollectionState = { isLeader: boolean + leaderId: string | null lockAbortController: AbortController | null heartbeatTimer: ReturnType | null latestTerm: number @@ -81,8 +100,8 @@ type CollectionState = { subscribers: Set<(message: ProtocolEnvelope) => void> } -// Adapter with pullSince support -type AdapterWithPullSince = PersistenceAdapter & { +// Adapter capabilities used by coordinator-side operations +type CoordinatorAdapter = PersistenceAdapter & { pullSince?: ( collectionId: string, fromRowVersion: number, @@ -105,13 +124,76 @@ type AdapterWithPullSince = PersistenceAdapter & { }> } +type ActiveRemoteSubsetAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise + transferred: boolean + released: boolean + terminalRelease: boolean + release: Promise | null +} + +type AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + released: true + awaitingOwner: true +} + +type RemoteSubsetAcquisition = + | ActiveRemoteSubsetAcquisition + | AwaitingRemoteSubsetOwnerAcquisition + | { + collectionId: string + requesterId: string + acquisitionId: string + released: true + } + +type OutboundRemoteSubsetAcquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean +} + +type AppliedEnvelope = + | { + appliedAt: number + requestType: `rpc:applyLocalMutations:req` + response: ApplyLocalMutationsResponse + } + | { + appliedAt: number + requestType: `rpc:applyCommittedTx:req` + response: ApplyCommittedTxResponse + } + +type InFlightEnvelope = + | { + requestType: `rpc:applyLocalMutations:req` + response: Promise + } + | { + requestType: `rpc:applyCommittedTx:req` + response: Promise + } + // --------------------------------------------------------------------------- // Options // --------------------------------------------------------------------------- export type BrowserCollectionCoordinatorOptions = { dbName: string - adapter?: AdapterWithPullSince + adapter?: CoordinatorAdapter } // --------------------------------------------------------------------------- @@ -121,11 +203,26 @@ export type BrowserCollectionCoordinatorOptions = { export class BrowserCollectionCoordinator implements PersistedCollectionCoordinator { private readonly nodeId = safeRandomUUID() private readonly dbName: string - private adapter: AdapterWithPullSince | null + private defaultAdapter: CoordinatorAdapter | null + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetIds = new Map< + string, + WeakMap + >() + private readonly outboundRemoteSubsetAcquisitions = new Map< + string, + OutboundRemoteSubsetAcquisition + >() + private readonly inboundRemoteSubsetAcquisitions = new Map< + string, + RemoteSubsetAcquisition + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopes = new Map() + private readonly inFlightEnvelopes = new Map() private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -133,18 +230,20 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): CoordinatorAdapter { + const adapter = + this.collectionAdapters.get(collectionId) ?? this.defaultAdapter + if (!adapter) { throw new Error( - `BrowserCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, + `BrowserCollectionCoordinator: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: BrowserCollectionCoordinatorOptions) { this.dbName = options.dbName - this.adapter = options.adapter ?? null + this.defaultAdapter = options.adapter ?? null this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) @@ -156,8 +255,38 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina * Called by `createBrowserWASQLitePersistence` to wire the internally-created * adapter into the coordinator. */ - setAdapter(adapter: AdapterWithPullSince): void { - this.adapter = adapter + setAdapter(adapter: CoordinatorAdapter): void { + this.defaultAdapter = adapter + } + + /** Register the persistence adapter that owns one collection. */ + setAdapterForCollection( + collectionId: string, + adapter: CoordinatorAdapter, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if (acquisition.collectionId !== collectionId) continue + acquisition.acquiredLeaderId = null + acquisition.forceReplay = true + } + void this.replayRemoteSubsetAcquisitions(collectionId) + this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + this.remoteSubsetOwners.delete(collectionId) + this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) + } } // ----------------------------------------------------------------------- @@ -197,23 +326,139 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId: string, options: LoadSubsetOptions, ): Promise { - if (this.isLeader(collectionId)) return + const transportedOptions = toTransportedLoadSubsetOptions(options) + let collectionIds = this.remoteSubsetIds.get(collectionId) + if (!collectionIds) { + collectionIds = new WeakMap() + this.remoteSubsetIds.set(collectionId, collectionIds) + } + let acquisitionId = collectionIds.get(options) + let acquisition = acquisitionId + ? this.outboundRemoteSubsetAcquisitions.get( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + ) + : undefined + if (!acquisition) { + acquisitionId = safeRandomUUID() + collectionIds.set(options, acquisitionId) + acquisition = { + collectionId, + acquisitionId, + options: transportedOptions, + acquiredLeaderId: null, + inFlight: null, + forceReplay: false, + } + this.outboundRemoteSubsetAcquisitions.set( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + acquisition, + ) + } - const response = await this.sendRPC<{ - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - }>(collectionId, { - type: `rpc:ensureRemoteSubset:req`, + await this.acquireRemoteSubset(acquisition) + } + + async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const collectionIds = this.remoteSubsetIds.get(collectionId) + const acquisitionId = collectionIds?.get(options) + if (!acquisitionId) return + const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) + if (!this.outboundRemoteSubsetAcquisitions.delete(key)) return + collectionIds!.delete(options) + + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, rpcId: safeRandomUUID(), - options, - }) + acquisitionId, + } + const response = this.isLeader(collectionId) + ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) + : await this.sendRPC(collectionId, request) if (!response.ok) { - throw new Error( - `ensureRemoteSubset failed: ${response.error ?? `unknown error`}`, + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + } + + private async acquireRemoteSubset( + acquisition: OutboundRemoteSubsetAcquisition, + ): Promise { + if (acquisition.inFlight) return acquisition.inFlight + + const route = { localOwner: false } + let resolveWork!: () => void + let rejectWork!: (error: unknown) => void + const work = new Promise((resolve, reject) => { + resolveWork = resolve + rejectWork = reject + }) + acquisition.inFlight = work + const run = async (): Promise => { + const request: Extract< + RPCRequest, + { type: `rpc:ensureRemoteSubset:req` } + > = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + } + route.localOwner = this.isLeader(acquisition.collectionId) + const response = route.localOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + + if (!response.ok) { + throw new Error(`ensureRemoteSubset failed: ${response.error}`) + } + acquisition.acquiredLeaderId = response.leaderId + } + void run().then(resolveWork, rejectWork) + let acquired = false + try { + await work + acquired = true + } catch (error) { + if (!route.localOwner) { + const owner = this.remoteSubsetOwners.get(acquisition.collectionId) + if (owner) reportRemoteSubsetOwnerError(owner, error) + } + throw error + } finally { + if (acquisition.inFlight === work) acquisition.inFlight = null + const current = this.collections.get(acquisition.collectionId) + const currentLeaderId = current?.isLeader + ? this.nodeId + : (current?.leaderId ?? null) + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, ) + if ( + acquired && + this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && + (acquisition.forceReplay || + (currentLeaderId !== null && + acquisition.acquiredLeaderId !== currentLeaderId)) + ) { + acquisition.forceReplay = false + void this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }) + } } } @@ -223,7 +468,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina spec: PersistedIndexSpec, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -267,6 +516,23 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }) } + async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const request: Extract = { + type: `rpc:applyCommittedTx:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + tx, + } + if (this.isLeader(collectionId)) { + return this.handleApplyCommittedTx(collectionId, request) + } + + return this.sendRPC(collectionId, request) + } + async pullSince( collectionId: string, fromRowVersion: number, @@ -293,6 +559,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina dispose(): void { this.disposed = true + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.postRemoteSubsetRelease(acquisition) + } + this.outboundRemoteSubsetAcquisitions.clear() + this.remoteSubsetIds.clear() + for (const [collectionId, state] of this.collections) { this.releaseLeadership(collectionId, state) } @@ -305,6 +577,14 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.channel.close() this.collections.clear() + this.collectionAdapters.clear() + for (const collectionId of this.remoteSubsetOwners.keys()) { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + } + this.remoteSubsetOwners.clear() + this.inboundRemoteSubsetAcquisitions.clear() + this.appliedEnvelopes.clear() + this.inFlightEnvelopes.clear() } // ----------------------------------------------------------------------- @@ -316,6 +596,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (!state) { state = { isLeader: false, + leaderId: null, lockAbortController: null, heartbeatTimer: null, latestTerm: 0, @@ -348,7 +629,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -358,8 +639,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.latestTerm++ state.isLeader = true + state.leaderId = this.nodeId this.emitHeartbeat(collectionId, state) + void this.replayRemoteSubsetAcquisitions(collectionId) state.heartbeatTimer = setInterval(() => { this.emitHeartbeat(collectionId, state) }, HEARTBEAT_INTERVAL_MS) @@ -377,7 +660,9 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina abortController.signal.addEventListener(`abort`, onAbort) }) } finally { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) state.isLeader = false + state.leaderId = null if (state.heartbeatTimer) { clearInterval(state.heartbeatTimer) state.heartbeatTimer = null @@ -399,9 +684,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } private releaseLeadership( - _collectionId: string, + collectionId: string, state: CollectionState, ): void { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) if (state.lockAbortController) { state.lockAbortController.abort() state.lockAbortController = null @@ -411,6 +697,68 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.heartbeatTimer = null } state.isLeader = false + state.leaderId = null + } + + private postRemoteSubsetRelease( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + } + if (this.isLeader(acquisition.collectionId)) { + void this.handleReleaseRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ).catch(() => { + // The owner already received the exact unload failure through onError. + }) + return + } + this.channel.postMessage({ + v: 1, + dbName: this.dbName, + collectionId: acquisition.collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: request, + } satisfies ProtocolEnvelope) + } + + private async replayRemoteSubsetAcquisitions( + collectionId: string, + ): Promise { + if (this.isDisposed()) return + const state = this.collections.get(collectionId) + const leaderId = state?.isLeader ? this.nodeId : state?.leaderId + if (!leaderId) return + + const replays: Array> = [] + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if ( + acquisition.collectionId !== collectionId || + (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) + ) { + continue + } + if (acquisition.inFlight) { + acquisition.forceReplay = true + continue + } + acquisition.forceReplay = false + replays.push( + this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }), + ) + } + await Promise.all(replays) } private emitHeartbeat(collectionId: string, state: CollectionState): void { @@ -448,6 +796,35 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const type = (payload as Record).type as string | undefined + if (type === `leader:heartbeat`) { + const heartbeat = payload as { + leaderId?: unknown + term?: unknown + latestSeq?: unknown + latestRowVersion?: unknown + } + if ( + typeof heartbeat.leaderId === `string` && + typeof heartbeat.term === `number` && + typeof heartbeat.latestSeq === `number` && + typeof heartbeat.latestRowVersion === `number` + ) { + const state = this.ensureCollectionState(envelope.collectionId) + if (heartbeat.term < state.latestTerm) return + const changedLeader = state.leaderId !== heartbeat.leaderId + state.leaderId = heartbeat.leaderId + state.latestTerm = Math.max(state.latestTerm, heartbeat.term) + state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) + state.latestRowVersion = Math.max( + state.latestRowVersion, + heartbeat.latestRowVersion, + ) + if (changedLeader) { + void this.replayRemoteSubsetAcquisitions(envelope.collectionId) + } + } + } + // Handle RPC responses (for pending outbound RPCs) if (type && type.endsWith(`:res`)) { const rpcId = (payload as { rpcId?: string }).rpcId @@ -464,7 +841,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (type && type.endsWith(`:req`)) { const collectionId = envelope.collectionId if (this.isLeader(collectionId)) { - void this.handleRPCRequest(collectionId, payload as RPCRequest) + void this.handleRPCRequest( + collectionId, + payload as RPCRequest, + envelope.senderId, + ) } return } @@ -487,15 +868,53 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina request: RPCRequest, ): Promise { let lastError: Error | undefined + let firstTransportCause: unknown + const mutationRequestType = isMutatingRPCRequest(request) + ? request.type + : undefined + const mutationRoute = mutationRequestType + ? this.captureMutationRoute(collectionId) + : undefined for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { await sleep(RPC_RETRY_DELAY_MS * attempt) } + if ( + mutationRoute && + mutationRequestType && + firstTransportCause !== undefined + ) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } + + if (this.isLeader(collectionId)) { + return (await this.dispatchRPCRequest( + collectionId, + request, + this.nodeId, + )) as T + } + try { return await this.sendRPCOnce(collectionId, request) } catch (error) { + if (this.isDisposed()) throw error + firstTransportCause ??= error + if (mutationRoute && mutationRequestType) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } lastError = error instanceof Error ? error : new Error(String(error)) } } @@ -503,6 +922,43 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina throw lastError ?? new Error(`RPC failed after retries`) } + private captureMutationRoute(collectionId: string): { + leaderId: string | null + term: number | null + } { + const state = this.collections.get(collectionId) + return { + leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), + term: state?.latestTerm ?? null, + } + } + + private assertMutationRouteUnchanged( + collectionId: string, + requestType: IndeterminateCommitRequestType, + previous: { leaderId: string | null; term: number | null }, + cause: unknown, + ): void { + const current = this.captureMutationRoute(collectionId) + if ( + previous.leaderId !== null && + previous.term !== null && + current.leaderId === previous.leaderId && + current.term === previous.term + ) { + return + } + throw new IndeterminateCommitError({ + collectionId, + requestType, + previousLeaderId: previous.leaderId, + previousTerm: previous.term, + currentLeaderId: current.leaderId, + currentTerm: current.term, + cause, + }) + } + private sendRPCOnce( collectionId: string, request: RPCRequest, @@ -542,38 +998,22 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private async handleRPCRequest( collectionId: string, request: RPCRequest, + requesterId: string, ): Promise { let response: RPCResponse try { - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - response = await this.handleEnsureRemoteSubset(collectionId, request) - break - case `rpc:ensurePersistedIndex:req`: - response = await this.handleEnsurePersistedIndex( - collectionId, - request, - ) - break - case `rpc:applyLocalMutations:req`: - response = await this.handleApplyLocalMutations(collectionId, request) - break - case `rpc:pullSince:req`: - response = await this.handlePullSince(collectionId, request) - break - default: - return - } + response = await this.dispatchRPCRequest( + collectionId, + request, + requesterId, + ) } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error) - response = { - type: request.type.replace(`:req`, `:res`) as RPCResponse[`type`], - rpcId: request.rpcId, - ok: false, - error: errorMessage, - } as RPCResponse + response = createRPCErrorResponse(request, error) + } + + if (this.isDisposed()) { + return } const envelope: ProtocolEnvelope = { @@ -587,16 +1027,357 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.channel.postMessage(envelope) } - private handleEnsureRemoteSubset( - _collectionId: string, - request: { type: `rpc:ensureRemoteSubset:req`; rpcId: string }, - ): RPCResponse { - // Leader doesn't need to do anything special — the remote subset - // is ensured by the leader's own sync connection + private dispatchRPCRequest( + collectionId: string, + request: RPCRequest, + requesterId: string, + ): Promise { + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return this.handleEnsureRemoteSubset(collectionId, request, requesterId) + case `rpc:releaseRemoteSubset:req`: + return this.handleReleaseRemoteSubset( + collectionId, + request, + requesterId, + ) + case `rpc:ensurePersistedIndex:req`: + return this.handleEnsurePersistedIndex(collectionId, request) + case `rpc:applyLocalMutations:req`: + return this.handleApplyLocalMutations(collectionId, request) + case `rpc:applyCommittedTx:req`: + return this.handleApplyCommittedTx(collectionId, request) + case `rpc:pullSince:req`: + return this.handlePullSince(collectionId, request) + } + } + + private async handleEnsureRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const existing = this.inboundRemoteSubsetAcquisitions.get(key) + const awaitingOwner = + existing && `awaitingOwner` in existing ? existing : undefined + if (existing) { + if (`owner` in existing) { + if (!existing.released) { + await existing.load + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + await existing.release + if (existing.terminalRelease) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } else if (!(`awaitingOwner` in existing)) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + } + + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new Error( + `BrowserCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, + ) + } + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + owner, + options: awaitingOwner?.options ?? request.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + if (awaitingOwner) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } else { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + reportRemoteSubsetOwnerError(owner, error) + throw error + } return { type: `rpc:ensureRemoteSubset:res`, rpcId: request.rpcId, ok: true, + leaderId: this.nodeId, + } + } + + private async handleReleaseRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } else if (`owner` in acquisition) { + acquisition.terminalRelease = true + await this.releaseRemoteSubsetAcquisition(acquisition) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + } else if (`awaitingOwner` in acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + } + } + + private releaseRemoteSubsetAcquisition( + acquisition: ActiveRemoteSubsetAcquisition, + ): Promise { + if (acquisition.release) return acquisition.release + acquisition.released = true + acquisition.release = (async () => { + try { + await acquisition.load + } catch { + // A returned promise transfers the lease even when initial loading fails. + } + try { + if (acquisition.transferred) { + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + } finally { + if (!acquisition.terminalRelease) { + const key = inboundRemoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.requesterId, + acquisition.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId: acquisition.collectionId, + requesterId: acquisition.requesterId, + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + released: true, + awaitingOwner: true, + }) + } + } + } + })() + return acquisition.release + } + + private releaseInboundRemoteSubsetAcquisitions( + collectionId: string, + owner?: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + (owner && acquisition.owner !== owner) + ) { + continue + } + void this.releaseRemoteSubsetAcquisition(acquisition).catch( + () => undefined, + ) + } + } + + private rebindRemoteInboundSubsetAcquisitions( + collectionId: string, + owner: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if (`awaitingOwner` in acquisition) { + if ( + acquisition.collectionId === collectionId && + acquisition.requesterId !== this.nodeId + ) { + void this.bindAwaitingRemoteSubsetAcquisition( + acquisition, + owner, + ).catch(() => { + // The owner receives the exact load failure through onError. + }) + } + continue + } + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + acquisition.requesterId === this.nodeId || + !acquisition.released || + acquisition.terminalRelease + ) { + continue + } + + void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( + () => undefined, + ) + } + } + + private async rebindRemoteInboundSubsetAcquisition( + previous: ActiveRemoteSubsetAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + await previous.release + if ( + previous.terminalRelease || + this.remoteSubsetOwners.get(previous.collectionId) !== owner + ) { + return + } + + const key = inboundRemoteSubsetAcquisitionKey( + previous.collectionId, + previous.requesterId, + previous.acquisitionId, + ) + const current = this.inboundRemoteSubsetAcquisitions.get(key) + if (current && current !== previous) { + if (`awaitingOwner` in current) { + await this.bindAwaitingRemoteSubsetAcquisition(current, owner) + } + return + } + + const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: previous.collectionId, + requesterId: previous.requesterId, + acquisitionId: previous.acquisitionId, + options: previous.options, + released: true, + awaitingOwner: true, + } + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) + } + + private async bindAwaitingRemoteSubsetAcquisition( + awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { + return + } + const key = inboundRemoteSubsetAcquisitionKey( + awaitingOwner.collectionId, + awaitingOwner.requesterId, + awaitingOwner.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId: awaitingOwner.collectionId, + requesterId: awaitingOwner.requesterId, + acquisitionId: awaitingOwner.acquisitionId, + owner, + options: awaitingOwner.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } + reportRemoteSubsetOwnerError(owner, error) + throw error } } @@ -610,7 +1391,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -632,8 +1413,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina mutations: Array }, ): Promise { - // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) + if (appliedEnvelope?.requestType === `rpc:applyLocalMutations:req`) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + if (appliedEnvelope) { return { type: `rpc:applyLocalMutations:res`, rpcId: request.rpcId, @@ -643,6 +1428,40 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } } + const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) + if (inFlightEnvelope?.requestType === `rpc:applyLocalMutations:req`) { + const response = await inFlightEnvelope.response + return { ...response, rpcId: request.rpcId } + } + if (inFlightEnvelope) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} is already in flight`, + } + } + + const response = this.applyLocalMutationsOnce(collectionId, request) + const pendingEnvelope: InFlightEnvelope = { + requestType: request.type, + response, + } + this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + try { + return await response + } finally { + if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(envelopeKey) + } + } + } + + private async applyLocalMutationsOnce( + collectionId: string, + request: Extract, + ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { return { @@ -663,6 +1482,21 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const rowVersion = state.latestRowVersion // Build and apply the persisted transaction + const rowMetadataMutations: Array = [] + for (const mutation of request.mutations) { + if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { + continue + } + rowMetadataMutations.push( + mutation.metadata === undefined + ? { type: `delete`, key: mutation.key } + : { + type: `set`, + key: mutation.key, + value: mutation.metadata, + }, + ) + } const tx = { txId: safeRandomUUID(), term, @@ -672,16 +1506,42 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina type: m.type, key: m.key, value: m.value, + ...(`metadataChanged` in m + ? { metadata: m.metadata, metadataChanged: m.metadataChanged } + : {}), })), + rowMetadataMutations, } - await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), - ) + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) - this.pruneAppliedEnvelopeIds() + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + if (this.isDisposed()) { + return response + } + this.appliedEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { + appliedAt: Date.now(), + requestType: request.type, + response, + }, + ) + this.pruneAppliedEnvelopes() // Broadcast tx:committed to all tabs const changedRows = request.mutations @@ -706,6 +1566,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina requiresFullReload: false, changedRows, deletedKeys, + rowMetadataMutations, }, } this.channel.postMessage(txCommitted) @@ -715,15 +1576,151 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina subscriber(txCommitted) } - return { - type: `rpc:applyLocalMutations:res`, + return response + } + + private async handleApplyCommittedTx( + collectionId: string, + request: Extract, + ): Promise { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopes.get(envelopeKey) + if (appliedEnvelope?.requestType === `rpc:applyCommittedTx:req`) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + if (appliedEnvelope) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} already applied`, + } + } + + const inFlightEnvelope = this.inFlightEnvelopes.get(envelopeKey) + if (inFlightEnvelope?.requestType === `rpc:applyCommittedTx:req`) { + const response = await inFlightEnvelope.response + return { ...response, rpcId: request.rpcId } + } + if (inFlightEnvelope) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error: `envelope ${request.envelopeId} is already in flight`, + } + } + + const response = this.applyCommittedTxOnce(collectionId, request) + const pendingEnvelope: InFlightEnvelope = { + requestType: request.type, + response, + } + this.inFlightEnvelopes.set(envelopeKey, pendingEnvelope) + try { + return await response + } finally { + if (this.inFlightEnvelopes.get(envelopeKey) === pendingEnvelope) { + this.inFlightEnvelopes.delete(envelopeKey) + } + } + } + + private async applyCommittedTxOnce( + collectionId: string, + request: Extract, + ): Promise { + const state = this.collections.get(collectionId) + if (!state || !state.isLeader) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, + } + } + + state.latestSeq++ + state.latestRowVersion++ + const tx: PersistedTx = { + ...request.tx, + term: state.latestTerm, + seq: state.latestSeq, + rowVersion: state.latestRowVersion, + } + + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + const response: ApplyCommittedTxResponse = { + type: `rpc:applyCommittedTx:res`, rpcId: request.rpcId, ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + if (this.isDisposed()) { + return response + } + this.appliedEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { + appliedAt: Date.now(), + requestType: request.type, + response, + }, + ) + this.pruneAppliedEnvelopes() + + const committedBase = { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + } + const committedPayload: TxCommitted = tx.truncate + ? { + ...committedBase, + requiresFullReload: true, + } + : { + ...committedBase, + requiresFullReload: false, + changedRows: tx.mutations + .filter((mutation) => mutation.type !== `delete`) + .map((mutation) => ({ + key: mutation.key, + value: mutation.value, + })), + deletedKeys: tx.mutations + .filter((mutation) => mutation.type === `delete`) + .map((mutation) => mutation.key), + rowMetadataMutations: tx.rowMetadataMutations, + collectionMetadataMutations: tx.collectionMetadataMutations, + } + const committed: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: committedPayload, + } + this.channel.postMessage(committed) + for (const subscriber of state.subscribers) { + subscriber(committed) } + + return response } private async handlePullSince( @@ -736,7 +1733,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -784,9 +1781,16 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const lockName = `tsdb:writer:${this.dbName}` for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const callbackState = { entered: false } try { - return await navigator.locks.request(lockName, async () => fn()) + return await navigator.locks.request(lockName, async () => { + callbackState.entered = true + return fn() + }) } catch (error) { + if (callbackState.entered) { + throw error + } if (error instanceof DOMException && error.name === `AbortError`) { throw error } @@ -808,12 +1812,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // Helpers // ----------------------------------------------------------------------- - private pruneAppliedEnvelopeIds(): void { + private pruneAppliedEnvelopes(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { - this.appliedEnvelopeIds.delete(id) + for (const [key, envelope] of this.appliedEnvelopes) { + if (envelope.appliedAt < cutoff) { + this.appliedEnvelopes.delete(key) } } } @@ -838,3 +1842,157 @@ function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } + +async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} + +function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { + return JSON.stringify([collectionId, envelopeId]) +} + +function remoteSubsetAcquisitionKey( + collectionId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, acquisitionId]) +} + +function inboundRemoteSubsetAcquisitionKey( + collectionId: string, + requesterId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, requesterId, acquisitionId]) +} + +function createRPCErrorResponse( + request: RPCRequest, + cause: unknown, +): RPCResponse { + const error = cause instanceof Error ? cause.message : String(cause) + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:releaseRemoteSubset:req`: + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:ensurePersistedIndex:req`: + return { + type: `rpc:ensurePersistedIndex:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:applyLocalMutations:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:applyCommittedTx:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:pullSince:req`: + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: false, + error, + } + } +} + +function isMutatingRPCRequest(request: RPCRequest): request is Extract< + RPCRequest, + { + type: IndeterminateCommitRequestType + } +> { + return ( + request.type === `rpc:applyLocalMutations:req` || + request.type === `rpc:applyCommittedTx:req` + ) +} + +function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { + sourceCode?: string | number + path?: string | ReadonlyArray +} { + const sourceCode = + typeof error.code === `string` || typeof error.code === `number` + ? error.code + : undefined + const path = + typeof error.path === `string` || + (Array.isArray(error.path) && + error.path.every( + (part) => typeof part === `string` || typeof part === `number`, + )) + ? (error.path as string | ReadonlyArray) + : undefined + return { + ...(sourceCode === undefined ? {} : { sourceCode }), + ...(path === undefined ? {} : { path }), + } +} diff --git a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts index b36e564900..8e76967a58 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts @@ -131,22 +131,28 @@ export function createBrowserWASQLitePersistence( }) adapterCache.set(cacheKey, adapter) - // Wire the adapter into the multi-tab coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, ensureIndex, etc.) - if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { - resolvedCoordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator: resolvedCoordinator, - }) + collectionId?: string, + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { + if (collectionId === undefined) { + resolvedCoordinator.setAdapter(adapter) + } else { + resolvedCoordinator.setAdapterForCollection(collectionId, adapter) + } + } + + return { + adapter, + coordinator: resolvedCoordinator, + } + } const defaultPersistence = createCollectionPersistence( `sync-absent`, @@ -155,8 +161,8 @@ export function createBrowserWASQLitePersistence( return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(mode, schemaVersion, collectionId), resolvePersistenceForMode: (mode) => createCollectionPersistence(mode, undefined), } diff --git a/packages/browser-db-sqlite-persistence/src/index.ts b/packages/browser-db-sqlite-persistence/src/index.ts index 992e8c092f..c6ab075fd8 100644 --- a/packages/browser-db-sqlite-persistence/src/index.ts +++ b/packages/browser-db-sqlite-persistence/src/index.ts @@ -11,9 +11,18 @@ export type { BrowserCollectionCoordinatorOptions } from './browser-coordinator' export { DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS, DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS, + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + RemoteSubsetWireValueError, persistedCollectionOptions, } from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, + IndeterminateCommitRequestType, + RemoteSubsetOwner, + RemoteSubsetWireExpression, + RemoteSubsetWireValue, + TransportedLoadSubsetOptions, } from '@tanstack/db-sqlite-persistence-core' diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5f554006e2..98fbcd751b 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,7 +1,18 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { IR } from '@tanstack/db' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' +import type { LoadSubsetOptions, Subscription } from '@tanstack/db' +import type { + ApplyCommittedTxRequest, + ApplyLocalMutationsRequest, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + PersistedTx, + PersistenceAdapter, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' -import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' // --------------------------------------------------------------------------- // BroadcastChannel mock @@ -12,6 +23,11 @@ const channels: Map< string, Set<{ onmessage: MessageHandler | null }> > = new Map() +let dropNextBroadcastMessage: ((data: unknown) => boolean) | undefined +let duplicateNextBroadcastMessage: + | ((data: unknown) => unknown | undefined) + | undefined +let observeBroadcastMessage: ((data: unknown) => void) | undefined class MockBroadcastChannel { readonly name: string @@ -26,6 +42,21 @@ class MockBroadcastChannel { } postMessage(data: unknown): void { + observeBroadcastMessage?.(data) + if (dropNextBroadcastMessage?.(data)) { + dropNextBroadcastMessage = undefined + return + } + + const duplicate = duplicateNextBroadcastMessage?.(data) + if (duplicate !== undefined) { + duplicateNextBroadcastMessage = undefined + } + // BroadcastChannel performs serialization during postMessage, so clone + // failures are synchronous rather than deferred to delivery. + const deliveries = ( + duplicate === undefined ? [data] : [data, duplicate] + ).map((delivery) => structuredClone(delivery)) const peers = channels.get(this.name) if (!peers) return // Deliver to all other instances on same channel (simulating cross-tab) @@ -33,7 +64,9 @@ class MockBroadcastChannel { if (peer !== this && peer.onmessage) { // Use microtask to simulate async delivery const handler = peer.onmessage - queueMicrotask(() => handler({ data: structuredClone(data) })) + for (const delivery of deliveries) { + queueMicrotask(() => handler({ data: structuredClone(delivery) })) + } } } } @@ -43,6 +76,15 @@ class MockBroadcastChannel { } } +function injectBroadcastMessage(channelName: string, data: unknown): void { + for (const endpoint of channels.get(channelName) ?? []) { + const handler = endpoint.onmessage + if (handler) { + queueMicrotask(() => handler({ data: structuredClone(data) })) + } + } +} + // --------------------------------------------------------------------------- // Web Locks mock // --------------------------------------------------------------------------- @@ -162,6 +204,9 @@ function installGlobals(): void { } function cleanupGlobals(): void { + dropNextBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + observeBroadcastMessage = undefined channels.clear() heldLocks.clear() lockQueues.clear() @@ -186,15 +231,15 @@ function createStubAdapter(): PersistenceAdapter & { latestSeq: number latestRowVersion: number }> - appliedTxs: Array<{ collectionId: string; txId: string }> + appliedTxs: Array<{ collectionId: string; tx: PersistedTx }> } { - const appliedTxs: Array<{ collectionId: string; txId: string }> = [] + const appliedTxs: Array<{ collectionId: string; tx: PersistedTx }> = [] return { appliedTxs, loadSubset: () => Promise.resolve([]), applyCommittedTx: (collectionId, tx) => { - appliedTxs.push({ collectionId, txId: tx.txId }) + appliedTxs.push({ collectionId, tx }) return Promise.resolve() }, ensureIndex: () => Promise.resolve(), @@ -221,9 +266,248 @@ function createCoordinator( dbName: `test-db`, adapter: adapter ?? createStubAdapter(), } - return new BrowserCollectionCoordinator(opts) + const coordinator = new BrowserCollectionCoordinator(opts) + liveCoordinators.add(coordinator) + return coordinator +} + +const liveCoordinators = new Set() + +function subsetWithNestedValue(value: unknown): LoadSubsetOptions { + return { + where: new IR.Func(`in`, [ + new IR.PropRef([`todos`, `status`]), + new IR.Value([`kept`, value]), + ]), + } +} + +function withUnusedUnloadSubset< + T extends (options: TransportedLoadSubsetOptions) => Promise | void, +>(owner: T): T & RemoteSubsetOwner { + // These fixtures exercise admission and delivery only. The required no-op + // release hook keeps that unrelated scope explicit without weakening the + // production coordinator contract. + return Object.assign(owner, { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) +} + +class UnsupportedSubsetValue { + constructor(readonly label: string) {} +} + +function withEnumerableExpando(value: T): T { + Object.defineProperty(value, `extra`, { + value: `would be lost`, + enumerable: true, + }) + return value +} + +function detachBufferWithView( + kind: `buffer` | `data-view` | `typed-array`, +): ArrayBuffer | DataView | Uint8Array { + const buffer = new ArrayBuffer(4) + const value = + kind === `buffer` + ? buffer + : kind === `data-view` + ? new DataView(buffer) + : new Uint8Array(buffer) + structuredClone(buffer, { transfer: [buffer] }) + return value +} + +type ResizableArrayBufferConstructor = { + new (byteLength: number, options: { maxByteLength: number }): ArrayBuffer +} + +type Float16ArrayConstructor = { + new (values: ArrayLike): ArrayBufferView +} + +function createResizableArrayBuffer(): ArrayBuffer | undefined { + try { + const buffer = new (ArrayBuffer as ResizableArrayBufferConstructor)(4, { + maxByteLength: 8, + }) + return (buffer as ArrayBuffer & { readonly resizable?: boolean }).resizable + ? buffer + : undefined + } catch { + return undefined + } } +const unsupportedValuePath = `options.where.args[1].value[1]` +const resizableArrayBufferValueCases = createResizableArrayBuffer() + ? ([ + [ + `resizable ArrayBuffer`, + () => createResizableArrayBuffer()!, + unsupportedValuePath, + ], + [ + `length-tracking DataView`, + () => new DataView(createResizableArrayBuffer()!), + unsupportedValuePath, + ], + [ + `length-tracking typed array`, + () => new Uint8Array(createResizableArrayBuffer()!), + unsupportedValuePath, + ], + ] as const) + : ([] as const) +const Float16ArrayValue = Reflect.get(globalThis, `Float16Array`) as + | Float16ArrayConstructor + | undefined +const float16ArrayValueCases = Float16ArrayValue + ? ([ + [ + `Float16Array outside the exported typed-array union`, + () => new Float16ArrayValue([1, 2]), + unsupportedValuePath, + ], + ] as const) + : ([] as const) + +const unsupportedSubsetValueCases = [ + [`function`, () => () => {}, unsupportedValuePath], + [`symbol`, () => Symbol(`unsupported`), unsupportedValuePath], + [`WeakMap`, () => new WeakMap(), unsupportedValuePath], + [`WeakSet`, () => new WeakSet(), unsupportedValuePath], + [`Promise`, () => Promise.resolve(`unsupported`), unsupportedValuePath], + [ + `custom prototype`, + () => new UnsupportedSubsetValue(`unsupported`), + unsupportedValuePath, + ], + [ + `custom prototype with a throwing constructor getter`, + () => { + const prototype = Object.create(null) as object + Object.defineProperty(prototype, `constructor`, { + get: () => { + throw new Error(`constructor getter must not be invoked`) + }, + }) + return Object.create(prototype) as object + }, + unsupportedValuePath, + ], + [ + `nonzero RegExp lastIndex`, + () => { + const regexp = /wire/g + regexp.lastIndex = 2 + return regexp + }, + `${unsupportedValuePath}.lastIndex`, + ], + [`SharedArrayBuffer`, () => new SharedArrayBuffer(4), unsupportedValuePath], + [ + `function-valued Map key`, + () => new Map([[() => {}, `value`]]), + `${unsupportedValuePath}.entries[0].key`, + ], + [ + `symbol-valued Map value`, + () => new Map([[`key`, Symbol(`unsupported`)]]), + `${unsupportedValuePath}.entries[0].value`, + ], + [ + `function-valued Set entry`, + () => new Set([() => {}]), + `${unsupportedValuePath}.values[0]`, + ], + [ + `detached view nested in a Map`, + () => new Map([[`key`, detachBufferWithView(`typed-array`)]]), + `${unsupportedValuePath}.entries[0].value`, + ], + [ + `detached ArrayBuffer`, + () => detachBufferWithView(`buffer`), + unsupportedValuePath, + ], + [ + `detached DataView`, + () => detachBufferWithView(`data-view`), + unsupportedValuePath, + ], + [ + `detached typed array`, + () => detachBufferWithView(`typed-array`), + unsupportedValuePath, + ], + [ + `symbol-keyed record`, + () => ({ [Symbol(`unsupported`)]: true }), + `${unsupportedValuePath}[Symbol(unsupported)]`, + ], + [ + `accessor property`, + () => + Object.defineProperty({}, `computed`, { + enumerable: true, + get: () => { + throw new Error(`accessor must not be invoked`) + }, + }), + `${unsupportedValuePath}.computed`, + ], + [ + `non-enumerable property`, + () => Object.defineProperty({}, `hidden`, { value: true }), + `${unsupportedValuePath}.hidden`, + ], + [ + `Date expando`, + () => withEnumerableExpando(new Date(0)), + `${unsupportedValuePath}.extra`, + ], + [ + `RegExp expando`, + () => withEnumerableExpando(/wire/), + `${unsupportedValuePath}.extra`, + ], + [ + `Map expando`, + () => withEnumerableExpando(new Map()), + `${unsupportedValuePath}.extra`, + ], + [ + `Set expando`, + () => withEnumerableExpando(new Set()), + `${unsupportedValuePath}.extra`, + ], + [ + `ArrayBuffer expando`, + () => withEnumerableExpando(new ArrayBuffer(4)), + `${unsupportedValuePath}.extra`, + ], + [ + `DataView expando`, + () => withEnumerableExpando(new DataView(new ArrayBuffer(4))), + `${unsupportedValuePath}.extra`, + ], + [ + `typed array expando`, + () => withEnumerableExpando(new Uint8Array(4)), + `${unsupportedValuePath}.extra`, + ], + [ + `array expando`, + () => withEnumerableExpando([]), + `${unsupportedValuePath}.extra`, + ], + ...resizableArrayBufferValueCases, + ...float16ArrayValueCases, +] as const + async function flush(ms: number = 10): Promise { await new Promise((resolve) => setTimeout(resolve, ms)) } @@ -237,7 +521,10 @@ describe(`BrowserCollectionCoordinator`, () => { installGlobals() }) - afterEach(() => { + afterEach(async () => { + for (const coordinator of liveCoordinators) coordinator.dispose() + liveCoordinators.clear() + await flush(0) cleanupGlobals() }) @@ -460,6 +747,3121 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + + it(`coalesces a local-mutation envelope while application is in flight`, async () => { + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let releaseApplication = (): void => {} + const applicationGate = new Promise((resolve) => { + releaseApplication = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + await applicationGate + return originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + const observedResponses: Array<{ + rpcId: string + term: number + seq: number + latestRowVersion: number + acceptedMutationIds: Array + }> = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + duplicateNextBroadcastMessage = (data) => { + const envelope = data as { payload?: { type?: string } } + if (envelope.payload?.type !== `rpc:applyLocalMutations:req`) { + return undefined + } + const duplicate = structuredClone(data) as { + payload: { rpcId: string } + } + duplicate.payload.rpcId = `coalesced-local-rpc` + return duplicate + } + observeBroadcastMessage = (data) => { + const payload = ( + data as { + payload?: { + type?: string + rpcId?: string + ok?: boolean + term?: number + seq?: number + latestRowVersion?: number + acceptedMutationIds?: Array + } + } + ).payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + observedResponses.push({ + rpcId: payload.rpcId!, + term: payload.term!, + seq: payload.seq!, + latestRowVersion: payload.latestRowVersion!, + acceptedMutationIds: payload.acceptedMutationIds!, + }) + } + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-in-flight`, + type: `insert`, + key: `todo-in-flight-local`, + value: { id: `todo-in-flight-local` }, + }, + ]) + await flush() + releaseApplication() + const response = await responsePromise + await flush() + + const responsesByRpcId = Object.fromEntries( + observedResponses.map(({ rpcId, ...result }) => [rpcId, result]), + ) + const expectedResult = { + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`mut-in-flight`], + } + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(responsesByRpcId).toEqual({ + [response.rpcId]: expectedResult, + 'coalesced-local-rpc': expectedResult, + }) + } finally { + releaseApplication() + observeBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`replays the exact local-mutation success after same-leader response loss`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + vi.useFakeTimers() + try { + let droppedResponse: unknown + dropNextBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type !== `rpc:applyLocalMutations:res`) return false + droppedResponse = structuredClone(payload) + return true + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-response-loss`, + type: `insert`, + key: `todo-local-response-loss`, + value: { id: `todo-local-response-loss` }, + }, + ]) + + await vi.advanceTimersByTimeAsync(0) + expect(droppedResponse).toMatchObject({ + type: `rpc:applyLocalMutations:res`, + ok: true, + }) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(10_200) + const response = await responsePromise + + expect(response).toEqual(droppedResponse) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(followerAdapter.appliedTxs).toEqual([]) + } finally { + leader.dispose() + follower.dispose() + vi.useRealTimers() + } + }) + + it(`fails indeterminate when a local-mutation success is lost across leader change`, async () => { + const retiredLeaderAdapter = createStubAdapter() + const requesterAdapter = createStubAdapter() + const retiredLeader = createCoordinator(retiredLeaderAdapter) + const requester = createCoordinator(requesterAdapter) + + retiredLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await flush(50) + + vi.useFakeTimers() + try { + let publications = 0 + observeBroadcastMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + `tx:committed` + ) { + publications++ + } + } + dropNextBroadcastMessage = (data) => + (data as { payload?: { type?: string } }).payload?.type === + `rpc:applyLocalMutations:res` + const outcomePromise = requester + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-requester-takeover`, + type: `insert`, + key: `local-requester-takeover`, + value: { id: `local-requester-takeover` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.advanceTimersByTimeAsync(0) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + retiredLeader.dispose() + await vi.advanceTimersByTimeAsync(0) + expect(requester.isLeader(`todos`)).toBe(true) + + await vi.advanceTimersByTimeAsync(31_000) + const outcome = await outcomePromise + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: (retiredLeader as unknown as { nodeId: string }) + .nodeId, + previousTerm: 1, + currentLeaderId: (requester as unknown as { nodeId: string }) + .nodeId, + currentTerm: 1, + cause: expect.objectContaining({ + message: expect.stringContaining(`timed out`), + }), + } satisfies Partial), + }) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + expect(requesterAdapter.appliedTxs).toEqual([]) + expect(publications).toBe(1) + } finally { + observeBroadcastMessage = undefined + retiredLeader.dispose() + requester.dispose() + vi.useRealTimers() + } + }) + + it(`fails indeterminate instead of retrying local mutations without an initial leader route`, async () => { + const coordinator = createCoordinator() + const transportError = new Error(`unknown leader response was lost`) + const attemptedMutations: Array = + [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + attemptedMutations.push(structuredClone(request.mutations)) + if (attemptedMutations.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + try { + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `browser-unknown-leader-local`, + type: `insert`, + key: `browser-unknown-leader-local`, + value: { id: `browser-unknown-leader-local` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedMutations).toEqual([ + [ + { + mutationId: `browser-unknown-leader-local`, + type: `insert`, + key: `browser-unknown-leader-local`, + value: { id: `browser-unknown-leader-local` }, + }, + ], + ]) + } finally { + coordinator.dispose() + } + }) + + it(`classifies local-mutation durability failures on local and follower routes`, async () => { + const failure = Object.assign( + new Error(`local mutation persistence failed`), + { + code: `SQLITE_IOERR`, + path: [`todos`, `rows`], + }, + ) + const adapter = createStubAdapter() + adapter.applyCommittedTx = vi.fn().mockRejectedValue(failure) + const leader = createCoordinator(adapter) + const follower = createCoordinator(createStubAdapter()) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + const followerResponse = await follower.requestApplyLocalMutations( + `todos`, + [ + { + mutationId: `remote-durability-failure`, + type: `insert`, + key: `remote-durability-failure`, + value: { id: `remote-durability-failure` }, + }, + ], + ) + + expect(followerResponse).toEqual({ + type: `rpc:applyLocalMutations:res`, + rpcId: expect.any(String), + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(failure.message), + sourceCode: `SQLITE_IOERR`, + path: [`todos`, `rows`], + }) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(1) + + await expect( + leader.requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-durability-failure`, + type: `insert`, + key: `local-durability-failure`, + value: { id: `local-durability-failure` }, + }, + ]), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR`, + path: [`todos`, `rows`], + cause: failure, + } satisfies Partial) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(2) + } finally { + leader.dispose() + follower.dispose() + } + }) + }) + + describe(`RPC - applyCommittedTx`, () => { + it(`routes a complete source transaction to the leader-owned adapter`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + const followerMessages: Array = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, (message) => { + if ((message.payload as { type?: string }).type === `tx:committed`) { + followerMessages.push(message.payload) + } + }) + await flush(50) + + const response = await follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx`, + term: 91, + seq: 92, + rowVersion: 93, + truncate: true, + mutations: [ + { + type: `insert`, + key: `todo-1`, + value: { id: `todo-1`, title: `Owned source row` }, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `todo-1`, value: { source: `remote` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: `next` } }, + ], + }) + + expect(response.ok).toBe(true) + expect(followerAdapter.appliedTxs).toEqual([]) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(leaderAdapter.appliedTxs[0]).toMatchObject({ + collectionId: `todos`, + tx: { + txId: `source-tx`, + truncate: true, + mutations: [ + { + type: `insert`, + key: `todo-1`, + value: { id: `todo-1`, title: `Owned source row` }, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `todo-1`, value: { source: `remote` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: `next` } }, + ], + }, + }) + expect(leaderAdapter.appliedTxs[0]!.tx.term).not.toBe(91) + expect(leaderAdapter.appliedTxs[0]!.tx.seq).not.toBe(92) + expect(leaderAdapter.appliedTxs[0]!.tx.rowVersion).not.toBe(93) + + await flush() + expect(followerMessages).toHaveLength(1) + expect(followerMessages[0]).toEqual({ + type: `tx:committed`, + term: expect.any(Number), + seq: expect.any(Number), + txId: `source-tx`, + latestRowVersion: expect.any(Number), + requiresFullReload: true, + }) + + leader.dispose() + follower.dispose() + }) + + it(`classifies committed durability failures on local and follower routes`, async () => { + const leaderAdapter = createStubAdapter() + const failure = Object.assign(new Error(`persistence failed`), { + code: `SQLITE_FULL`, + path: `/tmp/browser.sqlite`, + }) + leaderAdapter.applyCommittedTx = vi.fn().mockRejectedValue(failure) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(createStubAdapter()) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + const response = await follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + + expect(response).toEqual({ + type: `rpc:applyCommittedTx:res`, + rpcId: expect.any(String), + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(`persistence failed`), + sourceCode: `SQLITE_FULL`, + path: `/tmp/browser.sqlite`, + }) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + + await expect( + leader.requestApplyCommittedTx(`todos`, { + txId: `source-tx-local-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_FULL`, + path: `/tmp/browser.sqlite`, + cause: failure, + } satisfies Partial) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(2) + + leader.dispose() + follower.dispose() + }) + + it(`does not retry application failures after entering the writer lock`, async () => { + const applicationError = new Error(`persistence failed once`) + const adapter = createStubAdapter() + adapter.applyCommittedTx = vi.fn().mockRejectedValue(applicationError) + const coordinator = createCoordinator(adapter) + + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + + vi.useFakeTimers() + try { + let settled = false + const outcomePromise = coordinator + .requestApplyCommittedTx(`todos`, { + txId: `source-tx-application-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + .then( + (response) => { + settled = true + return { response } + }, + (error: unknown) => { + settled = true + return { error } + }, + ) + + await vi.advanceTimersByTimeAsync(0) + const firstTurn = { + applicationCalls: vi.mocked(adapter.applyCommittedTx).mock.calls + .length, + settled, + } + + coordinator.dispose() + await vi.advanceTimersByTimeAsync(5_000) + const outcome = await outcomePromise + + expect(firstTurn).toEqual({ applicationCalls: 1, settled: true }) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(1) + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `PersistedCollectionDurabilityError`, + cause: applicationError, + }), + }) + } finally { + coordinator.dispose() + vi.useRealTimers() + } + }) + + it(`replays the successful response when only that response is lost`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + vi.useFakeTimers() + try { + let droppedResponse: unknown + dropNextBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type !== `rpc:applyCommittedTx:res`) return false + droppedResponse = structuredClone(payload) + return true + } + + const responsePromise = follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-response-loss`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `todo-response-loss`, + value: { id: `todo-response-loss` }, + }, + ], + }) + + await vi.advanceTimersByTimeAsync(0) + expect(droppedResponse).toMatchObject({ + type: `rpc:applyCommittedTx:res`, + ok: true, + }) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(10_200) + const response = await responsePromise + + expect(response).toEqual(droppedResponse) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(followerAdapter.appliedTxs).toEqual([]) + } finally { + leader.dispose() + follower.dispose() + vi.useRealTimers() + } + }) + + it(`coalesces the same envelope while its first application is in flight`, async () => { + const leaderAdapter = createStubAdapter() + const followerAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let releaseApplication = (): void => {} + const applicationGate = new Promise((resolve) => { + releaseApplication = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + await applicationGate + return originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(followerAdapter) + const observedResponses: Array<{ + rpcId: string + term: number + seq: number + latestRowVersion: number + }> = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + duplicateNextBroadcastMessage = (data) => { + const envelope = data as { + payload?: { type?: string; rpcId?: string } + } + if (envelope.payload?.type !== `rpc:applyCommittedTx:req`) { + return undefined + } + const duplicate = structuredClone(data) as { + payload: { rpcId: string } + } + duplicate.payload.rpcId = `coalesced-rpc` + return duplicate + } + observeBroadcastMessage = (data) => { + const payload = ( + data as { + payload?: { + type?: string + rpcId?: string + ok?: boolean + term?: number + seq?: number + latestRowVersion?: number + } + } + ).payload + if ( + payload?.type === `rpc:applyCommittedTx:res` && + payload.ok === true + ) { + observedResponses.push({ + rpcId: payload.rpcId!, + term: payload.term!, + seq: payload.seq!, + latestRowVersion: payload.latestRowVersion!, + }) + } + } + + const responsePromise = follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-in-flight-envelope`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `todo-in-flight-envelope`, + value: { id: `todo-in-flight-envelope` }, + }, + ], + }) + await flush() + const callsBeforeRelease = vi.mocked(leaderAdapter.applyCommittedTx) + .mock.calls.length + + releaseApplication() + const response = await responsePromise + await flush() + + const responsesByRpcId = Object.fromEntries( + observedResponses.map(({ rpcId, ...position }) => [rpcId, position]), + ) + const expectedPosition = { + term: 1, + seq: 1, + latestRowVersion: 1, + } + expect({ + callsBeforeRelease, + appliedTxs: leaderAdapter.appliedTxs.length, + responsesByRpcId, + }).toEqual({ + callsBeforeRelease: 1, + appliedTxs: 1, + responsesByRpcId: { + [response.rpcId]: expectedPosition, + 'coalesced-rpc': expectedPosition, + }, + }) + expect(followerAdapter.appliedTxs).toEqual([]) + } finally { + releaseApplication() + observeBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`allows the same envelope to retry after its application fails`, async () => { + const applicationError = new Error(`first application failed`) + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + leaderAdapter.applyCommittedTx = vi + .fn() + .mockRejectedValueOnce(applicationError) + .mockImplementation((collectionId, tx) => + originalApply(collectionId, tx), + ) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + let requestEnvelope: unknown + const observedResponses: Array<{ ok?: boolean; error?: string }> = [] + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + observeBroadcastMessage = (data) => { + const payload = ( + data as { + payload?: { type?: string; ok?: boolean; error?: string } + } + ).payload + if ( + payload?.type === `rpc:applyCommittedTx:req` && + requestEnvelope === undefined + ) { + requestEnvelope = structuredClone(data) + } + if (payload?.type === `rpc:applyCommittedTx:res`) { + observedResponses.push({ + ok: payload.ok, + ...(payload.error === undefined ? {} : { error: payload.error }), + }) + } + } + + const firstResponse = await follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-retry-after-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + expect(firstResponse).toMatchObject({ + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(applicationError.message), + }) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + expect(requestEnvelope).toBeDefined() + + injectBroadcastMessage(`tsdb:coord:test-db`, requestEnvelope) + await flush() + + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(2) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + expect(observedResponses).toEqual([ + { + ok: false, + error: expect.stringContaining(applicationError.message), + }, + { ok: true }, + ]) + } finally { + observeBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`rejects a different mutation operation that reuses an in-flight envelope`, async () => { + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let releaseApplication = (): void => {} + const applicationGate = new Promise((resolve) => { + releaseApplication = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + await applicationGate + return originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + let crossOperationResponse: unknown + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + duplicateNextBroadcastMessage = (data) => { + const envelope = data as { + payload?: { type?: string; envelopeId?: string } + } + if (envelope.payload?.type !== `rpc:applyCommittedTx:req`) { + return undefined + } + const duplicate = structuredClone(data) as { + payload: Record + } + duplicate.payload = { + type: `rpc:applyLocalMutations:req`, + rpcId: `cross-operation-rpc`, + envelopeId: envelope.payload.envelopeId, + mutations: [], + } + return duplicate + } + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { rpcId?: string } }).payload + if (payload?.rpcId === `cross-operation-rpc`) { + crossOperationResponse = structuredClone(payload) + } + } + + const responsePromise = follower.requestApplyCommittedTx(`todos`, { + txId: `source-tx-cross-operation-envelope`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + await flush() + + expect(crossOperationResponse).toMatchObject({ + type: `rpc:applyLocalMutations:res`, + rpcId: `cross-operation-rpc`, + ok: false, + code: `CONFLICT`, + error: expect.stringContaining(`is already in flight`), + }) + expect(leaderAdapter.applyCommittedTx).toHaveBeenCalledTimes(1) + + releaseApplication() + const response = await responsePromise + expect(response).toMatchObject({ ok: true, seq: 1 }) + expect(leaderAdapter.appliedTxs).toHaveLength(1) + } finally { + releaseApplication() + observeBroadcastMessage = undefined + duplicateNextBroadcastMessage = undefined + leader.dispose() + follower.dispose() + } + }) + + it(`fails indeterminate when a committed success is lost across leader change`, async () => { + const retiredLeaderAdapter = createStubAdapter() + const requesterAdapter = createStubAdapter() + const retiredLeader = createCoordinator(retiredLeaderAdapter) + const requester = createCoordinator(requesterAdapter) + + retiredLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await flush(50) + expect(retiredLeader.isLeader(`todos`)).toBe(true) + expect(requester.isLeader(`todos`)).toBe(false) + + vi.useFakeTimers() + try { + let publications = 0 + observeBroadcastMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + `tx:committed` + ) { + publications++ + } + } + let droppedResponse: unknown + dropNextBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type !== `rpc:applyCommittedTx:res`) return false + droppedResponse = structuredClone(payload) + return true + } + + const outcomePromise = requester + .requestApplyCommittedTx(`todos`, { + txId: `source-tx-requester-takeover`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `todo-requester-takeover`, + value: { id: `todo-requester-takeover` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.advanceTimersByTimeAsync(0) + expect(droppedResponse).toMatchObject({ ok: true }) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + + retiredLeader.dispose() + await vi.advanceTimersByTimeAsync(0) + expect(requester.isLeader(`todos`)).toBe(true) + + await vi.advanceTimersByTimeAsync(31_000) + const outcome = await outcomePromise + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: (retiredLeader as unknown as { nodeId: string }) + .nodeId, + previousTerm: 1, + currentLeaderId: (requester as unknown as { nodeId: string }) + .nodeId, + currentTerm: 1, + cause: expect.objectContaining({ + message: expect.stringContaining(`timed out`), + }), + } satisfies Partial), + }) + expect(retiredLeaderAdapter.appliedTxs).toHaveLength(1) + expect(requesterAdapter.appliedTxs).toEqual([]) + expect(publications).toBe(1) + } finally { + observeBroadcastMessage = undefined + retiredLeader.dispose() + requester.dispose() + vi.useRealTimers() + } + }) + + it(`fails indeterminate instead of retrying a committed transaction without an initial leader route`, async () => { + const coordinator = createCoordinator() + const transportError = new Error(`unknown leader response was lost`) + const attemptedTransactions: Array = [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyCommittedTxRequest) => { + attemptedTransactions.push(structuredClone(request.tx)) + if (attemptedTransactions.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyCommittedTx:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + try { + const outcome = await coordinator + .requestApplyCommittedTx(`todos`, { + txId: `browser-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `browser-unknown-leader-committed`, + value: { id: `browser-unknown-leader-committed` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedTransactions).toEqual([ + { + txId: `browser-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `browser-unknown-leader-committed`, + value: { id: `browser-unknown-leader-committed` }, + }, + ], + }, + ]) + } finally { + coordinator.dispose() + } + }) + + it(`does not retain completed or in-flight committed transactions after disposal`, async () => { + const adapter = createStubAdapter() + const originalApply = adapter.applyCommittedTx + let heldApplyEntered = false + let releaseHeldApply = (): void => {} + const heldApplyGate = new Promise((resolve) => { + releaseHeldApply = resolve + }) + adapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + if (tx.txId === `held-during-disposal`) { + heldApplyEntered = true + await heldApplyGate + } + await originalApply(collectionId, tx) + }) + const coordinator = createCoordinator(adapter) + const internals = coordinator as unknown as { + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise<{ + ok: boolean + code?: string + }> + appliedEnvelopes: Map + inFlightEnvelopes: Map + } + const completedRequest: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: `completed-rpc`, + envelopeId: `completed-envelope`, + tx: { + txId: `completed-before-disposal`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + } + const heldRequest: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-rpc`, + envelopeId: `held-envelope`, + tx: { + txId: `held-during-disposal`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + } + + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + try { + expect( + await internals.handleApplyCommittedTx(`todos`, completedRequest), + ).toMatchObject({ ok: true }) + const heldResponse = internals.handleApplyCommittedTx( + `todos`, + heldRequest, + ) + await vi.waitFor(() => expect(heldApplyEntered).toBe(true)) + expect({ + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, + }).toEqual({ completed: 1, inFlight: 1 }) + + coordinator.dispose() + expect({ + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, + }).toEqual({ completed: 0, inFlight: 0 }) + + releaseHeldApply() + await expect(heldResponse).resolves.toMatchObject({ ok: true }) + expect({ + completed: internals.appliedEnvelopes.size, + inFlight: internals.inFlightEnvelopes.size, + }).toEqual({ completed: 0, inFlight: 0 }) + await expect( + internals.handleApplyCommittedTx(`todos`, completedRequest), + ).resolves.toMatchObject({ ok: false, code: `NOT_LEADER` }) + } finally { + releaseHeldApply() + coordinator.dispose() + } + }) + + it(`does not answer a held follower request after its leader is disposed`, async () => { + const leaderAdapter = createStubAdapter() + const originalApply = leaderAdapter.applyCommittedTx + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + leaderAdapter.applyCommittedTx = vi.fn(async (collectionId, tx) => { + applyEntered = true + await applyGate + await originalApply(collectionId, tx) + }) + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator(createStubAdapter()) + const leaderInternals = leader as unknown as { + appliedEnvelopes: Map + inFlightEnvelopes: Map + } + const followerInternals = follower as unknown as { + sendRPCOnce: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + let responsePosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:applyCommittedTx:res`) { + responsePosts++ + } + } + const outcomePromise = followerInternals + .sendRPCOnce(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-follower-rpc`, + envelopeId: `held-follower-envelope`, + tx: { + txId: `held-follower-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.waitFor(() => expect(applyEntered).toBe(true)) + leader.dispose() + releaseApply() + await vi.waitFor(() => expect(leaderAdapter.appliedTxs).toHaveLength(1)) + await flush() + + expect({ + responsePosts, + completed: leaderInternals.appliedEnvelopes.size, + inFlight: leaderInternals.inFlightEnvelopes.size, + }).toEqual({ responsePosts: 0, completed: 0, inFlight: 0 }) + + follower.dispose() + const outcome = await outcomePromise + expect(outcome).toEqual({ error: new Error(`coordinator disposed`) }) + } finally { + observeBroadcastMessage = undefined + releaseApply() + leader.dispose() + follower.dispose() + } + }) + }) + + describe(`RPC - ensureRemoteSubset`, () => { + it(`rejects a second live remote subset owner instead of replacing the first`, () => { + const coordinator = createCoordinator() + const first = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => Promise.resolve()), + { + loadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.resolve(), + ), + unloadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => {}), + onError: vi.fn(), + }, + ) + const second = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => Promise.resolve()), + { + loadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.resolve(), + ), + unloadSubset: vi.fn((_options: TransportedLoadSubsetOptions) => {}), + onError: vi.fn(), + }, + ) + const unregisterFirst = coordinator.registerRemoteSubsetOwner( + `todos`, + first, + ) + + try { + expect(() => + coordinator.registerRemoteSubsetOwner(`todos`, second), + ).toThrowError( + expect.objectContaining({ + name: `DuplicateRemoteSubsetOwnerError`, + collectionId: `todos`, + }), + ) + } finally { + unregisterFirst() + coordinator.dispose() + } + }) + + it(`rejects an unsupported nested membership value before local owner work`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + await expect( + coordinator.requestEnsureRemoteSubset( + `todos`, + subsetWithNestedValue(() => {}), + ), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + message: `Unsupported remote subset wire value at options.where.args[1].value[1]: function`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`rejects a sparse function argument at its exact Browser wire path`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const owner = withUnusedUnloadSubset(vi.fn()) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const args = [new IR.PropRef([`todos`, `status`])] + args.length = 2 + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`eq`, args), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`holds same-stack Browser subset reentry behind the original owner load`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let duplicate: Promise | undefined + let didReenter = false + const owner = Object.assign( + vi.fn(() => { + if (!didReenter) { + didReenter = true + duplicate = coordinator.requestEnsureRemoteSubset(`todos`, options) + } + return loadGate + }), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(duplicate).toBeDefined()) + const duplicateResult = duplicate!.then(() => { + duplicateSettled = true + }) + await flush(0) + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicateResult]) + } finally { + releaseLoad() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`compacts a terminal same-stack Browser release after the real owner load finishes`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let release: Promise | undefined + const events: Array = [] + const owner = Object.assign( + vi.fn(() => { + events.push(`load`) + release = coordinator.requestReleaseRemoteSubset(`todos`, options) + return loadGate + }), + { + unloadSubset: vi.fn(() => { + events.push(`unload`) + }), + onError: vi.fn(), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + } + + try { + let releaseSettled = false + const load = coordinator.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(release).toBeDefined()) + const terminalRelease = release!.then(() => { + releaseSettled = true + }) + await flush(0) + expect({ events: [...events], releaseSettled }).toEqual({ + events: [`load`], + releaseSettled: false, + }) + + releaseLoad() + await Promise.all([load, terminalRelease]) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + expect({ + events, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + events: [`load`, `unload`], + outbound: 0, + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + releaseLoad() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`releases a transferred Browser lease whose initial load rejected`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const loadError = new Error(`browser transferred load failed`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.reject(loadError), + ), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 20 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + const ensureError = await coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => undefined, + (error: unknown) => error, + ) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + await flush(0) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(ensureError).toBe(loadError) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( + owner.mock.calls[0]?.[0], + ) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`keeps a Browser release tombstone when a transferred load rejects concurrently`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const loadError = new Error(`browser concurrent transferred load failed`) + let rejectLoad = (_error: unknown): void => {} + const loadGate = new Promise((_resolve, reject) => { + rejectLoad = reject + }) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => loadGate), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 21 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + nodeId: string + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + handleEnsureRemoteSubset: ( + collectionId: string, + request: { + type: `rpc:ensureRemoteSubset:req` + rpcId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + }, + requesterId: string, + ) => Promise<{ ok: boolean }> + } + + try { + const ensure = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + const [outbound] = internals.outboundRemoteSubsetAcquisitions.values() + const release = coordinator + .requestReleaseRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + rejectLoad(loadError) + const outcomes = await Promise.all([ensure, release]) + const duplicate = await internals + .handleEnsureRemoteSubset( + `todos`, + { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `delayed-browser-duplicate`, + acquisitionId: outbound!.acquisitionId, + options: owner.mock.calls[0]![0], + }, + internals.nodeId, + ) + .then( + (response) => ({ status: `fulfilled` as const, response }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await flush(0) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(outcomes).toEqual([ + { status: `rejected`, error: loadError }, + { status: `fulfilled` }, + ]) + expect(duplicate).toEqual({ + status: `fulfilled`, + response: expect.objectContaining({ ok: true }), + }) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + rejectLoad(loadError) + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`rejects unsupported nested values before follower transport or owner work`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect(leader.isLeader(`todos`)).toBe(true) + expect(follower.isLeader(`todos`)).toBe(false) + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + await expect( + follower.requestEnsureRemoteSubset( + `todos`, + subsetWithNestedValue(() => {}), + ), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + message: `Unsupported remote subset wire value at options.where.args[1].value[1]: function`, + }) + expect({ subsetPosts, ownerCalls: owner.mock.calls.length }).toEqual({ + subsetPosts: 0, + ownerCalls: 0, + }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it.each(unsupportedSubsetValueCases)( + `rejects a nested %s through both local and follower admission`, + async (_label, createUnsupported, path) => { + const leader = createCoordinator() + const follower = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + const outcomes = await Promise.all( + [leader, follower].map((coordinator) => + coordinator + .requestEnsureRemoteSubset( + `todos`, + subsetWithNestedValue(createUnsupported()), + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ), + ), + ) + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + ok: false, + error: { name: `RemoteSubsetWireValueError`, path }, + }) + } + expect({ + subsetPosts, + ownerCalls: owner.mock.calls.length, + }).toEqual({ subsetPosts: 0, ownerCalls: 0 }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }, + ) + + it.each([ + [ + `where expression`, + { + where: new IR.Value({ nested: () => {} }), + }, + `options.where.value.nested`, + ], + [ + `non-identifier object key`, + { + where: new IR.Value({ [`dotted.key`]: () => {} }), + }, + `options.where.value["dotted.key"]`, + ], + [ + `property-reference path`, + { + where: new IR.PropRef([`todos`, (() => {}) as unknown as string]), + }, + `options.where.path[1]`, + ], + [ + `order expression`, + { + orderBy: [ + { + expression: new IR.Value({ nested: () => {} }), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ], + }, + `options.orderBy[0].expression.value.nested`, + ], + [ + `comparison options`, + { + orderBy: [ + { + expression: new IR.PropRef([`todos`, `title`]), + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + localeOptions: { matcher: () => {} }, + }, + }, + ], + }, + `options.orderBy[0].compareOptions.localeOptions.matcher`, + ], + [ + `cursor from expression`, + { + cursor: { + whereFrom: new IR.Value({ nested: () => {} }), + whereCurrent: new IR.Value(`current`), + }, + }, + `options.cursor.whereFrom.value.nested`, + ], + [ + `cursor current expression`, + { + cursor: { + whereFrom: new IR.Value(`from`), + whereCurrent: new IR.Value({ nested: () => {} }), + }, + }, + `options.cursor.whereCurrent.value.nested`, + ], + [ + `cursor key`, + { + cursor: { + whereFrom: new IR.Value(`from`), + whereCurrent: new IR.Value(`current`), + lastKey: () => {}, + }, + }, + `options.cursor.lastKey`, + ], + [`limit`, { limit: () => {} }, `options.limit`], + [`offset`, { offset: () => {} }, `options.offset`], + ] as const)( + `reports the exact path for an unsupported %s value`, + async (_label, options, path) => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + const outcomes = await Promise.all( + [leader, follower].map((coordinator) => + coordinator + .requestEnsureRemoteSubset( + `todos`, + options as unknown as LoadSubsetOptions, + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ), + ), + ) + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + ok: false, + error: { name: `RemoteSubsetWireValueError`, path }, + }) + } + expect({ + subsetPosts, + ownerCalls: owner.mock.calls.length, + }).toEqual({ subsetPosts: 0, ownerCalls: 0 }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }, + ) + + it.each([ + [ + `wire record reused as an expression`, + () => { + const shared = { bad: 1 } + return { + where: new IR.Func(`and`, [ + new IR.Value(shared), + shared as unknown as IR.BasicExpression, + ]), + } + }, + `options.where.args[1].type`, + ], + [ + `wire array reused as a property-reference path`, + () => { + const shared = [`todos`, 1] + return { + where: new IR.Func(`and`, [ + new IR.Value(shared), + new IR.PropRef(shared as unknown as Array), + ]), + } + }, + `options.where.args[1].path[1]`, + ], + [ + `wire array reused as an expression array`, + () => { + const shared = [1] + return { + where: new IR.Func(`and`, [ + new IR.Value(shared), + new IR.Func( + `nested`, + shared as unknown as Array, + ), + ]), + } + }, + `options.where.args[1].args[0]`, + ], + [ + `wire record reused as comparison options`, + () => { + const shared = { bad: 1 } + return { + where: new IR.Value(shared), + orderBy: [ + { + expression: new IR.PropRef([`todos`, `title`]), + compareOptions: shared, + }, + ], + } + }, + `options.orderBy[0].compareOptions.bad`, + ], + ] as const)( + `does not reuse a projected %s across incompatible schema roles`, + async (_label, createOptions, path) => { + const leader = createCoordinator() + const follower = createCoordinator() + const owner = withUnusedUnloadSubset(vi.fn(() => Promise.resolve())) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + let subsetPosts = 0 + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + observeBroadcastMessage = (data) => { + const payload = (data as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + } + + try { + const outcomes = await Promise.all( + [leader, follower].map((coordinator) => + coordinator + .requestEnsureRemoteSubset( + `todos`, + createOptions() as unknown as LoadSubsetOptions, + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ), + ), + ) + for (const outcome of outcomes) { + expect(outcome).toMatchObject({ + ok: false, + error: { name: `RemoteSubsetWireValueError`, path }, + }) + } + expect({ + subsetPosts, + ownerCalls: owner.mock.calls.length, + }).toEqual({ subsetPosts: 0, ownerCalls: 0 }) + } finally { + observeBroadcastMessage = undefined + unregisterOwner() + leader.dispose() + follower.dispose() + } + }, + ) + + it(`preserves supported native values, aliases, and cycles for both routes`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + const received: Array = [] + const owner = withUnusedUnloadSubset( + vi.fn((options: TransportedLoadSubsetOptions) => { + received.push(options) + return Promise.resolve() + }), + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + const shared = { label: `shared` } + const cycle: Record = { label: `cycle` } + cycle.self = cycle + const buffer = new ArrayBuffer(8) + const view = new DataView(buffer, 2, 4) + const typed = new Uint16Array(buffer, 0, 2) + typed.set([17, 42]) + const typedArrays = [ + new Int8Array([-1, 2]), + new Uint8Array([0, 255]), + new Uint8ClampedArray([0, 255]), + new Int16Array([-2, 3]), + new Uint16Array([2, 65_535]), + new Int32Array([-3, 4]), + new Uint32Array([3, 4_000_000_000]), + new Float32Array([-0, Number.NaN, Number.POSITIVE_INFINITY]), + new Float64Array([Number.NEGATIVE_INFINITY, Math.PI]), + new BigInt64Array([-4n, 5n]), + new BigUint64Array([4n, 5n]), + ] + const sparse = new Array(3) + sparse[1] = undefined + const reservedKeys = { constructor: `own constructor value` } + Object.defineProperty(reservedKeys, `__proto__`, { + value: { label: `own __proto__ value` }, + enumerable: true, + writable: true, + configurable: true, + }) + const richValue = { + undefinedValue: undefined, + nullValue: null, + booleanValue: true, + stringValue: `wire`, + bigintValue: 9_007_199_254_740_993n, + nanValue: Number.NaN, + positiveInfinity: Number.POSITIVE_INFINITY, + negativeInfinity: Number.NEGATIVE_INFINITY, + negativeZero: -0, + date: new Date(`2026-09-16T12:34:56.000Z`), + invalidDate: new Date(Number.NaN), + regexp: /wire/giu, + buffer, + view, + typed, + typedArrays, + map: new Map([[shared, cycle]]), + set: new Set([shared, cycle]), + sparse, + reservedKeys, + first: shared, + second: shared, + cycle, + } + const options = { + where: new IR.Func(`eq`, [ + new IR.PropRef([`todos`, `payload`]), + new IR.Value(richValue), + ]), + signal: new AbortController().signal, + subscription: { + on: () => () => {}, + } as unknown as Subscription, + } + + try { + await leader.requestEnsureRemoteSubset(`todos`, options) + await follower.requestEnsureRemoteSubset(`todos`, options) + expect(received).toHaveLength(2) + + for (const decoded of received) { + const value = ( + decoded.where as unknown as { + args: Array<{ value?: typeof richValue }> + } + ).args[1]!.value! + expect(value.undefinedValue).toBeUndefined() + expect(value.nullValue).toBeNull() + expect(value.booleanValue).toBe(true) + expect(value.stringValue).toBe(`wire`) + expect(value.bigintValue).toBe(9_007_199_254_740_993n) + expect(Object.is(value.nanValue, Number.NaN)).toBe(true) + expect( + Object.is(value.positiveInfinity, Number.POSITIVE_INFINITY), + ).toBe(true) + expect( + Object.is(value.negativeInfinity, Number.NEGATIVE_INFINITY), + ).toBe(true) + expect(Object.is(value.negativeZero, -0)).toBe(true) + expect(value.date.getTime()).toBe(richValue.date.getTime()) + expect(Number.isNaN(value.invalidDate.getTime())).toBe(true) + expect({ + source: value.regexp.source, + flags: value.regexp.flags, + lastIndex: value.regexp.lastIndex, + }).toEqual({ source: `wire`, flags: `giu`, lastIndex: 0 }) + expect(value.buffer).toBeInstanceOf(ArrayBuffer) + expect(value.view).toBeInstanceOf(DataView) + expect(value.typed).toBeInstanceOf(Uint16Array) + expect(Array.from(new Uint8Array(value.buffer))).toEqual( + Array.from(new Uint8Array(richValue.buffer)), + ) + expect({ + byteOffset: value.view.byteOffset, + byteLength: value.view.byteLength, + }).toEqual({ byteOffset: 2, byteLength: 4 }) + expect(Array.from(value.typed)).toEqual([17, 42]) + expect(value.view.buffer).toBe(value.buffer) + expect(value.typed.buffer).toBe(value.buffer) + expect(value.typedArrays).toHaveLength(typedArrays.length) + for (let index = 0; index < typedArrays.length; index++) { + const actual = value.typedArrays[index]! + const expected = typedArrays[index]! + expect(actual.constructor).toBe(expected.constructor) + expect(actual.byteOffset).toBe(expected.byteOffset) + expect(actual.byteLength).toBe(expected.byteLength) + expect( + Array.from( + new Uint8Array( + actual.buffer, + actual.byteOffset, + actual.byteLength, + ), + ), + ).toEqual( + Array.from( + new Uint8Array( + expected.buffer, + expected.byteOffset, + expected.byteLength, + ), + ), + ) + } + expect(value.map).toBeInstanceOf(Map) + expect(value.set).toBeInstanceOf(Set) + expect(value.first).toBe(value.second) + expect([...value.map.keys()]).toEqual([value.first]) + expect(value.map.get(value.first)).toBe(value.cycle) + expect(value.set.has(value.first)).toBe(true) + expect(value.set.has(value.cycle)).toBe(true) + expect(value.cycle.self).toBe(value.cycle) + expect(0 in value.sparse).toBe(false) + expect(1 in value.sparse).toBe(true) + expect(2 in value.sparse).toBe(false) + expect(Object.getPrototypeOf(value.reservedKeys)).toBe( + Object.prototype, + ) + expect( + Object.getOwnPropertyDescriptor(value.reservedKeys, `__proto__`), + ).toMatchObject({ + value: { label: `own __proto__ value` }, + enumerable: true, + writable: true, + configurable: true, + }) + expect(value.reservedKeys.constructor).toBe(`own constructor value`) + expect(decoded).not.toHaveProperty(`signal`) + expect(decoded).not.toHaveProperty(`subscription`) + } + // Local delivery need not clone identity, but it must receive the same + // validated, live-field-free wire domain as follower delivery. + expect(received[1]).not.toBe(options) + } finally { + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`direct leader requests await the registered collection owner`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + + let releaseOwner = (): void => {} + const owner = withUnusedUnloadSubset( + vi.fn( + () => + new Promise((resolve) => { + releaseOwner = resolve + }), + ), + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let settled = false + const request = coordinator + .requestEnsureRemoteSubset(`todos`, { limit: 2, offset: 1 }) + .then(() => { + settled = true + }) + await Promise.resolve() + const beforeRelease = { ownerCalls: owner.mock.calls.length, settled } + + releaseOwner() + await request + + expect(beforeRelease).toEqual({ ownerCalls: 1, settled: false }) + expect(owner).toHaveBeenCalledWith({ limit: 2, offset: 1 }) + } finally { + releaseOwner() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`preserves the receiver of an explicitly registered adapter owner`, async () => { + const receivers: Array = [] + const leaderAdapter = createStubAdapter() as ReturnType< + typeof createStubAdapter + > & { + ensureRemoteSubset: ( + collectionId: string, + options: { limit?: number }, + ) => Promise + } + leaderAdapter.ensureRemoteSubset = function ( + this: typeof leaderAdapter, + _collectionId, + _options, + ) { + receivers.push(this) + return Promise.resolve() + } + const leader = createCoordinator(leaderAdapter) + const follower = createCoordinator() + const owner = withUnusedUnloadSubset( + vi.fn((options: TransportedLoadSubsetOptions) => + leaderAdapter.ensureRemoteSubset(`todos`, options), + ), + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + + try { + await follower.requestEnsureRemoteSubset(`todos`, { limit: 1 }) + expect(receivers).toEqual([leaderAdapter]) + } finally { + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`scopes a reused subset options object to each Browser collection`, async () => { + const coordinator = createCoordinator() + const alpha = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const beta = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterAlpha = coordinator.registerRemoteSubsetOwner( + `alpha`, + alpha, + ) + const unregisterBeta = coordinator.registerRemoteSubsetOwner(`beta`, beta) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + await flush(50) + expect({ + alpha: coordinator.isLeader(`alpha`), + beta: coordinator.isLeader(`beta`), + }).toEqual({ alpha: true, beta: true }) + + const shared: LoadSubsetOptions = { limit: 2, offset: 1 } + try { + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + await coordinator.requestEnsureRemoteSubset(`beta`, shared) + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + + const afterAcquire = { + alphaLoads: alpha.mock.calls.length, + betaLoads: beta.mock.calls.length, + } + await coordinator.requestReleaseRemoteSubset(`alpha`, shared) + await coordinator.requestReleaseRemoteSubset(`beta`, shared) + + expect({ + afterAcquire, + alphaUnloads: alpha.unloadSubset.mock.calls.length, + betaUnloads: beta.unloadSubset.mock.calls.length, + }).toEqual({ + afterAcquire: { alphaLoads: 1, betaLoads: 1 }, + alphaUnloads: 1, + betaUnloads: 1, + }) + } finally { + unregisterAlpha() + unregisterBeta() + coordinator.dispose() + } + }) + + it(`replays after an earlier Browser leader responds behind a new heartbeat`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + let releaseOwner = (): void => {} + const owner = withUnusedUnloadSubset( + vi.fn( + () => + new Promise((resolve) => { + releaseOwner = resolve + }), + ), + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect({ + leader: leader.isLeader(`todos`), + follower: follower.isLeader(`todos`), + }).toEqual({ leader: true, follower: false }) + + const leaderInternals = leader as unknown as { + nodeId: string + channel: { postMessage: (message: unknown) => void } + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensureRemoteSubset:res`) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + let requestPosts = 0 + observeBroadcastMessage = (message) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, { + limit: 1, + }) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + + followerInternals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: `replacement-browser-leader`, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term: 2, + leaderId: `replacement-browser-leader`, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + releaseOwner() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + leader.isLeader = () => false + followerInternals.onChannelMessage(heldResponse) + await request + await flush() + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + acquiredLeaderId: leaderInternals.nodeId, + }) + } finally { + observeBroadcastMessage = undefined + releaseOwner() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`replays a held Browser lease after an A to B to A leader cycle`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const owner = Object.assign( + vi.fn().mockImplementationOnce(() => firstLoadGate), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + const leaderInternals = leader as unknown as { + nodeId: string + channel: { postMessage: (message: unknown) => void } + inboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if ( + type === `rpc:ensureRemoteSubset:res` && + heldResponse === undefined + ) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + let requestPosts = 0 + const wireRequests: Array<{ + acquisitionId: string + rpcId: string + }> = [] + observeBroadcastMessage = (message) => { + const payload = ( + message as { + payload?: { + type?: string + acquisitionId?: string + rpcId?: string + } + } + ).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) { + requestPosts++ + wireRequests.push({ + acquisitionId: payload.acquisitionId!, + rpcId: payload.rpcId!, + }) + } + } + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + followerInternals.onChannelMessage( + heartbeat(`replacement-browser-leader`, 2), + ) + leaderInternals.releaseInboundRemoteSubsetAcquisitions(`todos`) + followerInternals.onChannelMessage(heartbeat(leaderInternals.nodeId, 3)) + releaseFirstLoad() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + followerInternals.onChannelMessage(heldResponse) + await request + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(2)) + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + loads: 2, + unloads: 1, + inbound: 1, + acquiredLeaderId: leaderInternals.nodeId, + }) + expect(wireRequests).toHaveLength(2) + expect(wireRequests[0]!.acquisitionId).not.toBe(``) + expect(wireRequests[1]!.acquisitionId).toBe( + wireRequests[0]!.acquisitionId, + ) + expect(wireRequests[1]!.rpcId).not.toBe(wireRequests[0]!.rpcId) + + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + observeBroadcastMessage = undefined + releaseFirstLoad() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`reports a failed Browser replay once without self-retrying`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const replayError = new Error(`replacement owner is not ready`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(replayError), + { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + type Acquisition = { + acquiredLeaderId: string | null + inFlight: Promise | null + } + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + const [acquisition] = internals.outboundRemoteSubsetAcquisitions.values() + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => + expect(owner.unloadSubset).toHaveBeenCalledTimes(1), + ) + acquisition!.acquiredLeaderId = `retired-browser-leader` + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + + try { + await internals.replayRemoteSubsetAcquisitions(`todos`) + await flush() + expect({ + attempts: owner.mock.calls.length - 1, + ownerErrors, + acquiredLeaderId: acquisition!.acquiredLeaderId, + inFlight: acquisition!.inFlight, + unhandled, + }).toEqual({ + attempts: 1, + ownerErrors: [replayError], + acquiredLeaderId: `retired-browser-leader`, + inFlight: null, + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports a failed remote Browser follower replay to its local owner once`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + sendRPC: (collectionId: string, request: unknown) => Promise + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null; inFlight: Promise | null } + > + } + coordinator.isLeader = () => false + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + heartbeat(`remote-browser-a`, 1) + const ownerErrors: Array = [] + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + internals.sendRPC = vi.fn().mockResolvedValue({ + type: `rpc:ensureRemoteSubset:res`, + rpcId: `initial-browser`, + ok: true, + leaderId: `remote-browser-a`, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + const replayError = new Error(`remote Browser replay transport failed`) + const replayRPC = vi.fn().mockRejectedValue(replayError) + internals.sendRPC = replayRPC + heartbeat(`remote-browser-b`, 2) + await vi.waitFor(() => expect(replayRPC).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + const [acquisition] = + internals.outboundRemoteSubsetAcquisitions.values() + expect(acquisition?.inFlight).toBeNull() + }) + + const [acquisition] = + internals.outboundRemoteSubsetAcquisitions.values() + expect({ + ownerErrors, + attempts: replayRPC.mock.calls.length, + acquiredLeaderId: acquisition?.acquiredLeaderId, + retained: internals.outboundRemoteSubsetAcquisitions.size, + }).toEqual({ + ownerErrors: [replayError], + attempts: 1, + acquiredLeaderId: `remote-browser-a`, + retained: 1, + }) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`ignores a lower-term Browser heartbeat without replaying to its stale leader`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + coordinator.isLeader = () => false + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + collections: Map< + string, + { leaderId: string | null; latestTerm: number } + > + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + const replay = vi.fn(() => Promise.resolve()) + internals.replayRemoteSubsetAcquisitions = replay + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + + try { + heartbeat(`remote-browser-b`, 2) + replay.mockClear() + heartbeat(`retired-browser-a`, 1) + const state = internals.collections.get(`todos`) + + expect({ + leaderId: state?.leaderId, + latestTerm: state?.latestTerm, + replayCalls: replay.mock.calls.length, + }).toEqual({ + leaderId: `remote-browser-b`, + latestTerm: 2, + replayCalls: 0, + }) + } finally { + coordinator.dispose() + } + }) + + it(`rebinds a live Browser acquisition when its owner is replaced`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: vi.fn(), + }) + const unregisterFirst = coordinator.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + + try { + const initialAcquire = coordinator.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = coordinator.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + coordinator.dispose() + } + }) + + it(`rebinds a live remote Browser follower lease when the same leader replaces its owner`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect({ + leader: leader.isLeader(`todos`), + follower: follower.isLeader(`todos`), + }).toEqual({ leader: true, follower: false }) + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + let requestPosts = 0 + observeBroadcastMessage = (message) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + } + + try { + const initialAcquire = follower.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = leader.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect({ + requestPosts, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + }).toEqual({ requestPosts: 1, firstUnloads: 1 }) + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + observeBroadcastMessage = undefined + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`rebinds remote Browser demand when its replacement owner registers after release settlement`, async () => { + const leader = createCoordinator() + const follower = createCoordinator() + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect({ + leader: leader.isLeader(`todos`), + follower: follower.isLeader(`todos`), + }).toEqual({ leader: true, follower: false }) + const firstOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { offset: 22 } + const leaderInternals = leader as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + await follower.requestEnsureRemoteSubset(`todos`, options) + unregisterFirst() + await vi.waitFor(() => + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1), + ) + await flush(0) + + unregisterSecond = leader.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + await flush(0) + + expect({ + firstLoads: firstOwner.mock.calls.length, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + secondLoads: secondOwner.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ + firstLoads: 1, + firstUnloads: 1, + secondLoads: 1, + inbound: 1, + }) + + await follower.requestReleaseRemoteSubset(`todos`, options) + const [terminal] = + leaderInternals.inboundRemoteSubsetAcquisitions.values() + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + expect(secondOwner.unloadSubset.mock.calls[0]?.[0]).toBe( + secondOwner.mock.calls[0]?.[0], + ) + expect({ + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`reacquires the same Browser lease after leadership retirement`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => + expect(owner.unloadSubset).toHaveBeenCalledTimes(1), + ) + + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(owner).toHaveBeenCalledTimes(2) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`starts independent Browser lease replays without sibling head-of-line blocking`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + + type Acquisition = { + collectionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + } + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + const acquisitions = [ + ...internals.outboundRemoteSubsetAcquisitions.values(), + ] + for (const acquisition of acquisitions) { + acquisition.acquiredLeaderId = `retired-browser-leader` + } + let releaseFirstReplay = (): void => {} + const firstReplayGate = new Promise((resolve) => { + releaseFirstReplay = resolve + }) + const starts: Array = [] + internals.acquireRemoteSubset = async (acquisition) => { + starts.push(acquisition.options.offset) + if (acquisition.options.offset === 1) await firstReplayGate + } + + try { + const replay = internals.replayRemoteSubsetAcquisitions(`todos`) + await Promise.resolve() + const startsBeforeFirstFinished = [...starts] + expect(startsBeforeFirstFinished).toEqual([1, 2]) + releaseFirstReplay() + await replay + } finally { + releaseFirstReplay() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports owner unload rejection while completing sibling Browser cleanup`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + const unloadError = new Error(`browser owner unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 1 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map< + string, + { release: Promise | null } + > + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + + try { + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + const releases = [ + ...internals.inboundRemoteSubsetAcquisitions.values(), + ].map((acquisition) => acquisition.release!) + const outcomes = await Promise.allSettled(releases) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + outcomes: outcomes.map((outcome) => + outcome.status === `rejected` ? outcome.reason : outcome.status, + ), + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + }).toEqual({ + outcomes: [unloadError, `fulfilled`], + unloadOffsets: [1, 2], + ownerErrors: [unloadError], + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports local Browser disposal unload rejection once and completes cleanup`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const unloadError = new Error(`browser disposal unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 23 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 23 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 24 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + collections: Map + pendingRPCs: Map + remoteSubsetOwners: Map + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map + } + + try { + coordinator.dispose() + await flush(0) + + expect({ + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + collections: internals.collections.size, + pendingRPCs: internals.pendingRPCs.size, + owners: internals.remoteSubsetOwners.size, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + channelEndpoints: channels.get(`tsdb:coord:test-db`)?.size ?? 0, + }).toEqual({ + unloadOffsets: [23, 24], + ownerErrors: [unloadError], + unhandled: [], + collections: 0, + pendingRPCs: 0, + owners: 0, + outbound: 0, + inbound: 0, + channelEndpoints: 0, + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) }) describe(`RPC - pullSince`, () => { @@ -551,6 +3953,64 @@ describe(`BrowserCollectionCoordinator`, () => { }) describe(`dispose`, () => { + it(`stops Browser RPC retry after disposal and preserves the first failure`, async () => { + vi.useFakeTimers() + const coordinator = createCoordinator() + const firstError = new Error(`browser transport failed during disposal`) + let firstAttempt = true + const internals = coordinator as unknown as { + sendRPC: ( + collectionId: string, + request: { + type: `rpc:pullSince:req` + rpcId: string + fromRowVersion: number + }, + ) => Promise + sendRPCOnce: () => Promise + } + internals.sendRPCOnce = vi.fn(() => { + if (firstAttempt) { + firstAttempt = false + coordinator.dispose() + } + return Promise.reject(firstError) + }) + let outcome: { error: unknown } | undefined + + try { + const pending = internals + .sendRPC(`todos`, { + type: `rpc:pullSince:req`, + rpcId: `browser-disposal-rpc`, + fromRowVersion: 0, + }) + .then( + () => ({ error: undefined }), + (error: unknown) => ({ error }), + ) + .then((result) => { + outcome = result + return result + }) + await vi.advanceTimersByTimeAsync(0) + const immediate = { + calls: vi.mocked(internals.sendRPCOnce).mock.calls.length, + settled: outcome !== undefined, + } + await vi.runAllTimersAsync() + const finalOutcome = await pending + + expect(immediate).toEqual({ calls: 1, settled: true }) + expect(finalOutcome).toEqual({ error: firstError }) + expect(internals.sendRPCOnce).toHaveBeenCalledTimes(1) + } finally { + await vi.runAllTimersAsync() + vi.useRealTimers() + coordinator.dispose() + } + }) + it(`cleans up on dispose`, async () => { const coord = createCoordinator() coord.subscribe(`todos`, () => {}) diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts new file mode 100644 index 0000000000..f2ff086f13 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -0,0 +1,3666 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { setImmediate as nextTurn } from 'node:timers/promises' +import { fc } from '@fast-check/vitest' +import { IR, createCollection } from '@tanstack/db' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { BrowserCollectionCoordinator } from '../src/browser-coordinator' +import { + createBrowserWASQLitePersistence, + persistedCollectionOptions, +} from '../src' +import { createWASQLiteTestDatabase } from './helpers/wa-sqlite-test-db' +import type { + PersistedCollectionCoordinator, + PersistedMutationEnvelope, + PersistedTx, + PersistenceAdapter, + TransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' +import type { LoadSubsetOptions, Subscription, SyncConfig } from '@tanstack/db' +import type { BrowserWASQLiteDatabase } from '../src' + +const seedText = process.env.TANSTACK_DB_COORDINATOR_ORACLE_SEED ?? `165902` +const runText = process.env.TANSTACK_DB_COORDINATOR_ORACLE_RUNS ?? `12` +const SEED = Number(seedText) +const RUNS = Number(runText) +const PATH = process.env.TANSTACK_DB_COORDINATOR_ORACLE_PATH + +if (!/^-?\d+$/.test(seedText) || !Number.isSafeInteger(SEED)) { + throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_SEED`) +} +if (!/^\d+$/.test(runText) || !Number.isSafeInteger(RUNS) || RUNS < 1) { + throw new Error(`Invalid TANSTACK_DB_COORDINATOR_ORACLE_RUNS`) +} +if (PATH !== undefined && !/^\d+(?::\d+)*$/.test(PATH)) { + throw new Error( + `TANSTACK_DB_COORDINATOR_ORACLE_PATH must be a numeric shrink path`, + ) +} + +/* +Coverage owner and test cards +============================= + +This suite owns the RFC #1659 Workstream 2 cut across +BrowserCollectionCoordinator, browser persistence resolution, and the +persisted sync wrapper. browser-coordinator.test.ts is still the focused RPC +unit owner, and db-sqlite-persistence-core/tests/persisted.test.ts is still the +single-runtime owner. Putting this history in either file would replace the +other production boundary with a stub and lose the collection x tab x adapter +identity question. The real SQLite witness below extends the existing browser +runtime helper instead of claiming a new driver model. + +Routing law and source: +- RFC #1659 release invariants 3, 4, and 9; BrowserCollectionCoordinator's + collectionId-bearing PersistedCollectionCoordinator API; issues #1589 and + #1753. Configuration and adapter effects for collection A cannot be handled + by collection B's adapter, before or after leadership transfer. +- Domain: two distinct collections, distinct schema versions/policies, two + tabs, reordered request delivery, local writes, and leader -> follower + takeover. The reference is a Map keyed by (tab, collection), not the + coordinator's mutable adapter slot. +- Production/checkpoint: real coordinator RPC plus Web Locks seam; compare the + adapter identity that applies each collection's transaction after both the + initial and takeover phases. The fixed witness additionally checks real + SQLite row count, registry schema, and reset epoch after merely subscribing. + +Wire/ack law and source: +- RFC invariant 4, LoadSubsetOptions' immutable-request contract, and + LoadSubsetFn's applied-receipt contract; issue #1498. The emitted heartbeat, + RPC, and tx variants exercised by the partition below must survive + structuredClone. Subset expression/cursor/order/window semantics cross the + wire; live signal/subscription ownership stays local. A successful follower + response is permitted only after the leader's exact upstream subset load has + completed. +- Domain: a callback-bearing Subscription witness plus clone-safe expression, + limit, and offset requests. Requests and responses may be held separately. +- Production/checkpoint: BrowserCollectionCoordinator.sendRPC and the public + persisted on-demand leader collection. Record postMessage clone failures, + upstream load entry/completion, and follower settlement at response delivery. + +Write-ownership law and source: +- RFC invariants 3 and 9 and issue #1753. A sync-ingested persistent write has + the same supported-owner requirement as a local write. +- Domain: public source begin/write/commit after a modeled leadership change. +- Production/checkpoint: persistedCollectionOptions' wrapped source commit; + record collection-visible rows, adapter calls, and the owner active at the + exact applyCommittedTx call. + +Reach, challenge, replay, cleanup, and limits: +- Green calibration tests prove structuredClone rejects a function payload, + the leader upstream fixture can really enter/complete, and the ownership + checker rejects an unowned apply. Every RED test records a reached production + checkpoint before comparing the independent ledger. +- FastCheck prints seed/path. Replay with TANSTACK_DB_COORDINATOR_ORACLE_{SEED, + PATH,RUNS}; the thrown report retains the first failing (original) and final + shrunk candidate traces. +- Coordinators/collections/databases are disposed by failure-preserving cleanup. A final + lifecycle test proves no channel, held lock, queued lock, or delayed delivery + remains and that a fresh db name elects normally. +- The BroadcastChannel and Web Locks seams below perform real structuredClone + and real coordinator code but are deterministic Node controls. They do not + earn real-browser, multi-context, OPFS exclusive-handle, worker, Electric, + PowerSync, or service credit. Those cells remain required for GREEN closeout. +*/ + +type MessageHandler = (event: { data: unknown }) => void + +type DelayedDelivery = { + channel: string + source: ControlledBroadcastChannel + target: ControlledBroadcastChannel + data: unknown +} + +type CloneAttempt = { + data: unknown + errorName?: string +} + +class ControlledBroadcastChannel { + static readonly endpoints = new Map>() + static readonly delayed: Array = [] + static readonly cloneAttempts: Array = [] + static retiredDeliveries = 0 + + readonly name: string + onmessage: MessageHandler | null = null + + constructor(name: string) { + this.name = name + const peers = ControlledBroadcastChannel.endpoints.get(name) ?? new Set() + peers.add(this) + ControlledBroadcastChannel.endpoints.set(name, peers) + } + + postMessage(data: unknown): void { + let copy: unknown + try { + copy = structuredClone(data) + ControlledBroadcastChannel.cloneAttempts.push({ data }) + } catch (error) { + ControlledBroadcastChannel.cloneAttempts.push({ + data, + errorName: error instanceof Error ? error.name : String(error), + }) + throw error + } + + for (const target of ControlledBroadcastChannel.endpoints.get(this.name) ?? + []) { + if (target === this || !target.onmessage) continue + ControlledBroadcastChannel.delayed.push({ + channel: this.name, + source: this, + target, + data: copy, + }) + } + } + + close(): void { + const peers = ControlledBroadcastChannel.endpoints.get(this.name) + peers?.delete(this) + if (peers?.size === 0) { + ControlledBroadcastChannel.endpoints.delete(this.name) + } + for ( + let index = ControlledBroadcastChannel.delayed.length - 1; + index >= 0; + index-- + ) { + const delivery = ControlledBroadcastChannel.delayed[index] + if (delivery?.source === this || delivery?.target === this) { + ControlledBroadcastChannel.delayed.splice(index, 1) + ControlledBroadcastChannel.retiredDeliveries++ + } + } + this.onmessage = null + } + + static deliverAt(index: number): boolean { + const [delivery] = this.delayed.splice(index, 1) + if (!delivery) return false + delivery.target.onmessage?.({ data: delivery.data }) + return true + } + + static deliverWhere(predicate: (data: unknown) => boolean): boolean { + const index = this.delayed.findIndex((delivery) => predicate(delivery.data)) + return index >= 0 ? this.deliverAt(index) : false + } + + static reset(): void { + this.endpoints.clear() + this.delayed.length = 0 + this.cloneAttempts.length = 0 + this.retiredDeliveries = 0 + } +} + +type LockCallback = (lock: { name: string }) => Promise +type LockQueueEntry = { + callback: LockCallback + signal?: AbortSignal + resolve: (value: unknown) => void + reject: (error: unknown) => void +} + +const heldLocks = new Set() +const queuedLocks = new Map>() + +function grantNextLock(name: string): void { + if (heldLocks.has(name)) return + const queue = queuedLocks.get(name) + const next = queue?.shift() + if (!next) return + if (queue?.length === 0) queuedLocks.delete(name) + if (next.signal?.aborted) { + grantNextLock(name) + return + } + + heldLocks.add(name) + void Promise.resolve(next.callback({ name })).then( + (value) => { + heldLocks.delete(name) + next.resolve(value) + grantNextLock(name) + }, + (error) => { + heldLocks.delete(name) + next.reject(error) + grantNextLock(name) + }, + ) +} + +const controlledLocks = { + request: ( + name: string, + optionsOrCallback: { signal?: AbortSignal } | LockCallback, + maybeCallback?: LockCallback, + ): Promise => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + const signal = + typeof optionsOrCallback === `object` + ? optionsOrCallback.signal + : undefined + + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException(`Lock request aborted`, `AbortError`)) + return + } + const entry = { callback, signal, resolve, reject } + const queue = queuedLocks.get(name) ?? [] + queue.push(entry) + queuedLocks.set(name, queue) + signal?.addEventListener( + `abort`, + () => { + const current = queuedLocks.get(name) + const index = current?.indexOf(entry) ?? -1 + if (index >= 0) { + current!.splice(index, 1) + if (current!.length === 0) queuedLocks.delete(name) + reject(new DOMException(`Lock request aborted`, `AbortError`)) + } + }, + { once: true }, + ) + grantNextLock(name) + }) + }, +} + +let originalBroadcastChannelDescriptor: PropertyDescriptor | undefined +let originalNavigatorDescriptor: PropertyDescriptor | undefined + +function installControlledBrowserSeams(): void { + originalBroadcastChannelDescriptor = Object.getOwnPropertyDescriptor( + globalThis, + `BroadcastChannel`, + ) + originalNavigatorDescriptor = Object.getOwnPropertyDescriptor( + globalThis, + `navigator`, + ) + Object.defineProperty(globalThis, `BroadcastChannel`, { + value: ControlledBroadcastChannel, + writable: true, + configurable: true, + }) + Object.defineProperty(globalThis, `navigator`, { + value: { locks: controlledLocks }, + writable: true, + configurable: true, + }) +} + +function restoreControlledBrowserSeams(): void { + if (originalBroadcastChannelDescriptor) { + Object.defineProperty( + globalThis, + `BroadcastChannel`, + originalBroadcastChannelDescriptor, + ) + } else { + Reflect.deleteProperty(globalThis, `BroadcastChannel`) + } + if (originalNavigatorDescriptor) { + Object.defineProperty(globalThis, `navigator`, originalNavigatorDescriptor) + } else { + Reflect.deleteProperty(globalThis, `navigator`) + } + originalBroadcastChannelDescriptor = undefined + originalNavigatorDescriptor = undefined +} + +function payloadType(data: unknown): string | undefined { + if (!data || typeof data !== `object` || !(`payload` in data)) + return undefined + const payload = (data as { payload?: unknown }).payload + if (!payload || typeof payload !== `object` || !(`type` in payload)) { + return undefined + } + return String((payload as { type?: unknown }).type) +} + +function envelopeCollection(data: unknown): string | undefined { + if (!data || typeof data !== `object` || !(`collectionId` in data)) { + return undefined + } + return String((data as { collectionId?: unknown }).collectionId) +} + +async function pumpNetwork(): Promise { + for (let pass = 0; pass < 100; pass++) { + while (ControlledBroadcastChannel.deliverAt(0)) { + await Promise.resolve() + } + await nextTurn() + if (ControlledBroadcastChannel.delayed.length === 0) return + } + throw new Error(`controlled network failed to become idle`) +} + +async function waitFor( + predicate: () => boolean, + checkpoint: string, +): Promise { + for (let pass = 0; pass < 100; pass++) { + if (predicate()) return + await pumpNetwork() + await nextTurn() + } + throw new Error(`did not reach ${checkpoint}`) +} + +async function waitForWithoutDelivery( + predicate: () => boolean, + checkpoint: string, +): Promise { + for (let pass = 0; pass < 100; pass++) { + if (predicate()) return + await nextTurn() + } + throw new Error(`did not reach ${checkpoint}`) +} + +type CleanupDiagnostic = { + resource: string + error: string +} + +const NO_PRIMARY_FAILURE = Symbol(`no-primary-failure`) + +async function captureCleanup( + diagnostics: Array, + resource: string, + cleanup: () => Promise | void, +): Promise { + try { + await cleanup() + } catch (error) { + diagnostics.push({ + resource, + error: + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + }) + } +} + +async function withFailurePreservingCleanup( + body: () => Promise, + cleanups: ReadonlyArray Promise | void]>, +): Promise { + let primaryFailure: unknown | typeof NO_PRIMARY_FAILURE = NO_PRIMARY_FAILURE + const cleanupDiagnostics: Array = [] + try { + await body() + } catch (error) { + primaryFailure = error + } + for (const [resource, cleanup] of cleanups) { + await captureCleanup(cleanupDiagnostics, resource, cleanup) + } + expect.soft(cleanupDiagnostics).toEqual([]) + if (primaryFailure !== NO_PRIMARY_FAILURE) throw primaryFailure +} + +type AdapterCall = { + operation: `load` | `apply` | `index` | `position` | `remote-subset` | `pull` + adapterId: string + collectionId: string + ownerAtCall?: string + argument?: SemanticValue +} + +type PullFixtureResult = { + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array +} + +type RecordedSubsetOptions = LoadSubsetOptions | TransportedLoadSubsetOptions + +type RecordingAdapter = PersistenceAdapter & { + id: string + schemaVersion: number + policy: `sync-present-reset` | `sync-absent-error` + calls: Array + getStreamPosition: (collectionId: string) => Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> + ensureRemoteSubset: ( + collectionId: string, + options: RecordedSubsetOptions, + ) => Promise + pullSince: ( + collectionId: string, + fromRowVersion: number, + ) => Promise +} + +function createRecordingAdapter(options: { + id: string + schemaVersion?: number + policy?: RecordingAdapter[`policy`] + owner?: () => string | undefined + remoteSubsetOwner?: ( + collectionId: string, + options: RecordedSubsetOptions, + ) => Promise + pullResult?: PullFixtureResult +}): RecordingAdapter { + const calls: Array = [] + const record = ( + operation: AdapterCall[`operation`], + collectionId: string, + argument?: SemanticValue, + ) => { + calls.push({ + operation, + adapterId: options.id, + collectionId, + ownerAtCall: options.owner?.(), + ...(argument === undefined ? {} : { argument }), + }) + } + + return { + id: options.id, + schemaVersion: options.schemaVersion ?? 1, + policy: options.policy ?? `sync-present-reset`, + calls, + loadSubset: (collectionId) => { + record(`load`, collectionId) + return Promise.resolve([]) + }, + applyCommittedTx: (collectionId) => { + record(`apply`, collectionId) + return Promise.resolve() + }, + ensureIndex: (collectionId, signature, spec) => { + record(`index`, collectionId, semanticValue({ signature, spec })) + return Promise.resolve() + }, + getStreamPosition: (collectionId) => { + record(`position`, collectionId) + return Promise.resolve({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }) + }, + ensureRemoteSubset: (collectionId, subsetOptions) => { + record(`remote-subset`, collectionId, subsetSemantics(subsetOptions)) + if (!options.remoteSubsetOwner) { + throw new Error(`no remote subset owner for ${collectionId}`) + } + return options.remoteSubsetOwner(collectionId, subsetOptions) + }, + pullSince: (collectionId, fromRowVersion) => { + record(`pull`, collectionId, semanticValue({ fromRowVersion })) + const result = options.pullResult ?? { + latestRowVersion: 0, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + } + return Promise.resolve(structuredClone(result)) + }, + } +} + +type SemanticValue = + | null + | boolean + | number + | string + | Array + | { [key: string]: SemanticValue } + +function semanticValue(value: unknown): SemanticValue { + if ( + value === null || + typeof value === `boolean` || + typeof value === `number` || + typeof value === `string` + ) { + return value + } + if (value instanceof Date) return { date: value.toISOString() } + if (value instanceof Uint8Array) return { bytes: Array.from(value) } + if (Array.isArray(value)) return value.map(semanticValue) + if (typeof value === `object`) { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, semanticValue(entry)]), + ) + } + throw new Error(`unsupported semantic wire value: ${typeof value}`) +} + +function subsetSemantics( + options: LoadSubsetOptions | TransportedLoadSubsetOptions, +): SemanticValue { + return semanticValue({ + ...(options.where === undefined ? {} : { where: options.where }), + ...(options.orderBy === undefined ? {} : { orderBy: options.orderBy }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.cursor === undefined ? {} : { cursor: options.cursor }), + ...(options.offset === undefined ? {} : { offset: options.offset }), + }) +} + +function complexSubsetDemand(includeSubscription: boolean): LoadSubsetOptions { + return { + where: new IR.Func(`eq`, [ + new IR.PropRef([`alpha`, `createdAt`]), + new IR.Value(new Date(`2026-09-16T12:34:56.000Z`)), + ]), + orderBy: [ + { + expression: new IR.PropRef([`alpha`, `score`]), + compareOptions: { direction: `desc`, nulls: `last` }, + }, + ], + limit: 3, + cursor: { + whereFrom: new IR.Func(`gt`, [ + new IR.PropRef([`alpha`, `token`]), + new IR.Value(new Uint8Array([1, 2, 255])), + ]), + whereCurrent: new IR.Func(`eq`, [ + new IR.PropRef([`alpha`, `score`]), + new IR.Value(42), + ]), + lastKey: `a-42`, + }, + offset: 1, + ...(includeSubscription + ? { + subscription: { + on: () => () => {}, + } as unknown as Subscription, + } + : {}), + } +} + +const liveCoordinators = new Set() + +function createCoordinator( + dbName: string, + adapter: PersistenceAdapter, +): BrowserCollectionCoordinator { + const coordinator = new BrowserCollectionCoordinator({ dbName, adapter }) + liveCoordinators.add(coordinator) + return coordinator +} + +function registerCollectionAdapter( + coordinator: BrowserCollectionCoordinator, + collectionId: string, + adapter: RecordingAdapter, + registerOwner: boolean = true, +): void { + coordinator.setAdapterForCollection(collectionId, adapter) + if (!registerOwner) return + coordinator.registerRemoteSubsetOwner( + collectionId, + Object.assign( + (options: TransportedLoadSubsetOptions) => + adapter.ensureRemoteSubset(collectionId, options), + { + // These route/wire laws do not exercise lease release semantics; the + // dedicated remote-ownership oracle below owns exact unload behavior. + unloadSubset: () => {}, + onError: () => {}, + }, + ), + ) +} + +async function disposeCoordinator( + coordinator?: BrowserCollectionCoordinator, +): Promise { + if (!coordinator || !liveCoordinators.has(coordinator)) return + coordinator.dispose() + liveCoordinators.delete(coordinator) + await nextTurn() +} + +function coordinatorFixtureSnapshot() { + return { + delayed: ControlledBroadcastChannel.delayed.length, + endpoints: ControlledBroadcastChannel.endpoints.size, + heldLocks: heldLocks.size, + queuedLocks: Array.from(queuedLocks.values()).reduce( + (count, queue) => count + queue.length, + 0, + ), + } +} + +beforeEach(() => { + installControlledBrowserSeams() +}) + +afterEach(async () => { + const cleanupDiagnostics: Array = [] + for (const coordinator of liveCoordinators) { + await captureCleanup(cleanupDiagnostics, `afterEach live coordinator`, () => + coordinator.dispose(), + ) + } + liveCoordinators.clear() + await captureCleanup( + cleanupDiagnostics, + `afterEach lock release`, + async () => { + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `afterEach lock release`, + ) + }, + ) + const cleanupSnapshot = coordinatorFixtureSnapshot() + expect.soft(cleanupSnapshot).toEqual({ + delayed: 0, + endpoints: 0, + heldLocks: 0, + queuedLocks: 0, + }) + await captureCleanup( + cleanupDiagnostics, + `global browser descriptors`, + restoreControlledBrowserSeams, + ) + await captureCleanup(cleanupDiagnostics, `vitest timers and mocks`, () => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + ControlledBroadcastChannel.reset() + heldLocks.clear() + queuedLocks.clear() + expect.soft(cleanupDiagnostics).toEqual([]) +}) + +type SubsetAckTrace = { + ownerInvocation: `coordinator` | `fixture-fallback` + decoded: SemanticValue | undefined + expected: SemanticValue + prematureResponses: number + acknowledgedBeforeResponseDelivery: boolean + ownerCompletedBeforeResponse: boolean + acknowledgedAfterResponseDelivery: boolean +} + +type SubsetWireTrace = { + outcome: { ok: true } | { ok: false; errorName: string } | undefined + decoded: ReadonlyArray + expected: SemanticValue +} + +function subsetWireViolations(trace: SubsetWireTrace): Array { + const violations: Array = [] + if (trace.outcome?.ok !== true) violations.push(`subset-transport-failed`) + if ( + trace.decoded.length !== 1 || + JSON.stringify(trace.decoded[0]) !== JSON.stringify(trace.expected) + ) { + violations.push(`subset-semantics-mismatch`) + } + return violations +} + +function subsetAckViolations(trace: SubsetAckTrace): Array { + const violations: Array = [] + if (trace.ownerInvocation !== `coordinator`) { + violations.push(`leader-owner-not-invoked-by-coordinator`) + } + if (JSON.stringify(trace.decoded) !== JSON.stringify(trace.expected)) { + violations.push(`decoded-subset-mismatch`) + } + if (trace.prematureResponses !== 0) { + violations.push(`response-before-owner-completion`) + } + if (trace.acknowledgedBeforeResponseDelivery) { + violations.push(`ack-before-response-delivery`) + } + if (!trace.ownerCompletedBeforeResponse) { + violations.push(`response-before-owner-completion-checkpoint`) + } + if (!trace.acknowledgedAfterResponseDelivery) { + violations.push(`missing-ack-after-response`) + } + return violations +} + +async function observeOwnerEntryOrResponse( + ownerEntered: () => boolean, +): Promise<`owner-entered` | `response-queued` | `no-progress`> { + for (let turn = 0; turn < 20; turn++) { + if (ownerEntered()) return `owner-entered` + if ( + ControlledBroadcastChannel.delayed.some( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:res`, + ) + ) { + return `response-queued` + } + await nextTurn() + } + return `no-progress` +} + +describe(`per-collection coordinator wire and acknowledgement oracle`, () => { + it(`posts the callback-bearing subset witness as clone-safe wire data`, async () => { + const decoded: Array = [] + const leaderAdapter = createRecordingAdapter({ + id: `leader`, + remoteSubsetOwner: (_collectionId, options) => { + decoded.push(subsetSemantics(options)) + return Promise.resolve() + }, + }) + const followerAdapter = createRecordingAdapter({ id: `follower` }) + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + leader = createCoordinator(`clone-witness`, leaderAdapter) + follower = createCoordinator(`clone-witness`, followerAdapter) + registerCollectionAdapter(leader, `alpha`, leaderAdapter) + registerCollectionAdapter(follower, `alpha`, followerAdapter) + leader.subscribe(`alpha`, () => {}) + follower.subscribe(`alpha`, () => {}) + await waitFor( + () => leader!.isLeader(`alpha`) && !follower!.isLeader(`alpha`), + `clone-witness leader/follower topology`, + ) + const safeTransportBeforeRequest = + ControlledBroadcastChannel.cloneAttempts.filter( + (attempt) => attempt.errorName === undefined, + ).length + expect(safeTransportBeforeRequest).toBeGreaterThan(0) + + const options = complexSubsetDemand(true) + vi.useFakeTimers() + let outcome: { ok: true } | { ok: false; errorName: string } | undefined + void follower.requestEnsureRemoteSubset(`alpha`, options).then( + () => { + outcome = { ok: true } + }, + (error: unknown) => { + outcome = { + ok: false, + errorName: error instanceof Error ? error.name : String(error), + } + }, + ) + for (let step = 0; step < 12 && outcome === undefined; step++) { + while (ControlledBroadcastChannel.deliverAt(0)) { + await Promise.resolve() + } + await vi.advanceTimersByTimeAsync(100) + } + + expect(outcome).toBeDefined() + // Preserve the current-main DataCloneError as an observation without + // making the defect a positive reach requirement for the GREEN suite. + const observedCloneFault = ControlledBroadcastChannel.cloneAttempts.find( + (attempt) => attempt.errorName !== undefined, + ) + if (outcome?.ok === false) { + expect(outcome.errorName).toBe(observedCloneFault?.errorName) + } + // RED D3: after clone-safe projection, the registered leader owner must + // receive the exact supported semantics and the follower call succeeds. + expect( + subsetWireViolations({ + outcome, + decoded, + expected: subsetSemantics(options), + }), + ).toEqual([]) + }, [ + [ + `clone witness leader`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `clone witness follower`, + async () => { + await disposeCoordinator(follower) + }, + ], + [ + `clone witness timers`, + async () => { + vi.clearAllTimers() + vi.useRealTimers() + await nextTurn() + }, + ], + ]) + }) + + it(`calibrates the clone checker against safe and function-valued payloads`, () => { + const safe = { + collectionId: `alpha`, + where: { type: `eq`, field: `group`, value: `kept` }, + limit: 2, + } + const unsafe = { ...safe, callback: () => {} } + + expect(structuredClone(safe)).toEqual(safe) + expect(() => structuredClone(unsafe)).toThrowError( + expect.objectContaining({ name: `DataCloneError` }), + ) + + const expected = subsetSemantics(complexSubsetDemand(false)) + const allowed: SubsetWireTrace = { + outcome: { ok: true }, + decoded: [expected], + expected, + } + expect(subsetWireViolations(allowed)).toEqual([]) + expect( + subsetWireViolations({ + ...allowed, + outcome: { ok: false, errorName: `DataCloneError` }, + }), + ).toContain(`subset-transport-failed`) + expect( + subsetWireViolations({ ...allowed, decoded: [semanticValue({})] }), + ).toContain(`subset-semantics-mismatch`) + }) + + it(`round-trips supported subset semantics across the emitted message partition`, async () => { + const leaderAdapter = createRecordingAdapter({ + id: `partition-leader`, + remoteSubsetOwner: () => Promise.resolve(), + }) + const followerAdapter = createRecordingAdapter({ id: `partition-follower` }) + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + leader = createCoordinator(`wire-partition`, leaderAdapter) + follower = createCoordinator(`wire-partition`, followerAdapter) + registerCollectionAdapter(leader, `alpha`, leaderAdapter) + registerCollectionAdapter(follower, `alpha`, followerAdapter) + leader.subscribe(`alpha`, () => {}) + follower.subscribe(`alpha`, () => {}) + await waitFor( + () => leader!.isLeader(`alpha`) && !follower!.isLeader(`alpha`), + `wire partition leader/follower topology`, + ) + + const demand = complexSubsetDemand(false) + const subsetRequest = follower.requestEnsureRemoteSubset(`alpha`, demand) + const queuedSubset = ControlledBroadcastChannel.delayed.find( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:req`, + )?.data as { payload?: { options?: LoadSubsetOptions } } | undefined + expect(subsetSemantics(queuedSubset?.payload?.options ?? {})).toEqual( + subsetSemantics(demand), + ) + await pumpNetwork() + await subsetRequest + + const mutationValue = { + id: `native-values`, + at: new Date(`2026-09-16T12:34:56.000Z`), + bytes: new Uint8Array([0, 127, 255]), + } + const mutationRequest = follower.requestApplyLocalMutations(`alpha`, [ + { + mutationId: `native-values`, + type: `insert`, + key: `native-values`, + value: mutationValue, + }, + ]) + const queuedMutation = ControlledBroadcastChannel.delayed.find( + ({ data }) => payloadType(data) === `rpc:applyLocalMutations:req`, + )?.data as + | { + payload?: { + mutations?: Array<{ value?: unknown }> + } + } + | undefined + expect( + semanticValue(queuedMutation?.payload?.mutations?.[0]?.value), + ).toEqual(semanticValue(mutationValue)) + await pumpNetwork() + await mutationRequest + + const committedTx: PersistedTx = { + txId: `committed-native-values`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `update`, + key: `native-values`, + value: mutationValue, + }, + ], + rowMetadataMutations: [ + { + type: `set`, + key: `native-values`, + value: { source: `wire-partition` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 4 } }, + ], + } + const committedRequest = follower.requestApplyCommittedTx( + `alpha`, + committedTx, + ) + const queuedCommitted = ControlledBroadcastChannel.delayed.find( + ({ data }) => payloadType(data) === `rpc:applyCommittedTx:req`, + )?.data as { payload?: { tx?: PersistedTx } } | undefined + expect(semanticValue(queuedCommitted?.payload?.tx)).toEqual( + semanticValue(committedTx), + ) + await pumpNetwork() + const committedResponse = await committedRequest + expect(committedResponse).toEqual({ + type: `rpc:applyCommittedTx:res`, + rpcId: expect.any(String), + ok: true, + term: 1, + seq: 2, + latestRowVersion: 2, + }) + + const successfulPayloads = ControlledBroadcastChannel.cloneAttempts + .filter(({ errorName }) => errorName === undefined) + .map(({ data }) => + data && typeof data === `object` && `payload` in data + ? (data as { payload?: unknown }).payload + : undefined, + ) + const matchingCommittedResponses = successfulPayloads.filter( + (payload) => + payload !== undefined && + payload !== null && + typeof payload === `object` && + `type` in payload && + payload.type === `rpc:applyCommittedTx:res` && + `rpcId` in payload && + payload.rpcId === committedResponse.rpcId, + ) + const matchingCommittedEvents = successfulPayloads.filter( + (payload) => + payload !== undefined && + payload !== null && + typeof payload === `object` && + `type` in payload && + payload.type === `tx:committed` && + `txId` in payload && + payload.txId === committedTx.txId, + ) + expect(matchingCommittedResponses).toEqual([committedResponse]) + expect(matchingCommittedEvents).toEqual([ + { + type: `tx:committed`, + term: 1, + seq: 2, + txId: `committed-native-values`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [{ key: `native-values`, value: mutationValue }], + deletedKeys: [], + rowMetadataMutations: [ + { + type: `set`, + key: `native-values`, + value: { source: `wire-partition` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 4 } }, + ], + }, + ]) + + const indexRequest = follower.requestEnsurePersistedIndex( + `alpha`, + `score-index`, + { expressionSql: [`score`] }, + ) + await pumpNetwork() + await indexRequest + const pullRequest = follower.pullSince(`alpha`, 0) + await pumpNetwork() + await pullRequest + + const successfulTypes = new Set( + ControlledBroadcastChannel.cloneAttempts + .filter(({ errorName }) => errorName === undefined) + .map(({ data }) => payloadType(data)) + .filter((type): type is string => type !== undefined), + ) + expect(successfulTypes).toEqual( + new Set([ + `leader:heartbeat`, + `rpc:ensureRemoteSubset:req`, + `rpc:ensureRemoteSubset:res`, + `rpc:applyLocalMutations:req`, + `rpc:applyLocalMutations:res`, + `rpc:applyCommittedTx:req`, + `rpc:applyCommittedTx:res`, + `tx:committed`, + `rpc:ensurePersistedIndex:req`, + `rpc:ensurePersistedIndex:res`, + `rpc:pullSince:req`, + `rpc:pullSince:res`, + ]), + ) + }, [ + [ + `wire partition leader`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `wire partition follower`, + async () => { + await disposeCoordinator(follower) + }, + ], + ]) + }) + + it(`calibrates acknowledgement checks against entry-only and false-success faults`, () => { + const expected = subsetSemantics(complexSubsetDemand(false)) + const allowed: SubsetAckTrace = { + ownerInvocation: `coordinator`, + decoded: expected, + expected, + prematureResponses: 0, + acknowledgedBeforeResponseDelivery: false, + ownerCompletedBeforeResponse: true, + acknowledgedAfterResponseDelivery: true, + } + expect(subsetAckViolations(allowed)).toEqual([]) + expect( + subsetAckViolations({ + ...allowed, + ownerCompletedBeforeResponse: false, + }), + ).toContain(`response-before-owner-completion-checkpoint`) + expect( + subsetAckViolations({ + ...allowed, + ownerInvocation: `fixture-fallback`, + prematureResponses: 1, + }), + ).toEqual([ + `leader-owner-not-invoked-by-coordinator`, + `response-before-owner-completion`, + ]) + }) + + it(`acknowledges a follower subset only after the leader source completed it`, async () => { + const upstreamEvents: Array<{ + phase: `entered` | `completed` + semantics: SemanticValue + }> = [] + const releases: Array<() => void> = [] + const ownerDelegate: { + invoke?: ( + collectionId: string, + options: RecordedSubsetOptions, + ) => Promise + } = {} + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + let cleanupLeaderCollection: (() => Promise) | undefined + + await withFailurePreservingCleanup(async () => { + const leaderAdapter = createRecordingAdapter({ + id: `leader-alpha`, + remoteSubsetOwner: (collectionId, options) => { + if (!ownerDelegate.invoke) { + throw new Error(`leader owner not attached`) + } + return ownerDelegate.invoke(collectionId, options) + }, + }) + const followerAdapter = createRecordingAdapter({ + id: `follower-alpha`, + }) + leader = createCoordinator(`subset-ack`, leaderAdapter) + follower = createCoordinator(`subset-ack`, followerAdapter) + registerCollectionAdapter(leader, `alpha`, leaderAdapter, false) + registerCollectionAdapter(follower, `alpha`, followerAdapter, false) + const source: SyncConfig<{ id: string; group: string }, string> = { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: (options) => { + upstreamEvents.push({ + phase: `entered`, + semantics: subsetSemantics(options), + }) + return new Promise((resolve) => { + let released = false + releases.push(() => { + if (released) return + released = true + upstreamEvents.push({ + phase: `completed`, + semantics: subsetSemantics(options), + }) + resolve() + }) + }) + }, + } + }, + } + const leaderCollection = createCollection( + persistedCollectionOptions({ + id: `alpha`, + getKey: (row: { id: string; group: string }) => row.id, + syncMode: `on-demand`, + sync: source, + persistence: { adapter: leaderAdapter, coordinator: leader }, + }), + ) + cleanupLeaderCollection = () => leaderCollection.cleanup() + ownerDelegate.invoke = async (collectionId, options) => { + if (collectionId !== `alpha`) { + throw new Error(`wrong leader subset collection ${collectionId}`) + } + await Promise.resolve( + leaderCollection._sync.loadSubset( + options as unknown as LoadSubsetOptions, + ), + ) + } + + leaderCollection.startSyncImmediate() + follower.subscribe(`alpha`, () => {}) + await waitFor( + () => leader!.isLeader(`alpha`) && !follower!.isLeader(`alpha`), + `subset-ack leader/follower topology`, + ) + + const demand = complexSubsetDemand(false) + const expectedSemantics = subsetSemantics(demand) + let acknowledged = false + const acknowledgement = follower + .requestEnsureRemoteSubset(`alpha`, demand) + .then(() => { + acknowledged = true + }) + + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === `rpc:ensureRemoteSubset:req` && + envelopeCollection(data) === `alpha`, + ), + ).toBe(true) + const firstProgress = await observeOwnerEntryOrResponse(() => + upstreamEvents.some(({ phase }) => phase === `entered`), + ) + expect(firstProgress).not.toBe(`no-progress`) + const prematureResponses = ControlledBroadcastChannel.delayed.filter( + ({ data }) => + payloadType(data) === `rpc:ensureRemoteSubset:res` && + envelopeCollection(data) === `alpha`, + ) + let ownerInvocation: SubsetAckTrace[`ownerInvocation`] = `coordinator` + let ownerWork: Promise | undefined + if (!upstreamEvents.some(({ phase }) => phase === `entered`)) { + ownerInvocation = `fixture-fallback` + for (const response of prematureResponses) { + const index = ControlledBroadcastChannel.delayed.indexOf(response) + if (index >= 0) ControlledBroadcastChannel.delayed.splice(index, 1) + } + ownerWork = leaderAdapter.ensureRemoteSubset(`alpha`, demand) + await waitForWithoutDelivery( + () => upstreamEvents.some(({ phase }) => phase === `entered`), + `fallback leader owner entry`, + ) + } + const entered = upstreamEvents.find(({ phase }) => phase === `entered`) + expect( + ControlledBroadcastChannel.delayed.some( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:res`, + ), + ).toBe(false) + expect(acknowledged).toBe(false) + releases.at(-1)?.() + await ownerWork + await waitForWithoutDelivery( + () => upstreamEvents.some(({ phase }) => phase === `completed`), + `leader subset completion`, + ) + if (ownerInvocation === `fixture-fallback`) { + ControlledBroadcastChannel.delayed.push(...prematureResponses) + } else { + await waitForWithoutDelivery( + () => + ControlledBroadcastChannel.delayed.some( + ({ data }) => payloadType(data) === `rpc:ensureRemoteSubset:res`, + ), + `response after leader subset completion`, + ) + } + const acknowledgedBeforeResponseDelivery = acknowledged + expect(acknowledged).toBe(false) + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === `rpc:ensureRemoteSubset:res` && + envelopeCollection(data) === `alpha`, + ), + ).toBe(true) + await acknowledgement + const trace: SubsetAckTrace = { + ownerInvocation, + decoded: entered?.semantics, + expected: expectedSemantics, + prematureResponses: prematureResponses.length, + acknowledgedBeforeResponseDelivery, + ownerCompletedBeforeResponse: upstreamEvents.some( + ({ phase }) => phase === `completed`, + ), + acknowledgedAfterResponseDelivery: acknowledged, + } + // RED D13: one comparator owns both the production verdict and hostile + // entry-only/false-success calibrations above. + expect(subsetAckViolations(trace)).toEqual([]) + }, [ + [ + `pending subset source releases`, + () => { + for (const release of releases) release() + }, + ], + [ + `leader collection`, + async () => { + await cleanupLeaderCollection?.() + }, + ], + [ + `leader coordinator`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `follower coordinator`, + async () => { + await disposeCoordinator(follower) + }, + ], + ]) + }) +}) + +type RemoteLeaseEvent = { + phase: `load` | `unload` + token: number + semantics: SemanticValue + exactLoadedObject?: boolean +} + +class RemoteLeaseOwnerLedger { + private nextToken = 0 + private readonly tokens = new WeakMap() + private readonly loaded = new WeakSet() + readonly active = new Set() + readonly events: Array = [] + + private token(options: TransportedLoadSubsetOptions): number { + let token = this.tokens.get(options) + if (token === undefined) { + token = ++this.nextToken + this.tokens.set(options, token) + } + return token + } + + readonly loadSubset = async ( + options: TransportedLoadSubsetOptions, + ): Promise => { + const token = this.token(options) + this.loaded.add(options) + this.active.add(options) + this.events.push({ + phase: `load`, + token, + semantics: subsetSemantics(options as unknown as LoadSubsetOptions), + }) + } + + readonly unloadSubset = (options: TransportedLoadSubsetOptions): void => { + const token = this.token(options) + const exactLoadedObject = this.loaded.has(options) + this.active.delete(options) + this.events.push({ + phase: `unload`, + token, + semantics: subsetSemantics(options as unknown as LoadSubsetOptions), + exactLoadedObject, + }) + } + + /** Callable for today's owner boundary; properties are the crash-only owner. */ + readonly boundary = Object.assign( + (options: TransportedLoadSubsetOptions) => this.loadSubset(options), + { + loadSubset: this.loadSubset, + unloadSubset: this.unloadSubset, + onError: () => {}, + }, + ) + + snapshot() { + return { + loads: this.events.filter(({ phase }) => phase === `load`).length, + unloads: this.events.filter(({ phase }) => phase === `unload`).length, + active: this.active.size, + exactUnloads: this.events + .filter( + (event): event is RemoteLeaseEvent & { phase: `unload` } => + event.phase === `unload`, + ) + .every(({ exactLoadedObject }) => exactLoadedObject === true), + } + } +} + +type RemoteLeaseTakeoverTrace = { + initialOwnerAfterAcquire: ReturnType + retiredOwnerAfterTakeover: ReturnType + newOwnerAfterTakeover: ReturnType + newOwnerAfterRelease: ReturnType +} + +type RemoteLeaseTakeoverViolation = { + checkpoint: string + field: keyof ReturnType + expected: number | boolean + actual: number | boolean +} + +const expectedRemoteLeaseTakeoverTrace: RemoteLeaseTakeoverTrace = { + initialOwnerAfterAcquire: { + loads: 1, + unloads: 0, + active: 1, + exactUnloads: true, + }, + retiredOwnerAfterTakeover: { + loads: 1, + unloads: 1, + active: 0, + exactUnloads: true, + }, + newOwnerAfterTakeover: { + loads: 1, + unloads: 0, + active: 1, + exactUnloads: true, + }, + newOwnerAfterRelease: { + loads: 1, + unloads: 1, + active: 0, + exactUnloads: true, + }, +} + +function remoteLeaseTakeoverViolations( + actual: RemoteLeaseTakeoverTrace, +): Array { + const violations: Array = [] + for (const checkpoint of Object.keys( + expectedRemoteLeaseTakeoverTrace, + ) as Array) { + const expected = expectedRemoteLeaseTakeoverTrace[checkpoint] + for (const field of Object.keys(expected) as Array) { + if (actual[checkpoint][field] !== expected[field]) { + violations.push({ + checkpoint, + field, + expected: expected[field], + actual: actual[checkpoint][field], + }) + } + } + } + return violations +} + +type CollectionScopedLeaseSnapshot = Record< + `alpha` | `beta`, + { loads: number; unloads: number; active: number } +> + +function collectionScopedLeaseViolations( + actual: CollectionScopedLeaseSnapshot, + expected: CollectionScopedLeaseSnapshot, +): Array { + const violations: Array = [] + for (const collectionId of [`alpha`, `beta`] as const) { + for (const field of [`loads`, `unloads`, `active`] as const) { + if (actual[collectionId][field] !== expected[collectionId][field]) { + violations.push(`${collectionId}:${field}`) + } + } + } + return violations +} + +type RemoteLeaseHistory = { + siblingCount: number + releasePrefix: number + identical: boolean + duplicateRelease: boolean + reverseFinalRelease: boolean +} + +type RemoteLeaseHistoryTrace = { + firstAfterAcquire: ReturnType + firstAfterPrefixRelease: ReturnType + firstAfterTakeover: ReturnType + secondAfterTakeover: ReturnType + secondAfterFinalRelease: ReturnType +} + +const remoteLeaseHistoryArbitrary = fc.record({ + siblingCount: fc.integer({ min: 1, max: 3 }), + releasePrefix: fc.integer({ min: 0, max: 2 }), + identical: fc.boolean(), + duplicateRelease: fc.boolean(), + reverseFinalRelease: fc.boolean(), +}) + +function remoteLeaseHistoryViolations( + history: RemoteLeaseHistory, + actual: RemoteLeaseHistoryTrace, +): Array { + const releaseCount = Math.min(history.releasePrefix, history.siblingCount - 1) + const remaining = history.siblingCount - releaseCount + const expected: RemoteLeaseHistoryTrace = { + firstAfterAcquire: { + loads: history.siblingCount, + unloads: 0, + active: history.siblingCount, + exactUnloads: true, + }, + firstAfterPrefixRelease: { + loads: history.siblingCount, + unloads: releaseCount, + active: remaining, + exactUnloads: true, + }, + firstAfterTakeover: { + loads: history.siblingCount, + unloads: history.siblingCount, + active: 0, + exactUnloads: true, + }, + secondAfterTakeover: { + loads: remaining, + unloads: 0, + active: remaining, + exactUnloads: true, + }, + secondAfterFinalRelease: { + loads: remaining, + unloads: remaining, + active: 0, + exactUnloads: true, + }, + } + const violations: Array = [] + for (const checkpoint of Object.keys(expected) as Array< + keyof RemoteLeaseHistoryTrace + >) { + for (const field of Object.keys(expected[checkpoint]) as Array< + keyof ReturnType + >) { + if (actual[checkpoint][field] !== expected[checkpoint][field]) { + violations.push({ + checkpoint, + field, + expected: expected[checkpoint][field], + actual: actual[checkpoint][field], + }) + } + } + } + return violations +} + +describe(`remote subset ownership lease oracle`, () => { + it(`calibrates collection-scoped acquisition identity against a global-id mutant`, () => { + const expected: CollectionScopedLeaseSnapshot = { + alpha: { loads: 1, unloads: 0, active: 1 }, + beta: { loads: 1, unloads: 0, active: 1 }, + } + expect(collectionScopedLeaseViolations(expected, expected)).toEqual([]) + expect( + collectionScopedLeaseViolations( + { + alpha: { loads: 2, unloads: 0, active: 2 }, + beta: { loads: 1, unloads: 0, active: 1 }, + }, + expected, + ), + ).toEqual([`alpha:loads`, `alpha:active`]) + }) + + it(`keeps reused option identities independent per collection across bounded histories`, async () => { + type Action = { + kind: `ensure` | `release` + collectionId: `alpha` | `beta` + objectId: 0 | 1 + } + const histories: ReadonlyArray<{ + name: string + actions: ReadonlyArray + }> = [ + { + name: `shared object A-B-A`, + actions: [ + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `ensure`, collectionId: `beta`, objectId: 0 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 0 }, + ], + }, + { + name: `released object reacquired beside sibling collection`, + actions: [ + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `ensure`, collectionId: `beta`, objectId: 0 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 0 }, + ], + }, + { + name: `two identities with duplicate releases`, + actions: [ + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 1 }, + { kind: `ensure`, collectionId: `beta`, objectId: 0 }, + { kind: `ensure`, collectionId: `beta`, objectId: 1 }, + { kind: `ensure`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 1 }, + { kind: `release`, collectionId: `beta`, objectId: 1 }, + { kind: `release`, collectionId: `alpha`, objectId: 1 }, + { kind: `release`, collectionId: `alpha`, objectId: 0 }, + { kind: `release`, collectionId: `beta`, objectId: 0 }, + ], + }, + ] + + for (const [historyIndex, history] of histories.entries()) { + const owners = { + alpha: new RemoteLeaseOwnerLedger(), + beta: new RemoteLeaseOwnerLedger(), + } + const live = { + alpha: new Set(), + beta: new Set(), + } + const expected: CollectionScopedLeaseSnapshot = { + alpha: { loads: 0, unloads: 0, active: 0 }, + beta: { loads: 0, unloads: 0, active: 0 }, + } + const coordinator = createCoordinator( + `collection-scoped-lease-${historyIndex}`, + createRecordingAdapter({ id: `collection-scoped-${historyIndex}` }), + ) + const unregisterAlpha = coordinator.registerRemoteSubsetOwner( + `alpha`, + owners.alpha.boundary, + ) + const unregisterBeta = coordinator.registerRemoteSubsetOwner( + `beta`, + owners.beta.boundary, + ) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + const objects: [LoadSubsetOptions, LoadSubsetOptions] = [ + { limit: 1, offset: 0 }, + { limit: 2, offset: 1 }, + ] + + await withFailurePreservingCleanup(async () => { + await waitFor( + () => coordinator.isLeader(`alpha`) && coordinator.isLeader(`beta`), + `${history.name} leadership`, + ) + for (const [actionIndex, action] of history.actions.entries()) { + const liveObjects = live[action.collectionId] + const expectedCollection = expected[action.collectionId] + if (action.kind === `ensure`) { + await coordinator.requestEnsureRemoteSubset( + action.collectionId, + objects[action.objectId], + ) + if (!liveObjects.has(action.objectId)) { + liveObjects.add(action.objectId) + expectedCollection.loads++ + expectedCollection.active++ + } + } else { + await coordinator.requestReleaseRemoteSubset( + action.collectionId, + objects[action.objectId], + ) + if (liveObjects.delete(action.objectId)) { + expectedCollection.unloads++ + expectedCollection.active-- + } + } + + const alpha = owners.alpha.snapshot() + const beta = owners.beta.snapshot() + const actual: CollectionScopedLeaseSnapshot = { + alpha: { + loads: alpha.loads, + unloads: alpha.unloads, + active: alpha.active, + }, + beta: { + loads: beta.loads, + unloads: beta.unloads, + active: beta.active, + }, + } + expect( + collectionScopedLeaseViolations(actual, expected), + `${history.name} action ${actionIndex}: ${JSON.stringify(action)}; actual=${JSON.stringify(actual)} expected=${JSON.stringify(expected)}`, + ).toEqual([]) + } + }, [ + [`${history.name} alpha owner`, unregisterAlpha], + [`${history.name} beta owner`, unregisterBeta], + [ + `${history.name} coordinator`, + async () => { + await disposeCoordinator(coordinator) + }, + ], + ]) + } + }) + + it(`keeps one public acquisition live across leader takeover and releases both owners exactly`, async () => { + type Todo = { id: string; title: string } + const firstOwner = new RemoteLeaseOwnerLedger() + const secondOwner = new RemoteLeaseOwnerLedger() + const dbName = `remote-lease-takeover-fixed` + const collectionId = `lease-todos` + let firstLeader: BrowserCollectionCoordinator | undefined + let secondLeader: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + let unregisterFirst: (() => void) | undefined + let unregisterSecond: (() => void) | undefined + let cleanupCollection: (() => Promise) | undefined + + await withFailurePreservingCleanup(async () => { + firstLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `lease-first` }), + ) + secondLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `lease-second` }), + ) + requester = createCoordinator( + dbName, + createRecordingAdapter({ id: `lease-requester` }), + ) + unregisterFirst = firstLeader.registerRemoteSubsetOwner( + collectionId, + firstOwner.boundary, + ) + unregisterSecond = secondLeader.registerRemoteSubsetOwner( + collectionId, + secondOwner.boundary, + ) + firstLeader.subscribe(collectionId, () => {}) + secondLeader.subscribe(collectionId, () => {}) + await waitFor( + () => + firstLeader!.isLeader(collectionId) && + !secondLeader!.isLeader(collectionId), + `initial remote subset owner`, + ) + + const collection = createCollection( + persistedCollectionOptions({ + id: collectionId, + getKey: (todo) => todo.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { + adapter: createRecordingAdapter({ id: `lease-requester-runtime` }), + coordinator: requester, + }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + + const demand: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([collectionId, `group`]), + new IR.Value(`owned`), + ]), + limit: 2, + } + const acquisition = Promise.resolve(collection._sync.loadSubset(demand)) + await pumpNetwork() + await acquisition + const initialOwnerAfterAcquire = firstOwner.snapshot() + + await disposeCoordinator(firstLeader) + await waitFor( + () => secondLeader!.isLeader(collectionId), + `remote subset takeover leader`, + ) + // Deliver the new leader heartbeat and every acquisition replay it + // causes. The public acquisition remains live throughout this cut. + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const retiredOwnerAfterTakeover = firstOwner.snapshot() + const newOwnerAfterTakeover = secondOwner.snapshot() + + collection._sync.unloadSubset(demand) + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const newOwnerAfterRelease = secondOwner.snapshot() + + const trace: RemoteLeaseTakeoverTrace = { + initialOwnerAfterAcquire, + retiredOwnerAfterTakeover, + newOwnerAfterTakeover, + newOwnerAfterRelease, + } + expect(remoteLeaseTakeoverViolations(trace)).toEqual([]) + }, [ + [ + `remote lease requester collection`, + async () => { + await cleanupCollection?.() + }, + ], + [`first remote subset owner`, () => unregisterFirst?.()], + [`second remote subset owner`, () => unregisterSecond?.()], + [ + `first remote subset leader`, + async () => { + await disposeCoordinator(firstLeader) + }, + ], + [ + `second remote subset leader`, + async () => { + await disposeCoordinator(secondLeader) + }, + ], + [ + `remote subset requester`, + async () => { + await disposeCoordinator(requester) + }, + ], + ]) + }) + + it(`calibrates the lease checker against leak, missing replay, and inexact release mutants`, () => { + expect( + remoteLeaseTakeoverViolations(expectedRemoteLeaseTakeoverTrace), + ).toEqual([]) + const hostile: RemoteLeaseTakeoverTrace = structuredClone( + expectedRemoteLeaseTakeoverTrace, + ) + hostile.retiredOwnerAfterTakeover = { + loads: 1, + unloads: 0, + active: 1, + exactUnloads: true, + } + hostile.newOwnerAfterTakeover = { + loads: 0, + unloads: 0, + active: 0, + exactUnloads: true, + } + hostile.newOwnerAfterRelease.exactUnloads = false + expect( + remoteLeaseTakeoverViolations(hostile).map( + ({ checkpoint, field }) => `${checkpoint}:${field}`, + ), + ).toEqual([ + `retiredOwnerAfterTakeover:unloads`, + `retiredOwnerAfterTakeover:active`, + `newOwnerAfterTakeover:loads`, + `newOwnerAfterTakeover:active`, + `newOwnerAfterRelease:exactUnloads`, + ]) + }) + + it(`preserves independent sibling leases through release, retry-safe takeover, and cleanup histories`, async () => { + let originalHistory: RemoteLeaseHistory | undefined + let originalViolation: RemoteLeaseTakeoverViolation | undefined + let targetDiscriminant: string | undefined + const cleanupDiagnostics: Array< + CleanupDiagnostic & { history: RemoteLeaseHistory } + > = [] + const executionDiagnostics: Array< + CleanupDiagnostic & { history: RemoteLeaseHistory; checkpoint: string } + > = [] + let propertyFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + let run = 0 + + try { + await fc.assert( + fc.asyncProperty(remoteLeaseHistoryArbitrary, async (history) => { + const firstOwner = new RemoteLeaseOwnerLedger() + const secondOwner = new RemoteLeaseOwnerLedger() + const suffix = `${++run}-${history.siblingCount}-${history.releasePrefix}` + const dbName = `remote-lease-history-${suffix}` + const collectionId = `lease-history-${suffix}` + let firstLeader: BrowserCollectionCoordinator | undefined + let secondLeader: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + let unregisterFirst: (() => void) | undefined + let unregisterSecond: (() => void) | undefined + let cleanupCollection: (() => Promise) | undefined + let semanticFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + let checkpoint = `coordinator allocation` + + try { + firstLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `history-first` }), + ) + secondLeader = createCoordinator( + dbName, + createRecordingAdapter({ id: `history-second` }), + ) + requester = createCoordinator( + dbName, + createRecordingAdapter({ id: `history-requester` }), + ) + unregisterFirst = firstLeader.registerRemoteSubsetOwner( + collectionId, + firstOwner.boundary, + ) + unregisterSecond = secondLeader.registerRemoteSubsetOwner( + collectionId, + secondOwner.boundary, + ) + firstLeader.subscribe(collectionId, () => {}) + secondLeader.subscribe(collectionId, () => {}) + checkpoint = `initial leadership` + await waitFor( + () => + firstLeader!.isLeader(collectionId) && + !secondLeader!.isLeader(collectionId), + `generated remote subset owner`, + ) + + const collection = createCollection( + persistedCollectionOptions<{ id: string }, string>({ + id: collectionId, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return {} + }, + }, + persistence: { + adapter: createRecordingAdapter({ id: `history-runtime` }), + coordinator: requester, + }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + const demands = Array.from( + { length: history.siblingCount }, + (_, index): LoadSubsetOptions => ({ + limit: 1, + ...(history.identical ? {} : { offset: index }), + }), + ) + + checkpoint = `initial acquisitions` + for (const demand of demands) { + const acquisition = Promise.resolve( + collection._sync.loadSubset(demand), + ) + await pumpNetwork() + await acquisition + } + const firstAfterAcquire = firstOwner.snapshot() + + checkpoint = `prefix releases` + const releaseCount = Math.min( + history.releasePrefix, + history.siblingCount - 1, + ) + for (const demand of demands.slice(0, releaseCount)) { + collection._sync.unloadSubset(demand) + if (history.duplicateRelease) { + collection._sync.unloadSubset(demand) + } + } + await pumpNetwork() + const firstAfterPrefixRelease = firstOwner.snapshot() + + checkpoint = `leadership takeover` + await disposeCoordinator(firstLeader) + await waitFor( + () => secondLeader!.isLeader(collectionId), + `generated remote subset takeover`, + ) + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const firstAfterTakeover = firstOwner.snapshot() + const secondAfterTakeover = secondOwner.snapshot() + + checkpoint = `final releases` + const finalDemands = demands.slice(releaseCount) + if (history.reverseFinalRelease) finalDemands.reverse() + for (const demand of finalDemands) { + collection._sync.unloadSubset(demand) + if (history.duplicateRelease) { + collection._sync.unloadSubset(demand) + } + } + await pumpNetwork() + await nextTurn() + await pumpNetwork() + const secondAfterFinalRelease = secondOwner.snapshot() + + checkpoint = `semantic comparison` + const trace: RemoteLeaseHistoryTrace = { + firstAfterAcquire, + firstAfterPrefixRelease, + firstAfterTakeover, + secondAfterTakeover, + secondAfterFinalRelease, + } + const [violation] = remoteLeaseHistoryViolations(history, trace) + if (violation) { + const discriminant = `${violation.checkpoint}:${violation.field}` + if (targetDiscriminant === undefined) { + targetDiscriminant = discriminant + originalHistory = structuredClone(history) + originalViolation = structuredClone(violation) + } + if (discriminant === targetDiscriminant) { + semanticFailure = new Error( + `remote lease mismatch; discriminant=${discriminant}; originalViolation=${JSON.stringify(originalViolation)}; reducedViolation=${JSON.stringify(violation)}; original=${JSON.stringify(originalHistory)}; reduced=${JSON.stringify(history)}; trace=${JSON.stringify(trace)}`, + ) + } + } + } catch (error) { + executionDiagnostics.push({ + resource: `generated remote lease execution`, + error: + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + history: structuredClone(history), + checkpoint, + }) + } + + const historyCleanupDiagnostics: Array = [] + await captureCleanup( + historyCleanupDiagnostics, + `generated remote lease collection`, + async () => { + await cleanupCollection?.() + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `generated first owner registration`, + () => unregisterFirst?.(), + ) + await captureCleanup( + historyCleanupDiagnostics, + `generated second owner registration`, + () => unregisterSecond?.(), + ) + for (const [resource, coordinator] of [ + [`generated first leader`, firstLeader], + [`generated second leader`, secondLeader], + [`generated requester`, requester], + ] as const) { + await captureCleanup( + historyCleanupDiagnostics, + resource, + async () => { + await disposeCoordinator(coordinator) + }, + ) + } + await captureCleanup( + historyCleanupDiagnostics, + `generated remote lease transport and locks`, + async () => { + await waitForWithoutDelivery( + () => + heldLocks.size === 0 && + queuedLocks.size === 0 && + ControlledBroadcastChannel.delayed.length === 0, + `generated remote lease cleanup`, + ) + }, + ) + cleanupDiagnostics.push( + ...historyCleanupDiagnostics.map((diagnostic) => ({ + ...diagnostic, + history: structuredClone(history), + })), + ) + if (semanticFailure !== NO_PRIMARY_FAILURE) throw semanticFailure + }), + { + seed: SEED, + numRuns: RUNS, + ...(PATH === undefined ? {} : { path: PATH }), + examples: [ + [ + { + siblingCount: 2, + releasePrefix: 1, + identical: true, + duplicateRelease: true, + reverseFinalRelease: true, + }, + ], + ], + }, + ) + } catch (error) { + propertyFailure = error + } + + expect.soft(cleanupDiagnostics).toEqual([]) + expect.soft(executionDiagnostics).toEqual([]) + if (propertyFailure !== NO_PRIMARY_FAILURE) throw propertyFailure + }) +}) + +type RawCollectionSnapshot = { + registryRowCount: number + schemaVersion: number + resetEpoch: number + rowCount: number +} + +type RawSnapshotViolation = { + field: keyof RawCollectionSnapshot + expected: number + actual: number +} + +function rawSnapshotViolations( + actual: RawCollectionSnapshot, + expected: RawCollectionSnapshot, +): Array { + return ( + [`registryRowCount`, `schemaVersion`, `resetEpoch`, `rowCount`] as const + ).flatMap((field) => + actual[field] === expected[field] + ? [] + : [{ field, expected: expected[field], actual: actual[field] }], + ) +} + +type FreshInitOutcome = + | { status: `fulfilled` } + | { status: `rejected`; error: string } + +type FreshInitViolation = + | { kind: `missing-outcome`; index: number } + | { kind: `unexpected-outcome`; index: number } + | { kind: `rejected`; index: number; error: string } + +function freshInitViolations( + outcomes: ReadonlyArray, +): Array { + const violations: Array = [] + for (let index = 0; index < 2; index++) { + const outcome = outcomes[index] + if (!outcome) { + violations.push({ kind: `missing-outcome`, index }) + } else if (outcome.status === `rejected`) { + violations.push({ kind: `rejected`, index, error: outcome.error }) + } + } + for (let index = 2; index < outcomes.length; index++) { + violations.push({ kind: `unexpected-outcome`, index }) + } + return violations +} + +async function readRawCollectionSnapshot( + database: BrowserWASQLiteDatabase, + collectionId: string, +): Promise { + const registry = await database.execute<{ + table_name: string + schema_version: number + }>( + `SELECT table_name, schema_version FROM collection_registry WHERE collection_id = ?`, + [collectionId], + ) + const row = registry[0] + if (!row) throw new Error(`missing registry row for ${collectionId}`) + const reset = await database.execute<{ reset_epoch: number }>( + `SELECT reset_epoch FROM collection_reset_epoch WHERE collection_id = ?`, + [collectionId], + ) + const tableName = `"${row.table_name.replaceAll(`"`, `""`)}"` + const count = await database.execute<{ count: number }>( + `SELECT COUNT(*) AS count FROM ${tableName}`, + ) + return { + registryRowCount: registry.length, + schemaVersion: row.schema_version, + resetEpoch: reset[0]?.reset_epoch ?? -1, + rowCount: count[0]?.count ?? -1, + } +} + +describe(`per-collection adapter and SQLite-state oracle`, () => { + it(`preserves alpha rows, schema, and reset epoch when beta registers a different adapter`, async () => { + let directory: string | undefined + let database: BrowserWASQLiteDatabase | undefined + let coordinator: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + directory = mkdtempSync(join(tmpdir(), `db-coordinator-routing-`)) + const createdDatabase = createWASQLiteTestDatabase({ + filename: join(directory, `state.sqlite`), + }) + database = createdDatabase + const createdCoordinator = createCoordinator( + `real-sqlite-routing`, + createRecordingAdapter({ id: `bootstrap` }), + ) + coordinator = createdCoordinator + const persistence = createBrowserWASQLitePersistence({ + database: createdDatabase, + coordinator: createdCoordinator, + }) + const resolve = ( + collectionId: string, + schemaVersion: number, + ): PersistenceAdapter => { + const resolved = persistence.resolvePersistenceForCollection?.({ + collectionId, + mode: `sync-present`, + schemaVersion, + }) + if (!resolved) throw new Error(`missing per-collection resolver`) + return resolved.adapter + } + + const alphaAdapter = resolve(`alpha`, 11) + await alphaAdapter.applyCommittedTx(`alpha`, { + txId: `alpha-seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `a-1`, + value: { id: `a-1`, group: `kept` }, + }, + ], + }) + const expected: RawCollectionSnapshot = { + registryRowCount: 1, + schemaVersion: 11, + resetEpoch: 0, + rowCount: 1, + } + const before = await readRawCollectionSnapshot(createdDatabase, `alpha`) + expect(rawSnapshotViolations(before, expected)).toEqual([]) + + // Merely resolving beta replaces the coordinator's one adapter slot. + // The later alpha leadership read therefore runs through beta's schema. + const betaAdapter = resolve(`beta`, 22) + await betaAdapter.loadSubset(`beta`, {}) + createdCoordinator.subscribe(`alpha`, () => {}) + await waitFor( + () => createdCoordinator.isLeader(`alpha`), + `alpha leadership after beta registration`, + ) + const after = await readRawCollectionSnapshot(createdDatabase, `alpha`) + + // RED D2 fixed cross-collection contamination witness. The shared + // comparator reports schema, reset, and row losses independently. + expect(rawSnapshotViolations(after, expected)).toEqual([]) + }, [ + [ + `real SQLite coordinator`, + async () => { + await disposeCoordinator(coordinator) + }, + ], + [ + `real SQLite database`, + async () => { + await Promise.resolve(database?.close?.()) + }, + ], + [ + `real SQLite directory`, + () => { + if (directory) rmSync(directory, { recursive: true, force: true }) + }, + ], + ]) + }) + + it(`initializes one fresh collection safely across matching adapter instances`, async () => { + let directory: string | undefined + let database: BrowserWASQLiteDatabase | undefined + + await withFailurePreservingCleanup(async () => { + directory = mkdtempSync(join(tmpdir(), `db-adapter-init-race-`)) + const createdDatabase = createWASQLiteTestDatabase({ + filename: join(directory, `state.sqlite`), + }) + database = createdDatabase + const createAdapter = (): PersistenceAdapter => { + const persistence = createBrowserWASQLitePersistence({ + database: createdDatabase, + }) + const resolved = persistence.resolvePersistenceForCollection?.({ + collectionId: `shared`, + mode: `sync-present`, + schemaVersion: 7, + }) + if (!resolved) throw new Error(`missing per-collection resolver`) + return resolved.adapter + } + const first = createAdapter() + const second = createAdapter() + + const outcomes = await Promise.allSettled([ + first.loadSubset(`shared`, {}), + second.loadSubset(`shared`, {}), + ]) + const observed: Array = outcomes.map((outcome) => + outcome.status === `fulfilled` + ? { status: outcome.status } + : { + status: outcome.status, + error: + outcome.reason instanceof Error + ? outcome.reason.message + : String(outcome.reason), + }, + ) + + const initialized = await readRawCollectionSnapshot( + createdDatabase, + `shared`, + ) + expect( + rawSnapshotViolations(initialized, { + registryRowCount: 1, + schemaVersion: 7, + resetEpoch: 0, + rowCount: 0, + }), + ).toEqual([]) + + // Separate #1753 checkpoint: same collection and same version/policy is + // legal in two tabs. Both production adapter initializations must finish; + // this does not rely on the cross-version reset assertion above. + expect(freshInitViolations(observed)).toEqual([]) + }, [ + [ + `race SQLite database`, + async () => { + await Promise.resolve(database?.close?.()) + }, + ], + [ + `race SQLite directory`, + () => { + if (directory) rmSync(directory, { recursive: true, force: true }) + }, + ], + ]) + }) + + it(`calibrates SQLite-state and fresh-registry checks against hostile results`, () => { + const expected: RawCollectionSnapshot = { + registryRowCount: 1, + schemaVersion: 11, + resetEpoch: 0, + rowCount: 1, + } + expect(rawSnapshotViolations(expected, expected)).toEqual([]) + expect( + rawSnapshotViolations( + { + registryRowCount: 0, + schemaVersion: 22, + resetEpoch: 1, + rowCount: 0, + }, + expected, + ).map(({ field }) => field), + ).toEqual([`registryRowCount`, `schemaVersion`, `resetEpoch`, `rowCount`]) + + const allowed: Array = [ + { status: `fulfilled` }, + { status: `fulfilled` }, + ] + expect(freshInitViolations(allowed)).toEqual([]) + expect( + freshInitViolations([ + allowed[0]!, + { + status: `rejected`, + error: `UNIQUE constraint failed: collection_registry`, + }, + ]), + ).toEqual([ + { + kind: `rejected`, + index: 1, + error: `UNIQUE constraint failed: collection_registry`, + }, + ]) + expect(freshInitViolations([allowed[0]!])).toEqual([ + { kind: `missing-outcome`, index: 1 }, + ]) + }) +}) + +type CollectionName = `alpha` | `beta` +type TabName = `leader` | `follower` + +type RoutingHistory = { + alphaVersion: number + betaVersion: number + registrationOrder: [CollectionName, CollectionName] + firstDeliveryOrder: [CollectionName, CollectionName] + secondDeliveryOrder: [CollectionName, CollectionName] +} + +type RoutedApply = { + phase: `initial-owner` | `after-takeover` + collectionId: CollectionName + adapterId: string +} + +type RouteViolation = { + kind: `missing-apply` | `duplicate-apply` | `misrouted-apply` + checkpoint: RoutedApply[`phase`] + collectionId: CollectionName + expectedAdapterId: string + actualAdapterIds: Array +} + +type CoordinatorRouteObservation = { + phase: RoutedApply[`phase`] + operation: `remote-subset` | `index` | `pull` + collectionId: CollectionName + calls: Array<{ + adapterId: string + collectionId: string + argument?: SemanticValue + }> + result: SemanticValue +} + +type CoordinatorRouteViolation = { + operation: CoordinatorRouteObservation[`operation`] + checkpoint: CoordinatorRouteObservation[`phase`] + collectionId: CollectionName + field: `calls` | `result` + expected: SemanticValue + actual: SemanticValue +} + +function coordinatorRouteViolations( + observed: ReadonlyArray, + expected: ReadonlyArray, +): Array { + const violations: Array = [] + for (const expectedCell of expected) { + const actualCell = observed.find( + ({ phase, operation, collectionId }) => + phase === expectedCell.phase && + operation === expectedCell.operation && + collectionId === expectedCell.collectionId, + ) + for (const field of [`calls`, `result`] as const) { + const expectedValue = semanticValue(expectedCell[field]) + const actualValue = semanticValue(actualCell?.[field] ?? []) + if (JSON.stringify(actualValue) !== JSON.stringify(expectedValue)) { + violations.push({ + operation: expectedCell.operation, + checkpoint: expectedCell.phase, + collectionId: expectedCell.collectionId, + field, + expected: expectedValue, + actual: actualValue, + }) + } + } + } + return violations +} + +function routeViolationDiscriminant(violation: RouteViolation): string { + return `${violation.kind}:${violation.checkpoint}:${violation.collectionId}` +} + +function routeViolations( + observed: ReadonlyArray, + expected: ReadonlyArray, +): Array { + const violations: Array = [] + for (const expectedRoute of expected) { + const actualAdapterIds = observed + .filter( + ({ phase, collectionId }) => + phase === expectedRoute.phase && + collectionId === expectedRoute.collectionId, + ) + .map(({ adapterId }) => adapterId) + if (actualAdapterIds.length === 0) { + violations.push({ + kind: `missing-apply`, + checkpoint: expectedRoute.phase, + collectionId: expectedRoute.collectionId, + expectedAdapterId: expectedRoute.adapterId, + actualAdapterIds, + }) + continue + } + if (actualAdapterIds.length !== 1) { + violations.push({ + kind: `duplicate-apply`, + checkpoint: expectedRoute.phase, + collectionId: expectedRoute.collectionId, + expectedAdapterId: expectedRoute.adapterId, + actualAdapterIds, + }) + continue + } + if (actualAdapterIds[0] !== expectedRoute.adapterId) { + violations.push({ + kind: `misrouted-apply`, + checkpoint: expectedRoute.phase, + collectionId: expectedRoute.collectionId, + expectedAdapterId: expectedRoute.adapterId, + actualAdapterIds, + }) + } + } + return violations +} + +class PerCollectionRouteOracle { + private readonly routes = new Map() + private readonly owners = new Map() + + register(tab: TabName, collection: CollectionName, adapterId: string): void { + this.routes.set(`${tab}:${collection}`, adapterId) + } + + setOwner(collection: CollectionName, tab: TabName): void { + this.owners.set(collection, tab) + } + + expectedApply( + phase: RoutedApply[`phase`], + collectionId: CollectionName, + ): RoutedApply { + const owner = this.owners.get(collectionId) + const adapterId = owner + ? this.routes.get(`${owner}:${collectionId}`) + : undefined + if (!owner || !adapterId) { + throw new Error(`oracle has no owner/route for ${collectionId}`) + } + return { phase, collectionId, adapterId } + } +} + +const orderArbitrary = fc + .boolean() + .map((forward): [CollectionName, CollectionName] => + forward ? [`alpha`, `beta`] : [`beta`, `alpha`], + ) + +const routingHistoryArbitrary: fc.Arbitrary = fc + .record({ + alphaVersion: fc.integer({ min: 1, max: 50 }), + versionDelta: fc.integer({ min: 1, max: 50 }), + registrationOrder: orderArbitrary, + firstDeliveryOrder: orderArbitrary, + secondDeliveryOrder: orderArbitrary, + }) + .map(({ alphaVersion, versionDelta, ...history }) => ({ + ...history, + alphaVersion, + betaVersion: alphaVersion + versionDelta, + })) + +function mutation( + collectionId: CollectionName, + phase: RoutedApply[`phase`], +): Array { + return [ + { + mutationId: `${phase}:${collectionId}`, + type: `insert`, + key: `${phase}:${collectionId}`, + value: { id: `${phase}:${collectionId}`, collectionId }, + }, + ] +} + +function applyObservation( + phase: RoutedApply[`phase`], + calls: ReadonlyArray, +): Array { + return calls + .filter( + (call): call is AdapterCall & { collectionId: CollectionName } => + call.operation === `apply` && + (call.collectionId === `alpha` || call.collectionId === `beta`), + ) + .map(({ collectionId, adapterId }) => ({ + phase, + collectionId, + adapterId, + })) + .sort((left, right) => left.collectionId.localeCompare(right.collectionId)) +} + +async function requestBothCollections( + requester: BrowserCollectionCoordinator, + phase: RoutedApply[`phase`], + deliveryOrder: [CollectionName, CollectionName], +): Promise { + const pending = ([`alpha`, `beta`] as const).map((collectionId) => + requester.requestApplyLocalMutations( + collectionId, + mutation(collectionId, phase), + ), + ) + for (const collectionId of deliveryOrder) { + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === `rpc:applyLocalMutations:req` && + envelopeCollection(data) === collectionId, + ), + ).toBe(true) + await nextTurn() + } + await pumpNetwork() + const responses = await Promise.all(pending) + expect(responses.every((response) => response.ok)).toBe(true) +} + +describe(`generated collection-route histories`, () => { + it(`calibrates collection-key comparison against one peer-routed apply`, () => { + const expected: Array = [ + { + phase: `initial-owner`, + collectionId: `alpha`, + adapterId: `leader-alpha`, + }, + { + phase: `initial-owner`, + collectionId: `beta`, + adapterId: `leader-beta`, + }, + ] + const misrouted: Array = [ + expected[0]!, + { ...expected[1]!, adapterId: `leader-alpha` }, + ] + + expect(routeViolations(expected, expected)).toEqual([]) + const [violation] = routeViolations(misrouted, expected) + expect(violation).toEqual({ + kind: `misrouted-apply`, + checkpoint: `initial-owner`, + collectionId: `beta`, + expectedAdapterId: `leader-beta`, + actualAdapterIds: [`leader-alpha`], + }) + expect(routeViolationDiscriminant(violation!)).toBe( + `misrouted-apply:initial-owner:beta`, + ) + + const routedCell: CoordinatorRouteObservation = { + phase: `initial-owner`, + operation: `index`, + collectionId: `alpha`, + calls: [ + { + adapterId: `leader-alpha`, + collectionId: `alpha`, + argument: semanticValue({ signature: `alpha-index` }), + }, + ], + result: semanticValue({ status: `fulfilled` }), + } + expect(coordinatorRouteViolations([routedCell], [routedCell])).toEqual([]) + expect( + coordinatorRouteViolations( + [{ ...routedCell, calls: [] }], + [routedCell], + ).map(({ operation, checkpoint, collectionId, field }) => ({ + operation, + checkpoint, + collectionId, + field, + })), + ).toEqual([ + { + operation: `index`, + checkpoint: `initial-owner`, + collectionId: `alpha`, + field: `calls`, + }, + ]) + }) + + it(`routes every adapter-bound RPC by collection before and after leadership transfer`, async () => { + const makePullResult = ( + marker: string, + latestRowVersion: number, + ): PullFixtureResult => ({ + latestRowVersion, + requiresFullReload: false, + changedKeys: [`${marker}:changed`], + deletedKeys: [`${marker}:deleted`], + }) + const leaderPullResults = { + alpha: makePullResult(`leader-alpha`, 101), + beta: makePullResult(`leader-beta`, 202), + } + const followerPullResults = { + alpha: makePullResult(`follower-alpha`, 303), + beta: makePullResult(`follower-beta`, 404), + } + const createPhaseAdapters = ( + tab: TabName, + pullResults: Record, + ): Record => ({ + alpha: createRecordingAdapter({ + id: `${tab}-alpha`, + remoteSubsetOwner: () => Promise.resolve(), + pullResult: pullResults.alpha, + }), + beta: createRecordingAdapter({ + id: `${tab}-beta`, + remoteSubsetOwner: () => Promise.resolve(), + pullResult: pullResults.beta, + }), + }) + const leaderAdapters = createPhaseAdapters(`leader`, leaderPullResults) + const followerAdapters = createPhaseAdapters( + `follower`, + followerPullResults, + ) + const requesterAdapter = createRecordingAdapter({ id: `requester` }) + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + const observed: Array = [] + const expected: Array = [] + const dbName = `adapter-bound-rpc-matrix` + leader = createCoordinator(dbName, leaderAdapters.alpha) + follower = createCoordinator(dbName, followerAdapters.alpha) + for (const collectionId of [`alpha`, `beta`] as const) { + // Register beta last so a global mutable adapter misroutes alpha. + registerCollectionAdapter( + leader, + collectionId, + leaderAdapters[collectionId], + ) + registerCollectionAdapter( + follower, + collectionId, + followerAdapters[collectionId], + ) + leader.subscribe(collectionId, () => {}) + follower.subscribe(collectionId, () => {}) + } + await waitFor( + () => + leader!.isLeader(`alpha`) && + leader!.isLeader(`beta`) && + !follower!.isLeader(`alpha`) && + !follower!.isLeader(`beta`), + `adapter-bound initial leadership`, + ) + + const exercisePhase = async ( + phase: RoutedApply[`phase`], + rpcRequester: BrowserCollectionCoordinator, + ownerAdapters: Record, + pullResults: Record, + ): Promise => { + for (const operation of [`remote-subset`, `index`, `pull`] as const) { + for (const collectionId of [`alpha`, `beta`] as const) { + const before = new Map( + Object.values(ownerAdapters).map((adapter) => [ + adapter, + adapter.calls.length, + ]), + ) + let requestType: string + let argument: SemanticValue + let pending: Promise + let expectedResult: SemanticValue + + if (operation === `remote-subset`) { + const demand: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([collectionId, `group`]), + new IR.Value(`${phase}:${collectionId}`), + ]), + limit: collectionId === `alpha` ? 1 : 2, + offset: phase === `initial-owner` ? 0 : 1, + } + requestType = `rpc:ensureRemoteSubset:req` + argument = subsetSemantics(demand) + pending = rpcRequester + .requestEnsureRemoteSubset(collectionId, demand) + .then(() => ({ status: `fulfilled` })) + expectedResult = semanticValue({ status: `fulfilled` }) + } else if (operation === `index`) { + const signature = `${phase}:${collectionId}:index` + const spec = { + expressionSql: [`${collectionId}_score`, phase], + } + requestType = `rpc:ensurePersistedIndex:req` + argument = semanticValue({ signature, spec }) + pending = rpcRequester + .requestEnsurePersistedIndex(collectionId, signature, spec) + .then(() => ({ status: `fulfilled` })) + expectedResult = semanticValue({ status: `fulfilled` }) + } else { + const fromRowVersion = + (phase === `initial-owner` ? 0 : 100) + + (collectionId === `alpha` ? 11 : 22) + requestType = `rpc:pullSince:req` + argument = semanticValue({ fromRowVersion }) + pending = rpcRequester.pullSince(collectionId, fromRowVersion) + const pullResult = pullResults[collectionId] + expectedResult = semanticValue({ + ok: true, + latestTerm: 1, + latestSeq: 0, + latestRowVersion: pullResult.latestRowVersion, + requiresFullReload: false, + changedKeys: pullResult.changedKeys, + deletedKeys: pullResult.deletedKeys, + }) + } + + expect( + ControlledBroadcastChannel.deliverWhere( + (data) => + payloadType(data) === requestType && + envelopeCollection(data) === collectionId, + ), + ).toBe(true) + await pumpNetwork() + const result = await pending + const calls = Object.values(ownerAdapters) + .flatMap((adapter) => + adapter.calls + .slice(before.get(adapter) ?? 0) + .filter((call) => call.operation === operation) + .map( + ({ + adapterId, + collectionId: calledId, + argument: callArgument, + }) => ({ + adapterId, + collectionId: calledId, + ...(callArgument === undefined + ? {} + : { argument: callArgument }), + }), + ), + ) + .sort((left, right) => + left.adapterId.localeCompare(right.adapterId), + ) + const resultValue = + operation === `pull` + ? semanticValue( + result && typeof result === `object` + ? Object.fromEntries( + Object.entries(result).filter( + ([key]) => key !== `type` && key !== `rpcId`, + ), + ) + : result, + ) + : semanticValue(result) + observed.push({ + phase, + operation, + collectionId, + calls, + result: resultValue, + }) + expected.push({ + phase, + operation, + collectionId, + calls: [ + { + adapterId: ownerAdapters[collectionId].id, + collectionId, + argument, + }, + ], + result: expectedResult, + }) + } + } + } + + await exercisePhase( + `initial-owner`, + follower, + leaderAdapters, + leaderPullResults, + ) + await disposeCoordinator(leader) + await waitFor( + () => follower!.isLeader(`alpha`) && follower!.isLeader(`beta`), + `adapter-bound follower takeover`, + ) + requester = createCoordinator(dbName, requesterAdapter) + await exercisePhase( + `after-takeover`, + requester, + followerAdapters, + followerPullResults, + ) + + // Each violation retains operation, phase, collection, and whether the + // loss was the exact routed call or the exact response. + expect(coordinatorRouteViolations(observed, expected)).toEqual([]) + }, [ + [ + `adapter-bound initial leader`, + async () => { + await disposeCoordinator(leader) + }, + ], + [ + `adapter-bound takeover leader`, + async () => { + await disposeCoordinator(follower) + }, + ], + [ + `adapter-bound post-takeover requester`, + async () => { + await disposeCoordinator(requester) + }, + ], + ]) + }) + + it(`uses the collection's registered adapter before and after leadership transfer`, async () => { + let originalFailingTrace: RoutingHistory | undefined + let originalViolation: RouteViolation | undefined + let targetDiscriminant: string | undefined + const cleanupDiagnostics: Array< + CleanupDiagnostic & { history: RoutingHistory } + > = [] + const executionDiagnostics: Array< + CleanupDiagnostic & { history: RoutingHistory; checkpoint: string } + > = [] + let propertyFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + + try { + await fc.assert( + fc.asyncProperty(routingHistoryArbitrary, async (history) => { + const oracle = new PerCollectionRouteOracle() + const leaderAdapters = { + alpha: createRecordingAdapter({ + id: `leader-alpha-v${history.alphaVersion}-reset`, + schemaVersion: history.alphaVersion, + policy: `sync-present-reset`, + }), + beta: createRecordingAdapter({ + id: `leader-beta-v${history.betaVersion}-error`, + schemaVersion: history.betaVersion, + policy: `sync-absent-error`, + }), + } + const followerAdapters = { + alpha: createRecordingAdapter({ + id: `follower-alpha-v${history.alphaVersion}-reset`, + schemaVersion: history.alphaVersion, + policy: `sync-present-reset`, + }), + beta: createRecordingAdapter({ + id: `follower-beta-v${history.betaVersion}-error`, + schemaVersion: history.betaVersion, + policy: `sync-absent-error`, + }), + } + const dbName = `generated-route-${history.alphaVersion}-${history.betaVersion}` + const initialCollection = history.registrationOrder[0] + let leader: BrowserCollectionCoordinator | undefined + let follower: BrowserCollectionCoordinator | undefined + const observed: Array = [] + const expected: Array = [] + let semanticFailure: unknown | typeof NO_PRIMARY_FAILURE = + NO_PRIMARY_FAILURE + let checkpoint = `coordinator allocation` + + try { + leader = createCoordinator( + dbName, + leaderAdapters[initialCollection], + ) + follower = createCoordinator( + dbName, + followerAdapters[initialCollection], + ) + + for (const collectionId of history.registrationOrder) { + registerCollectionAdapter( + leader, + collectionId, + leaderAdapters[collectionId], + ) + registerCollectionAdapter( + follower, + collectionId, + followerAdapters[collectionId], + ) + oracle.register( + `leader`, + collectionId, + leaderAdapters[collectionId].id, + ) + oracle.register( + `follower`, + collectionId, + followerAdapters[collectionId].id, + ) + } + + checkpoint = `initial leadership` + for (const collectionId of [`alpha`, `beta`] as const) { + leader.subscribe(collectionId, () => {}) + follower.subscribe(collectionId, () => {}) + oracle.setOwner(collectionId, `leader`) + } + await waitFor( + () => + leader!.isLeader(`alpha`) && + leader!.isLeader(`beta`) && + !follower!.isLeader(`alpha`) && + !follower!.isLeader(`beta`), + `generated initial two-collection leadership`, + ) + + checkpoint = `initial routed applies` + await requestBothCollections( + follower, + `initial-owner`, + history.firstDeliveryOrder, + ) + observed.push( + ...applyObservation(`initial-owner`, [ + ...leaderAdapters.alpha.calls, + ...leaderAdapters.beta.calls, + ]), + ) + expected.push( + oracle.expectedApply(`initial-owner`, `alpha`), + oracle.expectedApply(`initial-owner`, `beta`), + ) + + checkpoint = `leadership transfer` + await disposeCoordinator(leader) + for (const collectionId of [`alpha`, `beta`] as const) { + oracle.setOwner(collectionId, `follower`) + } + await waitFor( + () => follower!.isLeader(`alpha`) && follower!.isLeader(`beta`), + `generated follower takeover`, + ) + + checkpoint = `post-takeover routed applies` + // The new leader's direct calls still exercise the same production + // handler and must look up adapters per collection. + for (const collectionId of history.secondDeliveryOrder) { + const response = await follower.requestApplyLocalMutations( + collectionId, + mutation(collectionId, `after-takeover`), + ) + expect(response.ok).toBe(true) + } + observed.push( + ...applyObservation(`after-takeover`, [ + ...followerAdapters.alpha.calls, + ...followerAdapters.beta.calls, + ]), + ) + expected.push( + oracle.expectedApply(`after-takeover`, `alpha`), + oracle.expectedApply(`after-takeover`, `beta`), + ) + + const [violation] = routeViolations(observed, expected) + if (violation) { + const discriminant = routeViolationDiscriminant(violation) + if (targetDiscriminant === undefined) { + targetDiscriminant = discriminant + originalViolation = structuredClone(violation) + originalFailingTrace = structuredClone(history) + } + // A smaller candidate is a valid reduction only when it reaches + // the same checkpoint and misroutes the same collection. + if (discriminant === targetDiscriminant) { + semanticFailure = new Error( + `per-collection route mismatch; discriminant=${discriminant}; originalViolation=${JSON.stringify(originalViolation)}; reducedViolation=${JSON.stringify(violation)}; original=${JSON.stringify(originalFailingTrace)}; reduced=${JSON.stringify(history)}; expected=${JSON.stringify(expected)}; observed=${JSON.stringify(observed)}`, + ) + } + } + } catch (error) { + executionDiagnostics.push({ + resource: `generated route execution`, + error: + error instanceof Error + ? `${error.name}: ${error.message}` + : String(error), + history: structuredClone(history), + checkpoint, + }) + } + + const historyCleanupDiagnostics: Array = [] + await captureCleanup( + historyCleanupDiagnostics, + `leader coordinator`, + async () => { + await disposeCoordinator(leader) + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `follower coordinator`, + async () => { + await disposeCoordinator(follower) + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `coordinator locks`, + async () => { + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `generated history lock cleanup`, + ) + }, + ) + await captureCleanup( + historyCleanupDiagnostics, + `coordinator transport`, + () => { + const snapshot = { + endpoints: ControlledBroadcastChannel.endpoints.size, + delayed: ControlledBroadcastChannel.delayed.length, + } + if (snapshot.endpoints !== 0 || snapshot.delayed !== 0) { + throw new Error(JSON.stringify(snapshot)) + } + }, + ) + cleanupDiagnostics.push( + ...historyCleanupDiagnostics.map((diagnostic) => ({ + ...diagnostic, + history: structuredClone(history), + })), + ) + + if (semanticFailure !== NO_PRIMARY_FAILURE) throw semanticFailure + }), + { + seed: SEED, + numRuns: RUNS, + ...(PATH === undefined ? {} : { path: PATH }), + }, + ) + } catch (error) { + propertyFailure = error + } + + // Cleanup diagnostics are evaluated outside FastCheck, so cleanup cannot + // become the predicate that selects or shrinks a semantic route failure. + expect.soft(cleanupDiagnostics).toEqual([]) + // Setup/reach/fixture failures are also reported outside the property; + // only the locked semantic discriminant is eligible for shrinking. + expect.soft(executionDiagnostics).toEqual([]) + if (propertyFailure !== NO_PRIMARY_FAILURE) throw propertyFailure + }) +}) + +type OwnershipObservation = { + collectionId: string + ownerAtCall?: string +} + +function findUnownedWrites( + observations: ReadonlyArray, + electedOwner: string, +): Array { + return observations.filter( + (observation) => observation.ownerAtCall !== electedOwner, + ) +} + +describe(`sync-ingested write ownership oracle`, () => { + it(`applies a source commit through the elected persistence owner after leadership changes`, async () => { + type Todo = { id: string; title: string } + type SourceParams = Parameters[`sync`]>[0] + + let electedOwner = `tab-a` + let activeWriter: string | undefined + let sourceParams: SourceParams | undefined + let ownerRequests = 0 + const adapter = createRecordingAdapter({ + id: `sync-alpha`, + owner: () => activeWriter, + }) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `tab-a`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => electedOwner === `tab-a`, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + // Remote subset ownership is unrelated to this write-owner law. + requestEnsureRemoteSubset: async () => {}, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: async (collectionId, tx) => { + ownerRequests++ + const ownerTx = { + ...tx, + term: 1, + seq: ownerRequests, + rowVersion: ownerRequests, + } + activeWriter = electedOwner + try { + await adapter.applyCommittedTx(collectionId, ownerTx) + } finally { + activeWriter = undefined + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: `owned-${ownerRequests}`, + ok: true, + term: 1, + seq: ownerRequests, + latestRowVersion: ownerRequests, + } + }, + } + let cleanupCollection: (() => Promise) | undefined + + await withFailurePreservingCleanup(async () => { + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-alpha`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + await waitForWithoutDelivery( + () => sourceParams !== undefined, + `wrapped source controls`, + ) + + // Legal leadership history: the original tab loses ownership before an + // external source callback ingests its next transaction. + electedOwner = `tab-b` + sourceParams!.begin() + sourceParams!.write({ + type: `insert`, + value: { id: `sync-1`, title: `from source` }, + }) + const receipt = sourceParams!.commit() + if (receipt !== true) await receipt + + const applies = adapter.calls + .filter(({ operation }) => operation === `apply`) + .map(({ collectionId, ownerAtCall }) => ({ + collectionId, + ownerAtCall, + })) + + // Positive production reach: the source write became collection-visible + // and reached exactly one persistence apply call. + expect(collection.get(`sync-1`)).toMatchObject({ + id: `sync-1`, + title: `from source`, + }) + expect(applies).toHaveLength(1) + // Ownership checkpoint: a direct-adapter bypass runs outside the elected + // tab-b writer scope. Owner-routed implementations pass this unchanged + // regardless of request count. + expect(findUnownedWrites(applies, electedOwner)).toEqual([]) + }, [ + [ + `sync-ingested collection`, + async () => { + await cleanupCollection?.() + }, + ], + ]) + }) + + it(`calibrates the owner checker against one bypass and one owned write`, () => { + expect( + findUnownedWrites( + [ + { collectionId: `alpha`, ownerAtCall: `tab-b` }, + { collectionId: `beta` }, + ], + `tab-b`, + ), + ).toEqual([{ collectionId: `beta` }]) + }) + + it(`routes a rich source commit through the new elected Browser owner`, async () => { + type Todo = { id: string; title: string } + type SourceParams = Parameters[`sync`]>[0] + type TxAdapter = PersistenceAdapter & { applied: Array } + + const createTxAdapter = (): TxAdapter => { + const applied: Array = [] + return { + applied, + loadSubset: () => Promise.resolve([]), + applyCommittedTx: (_collectionId, tx) => { + applied.push(structuredClone(tx)) + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + getStreamPosition: () => + Promise.resolve({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }), + } + } + + const retiredAdapter = createTxAdapter() + const electedAdapter = createTxAdapter() + const requesterAdapter = createTxAdapter() + const originalElectedApply = + electedAdapter.applyCommittedTx.bind(electedAdapter) + let electedApplyEntered = false + let releaseElectedApply = (): void => {} + const electedApplyGate = new Promise((resolve) => { + releaseElectedApply = resolve + }) + electedAdapter.applyCommittedTx = async (collectionId, tx) => { + electedApplyEntered = true + await electedApplyGate + await originalElectedApply(collectionId, tx) + } + let retired: BrowserCollectionCoordinator | undefined + let elected: BrowserCollectionCoordinator | undefined + let requester: BrowserCollectionCoordinator | undefined + let cleanupCollection: (() => Promise) | undefined + const electedCommitted: Array = [] + const requesterCommitted: Array = [] + + await withFailurePreservingCleanup(async () => { + const dbName = `source-owner-transfer` + retired = createCoordinator(dbName, retiredAdapter) + elected = createCoordinator(dbName, electedAdapter) + requester = createCoordinator(dbName, requesterAdapter) + retired.subscribe(`todos`, () => {}) + elected.subscribe(`todos`, (message) => { + const payload = message.payload as { type?: string } + if (payload.type === `tx:committed`) { + electedCommitted.push(structuredClone(payload)) + } + }) + requester.subscribe(`todos`, (message) => { + const payload = message.payload as { type?: string } + if (payload.type === `tx:committed`) { + requesterCommitted.push(structuredClone(payload)) + } + }) + await waitFor( + () => + retired!.isLeader(`todos`) && + !elected!.isLeader(`todos`) && + !requester!.isLeader(`todos`), + `initial source owner`, + ) + + await disposeCoordinator(retired) + await waitFor( + () => elected!.isLeader(`todos`) && !requester!.isLeader(`todos`), + `transferred source owner`, + ) + + let sourceParams: SourceParams | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `todos`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter: requesterAdapter, coordinator: requester }, + }), + ) + cleanupCollection = () => collection.cleanup() + collection.startSyncImmediate() + await waitForWithoutDelivery( + () => sourceParams !== undefined, + `Browser source controls`, + ) + + sourceParams!.begin() + sourceParams!.metadata?.collection.set(`resume`, { offset: 11 }) + sourceParams!.truncate() + sourceParams!.write({ + type: `insert`, + value: { id: `rich`, title: `through Browser owner` }, + metadata: { source: `browser-sync` }, + }) + const receipt = sourceParams!.commit() + let receiptSettled = receipt === true + const receiptPromise = receipt === true ? Promise.resolve() : receipt + void receiptPromise.then( + () => { + receiptSettled = true + }, + () => { + receiptSettled = true + }, + ) + + await pumpNetwork() + await waitForWithoutDelivery( + () => electedApplyEntered, + `elected Browser source apply entry`, + ) + expect({ + receiptSettled, + electedApplies: electedAdapter.applied.length, + requesterInvalidations: requesterCommitted.length, + }).toEqual({ + receiptSettled: false, + electedApplies: 0, + requesterInvalidations: 0, + }) + + releaseElectedApply() + await pumpNetwork() + await receiptPromise + await pumpNetwork() + + expect(collection.get(`rich`)).toMatchObject({ + id: `rich`, + title: `through Browser owner`, + }) + expect(retiredAdapter.applied).toEqual([]) + expect(requesterAdapter.applied).toEqual([]) + expect(electedAdapter.applied).toHaveLength(1) + expect(electedAdapter.applied[0]).toMatchObject({ + truncate: true, + mutations: [ + { + type: `update`, + key: `rich`, + value: { id: `rich`, title: `through Browser owner` }, + }, + ], + rowMetadataMutations: [ + { + type: `set`, + key: `rich`, + value: { source: `browser-sync` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 11 } }, + ], + }) + const committedTxId = electedAdapter.applied[0]!.txId + const expectedCommitted = { + type: `tx:committed`, + term: 1, + seq: 1, + txId: committedTxId, + latestRowVersion: 1, + requiresFullReload: true, + } + expect(electedCommitted).toEqual([expectedCommitted]) + expect(requesterCommitted).toEqual([expectedCommitted]) + }, [ + [ + `elected Browser source apply gate`, + () => { + releaseElectedApply() + }, + ], + [ + `Browser source collection`, + async () => { + await cleanupCollection?.() + }, + ], + [ + `retired Browser source owner`, + async () => { + await disposeCoordinator(retired) + }, + ], + [ + `elected Browser source owner`, + async () => { + await disposeCoordinator(elected) + }, + ], + [ + `Browser source requester`, + async () => { + await disposeCoordinator(requester) + }, + ], + ]) + }) +}) + +describe(`coordinator fixture lifecycle`, () => { + it(`releases delayed transport and leadership state before a fresh lifecycle`, async () => { + let first: BrowserCollectionCoordinator | undefined + let peer: BrowserCollectionCoordinator | undefined + let fresh: BrowserCollectionCoordinator | undefined + + await withFailurePreservingCleanup(async () => { + first = createCoordinator( + `cleanup-first`, + createRecordingAdapter({ id: `first` }), + ) + peer = createCoordinator( + `cleanup-first`, + createRecordingAdapter({ id: `peer` }), + ) + first.subscribe(`alpha`, () => {}) + peer.subscribe(`alpha`, () => {}) + await waitFor( + () => first!.isLeader(`alpha`) && !peer!.isLeader(`alpha`), + `first lifecycle leadership`, + ) + + const retiredBefore = ControlledBroadcastChannel.retiredDeliveries + first.publish(`alpha`, { + v: 1, + dbName: `cleanup-first`, + collectionId: `alpha`, + senderId: first.getNodeId(), + ts: Date.now(), + payload: { type: `lifecycle:pending` }, + }) + expect(ControlledBroadcastChannel.delayed.length).toBeGreaterThan(0) + await disposeCoordinator(first) + await disposeCoordinator(peer) + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `first lifecycle release`, + ) + expect(ControlledBroadcastChannel.retiredDeliveries).toBeGreaterThan( + retiredBefore, + ) + expect(coordinatorFixtureSnapshot()).toEqual({ + delayed: 0, + endpoints: 0, + heldLocks: 0, + queuedLocks: 0, + }) + + fresh = createCoordinator( + `cleanup-fresh`, + createRecordingAdapter({ id: `fresh` }), + ) + fresh.subscribe(`alpha`, () => {}) + await waitFor( + () => fresh!.isLeader(`alpha`), + `fresh lifecycle leadership`, + ) + expect(fresh.isLeader(`alpha`)).toBe(true) + await disposeCoordinator(fresh) + await waitForWithoutDelivery( + () => heldLocks.size === 0 && queuedLocks.size === 0, + `fresh lifecycle release`, + ) + expect(coordinatorFixtureSnapshot()).toEqual({ + delayed: 0, + endpoints: 0, + heldLocks: 0, + queuedLocks: 0, + }) + }, [ + [ + `first lifecycle coordinator`, + async () => { + await disposeCoordinator(first) + }, + ], + [ + `first lifecycle peer`, + async () => { + await disposeCoordinator(peer) + }, + ], + [ + `fresh lifecycle coordinator`, + async () => { + await disposeCoordinator(fresh) + }, + ], + ]) + }) +}) diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 1ed225957f..87bdc89371 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -25,17 +25,39 @@ binding. Provide a runtime `SQLiteDriver` implementation from a wrapper package. - `TxCommitted` - `EnsureRemoteSubsetRequest` - `EnsureRemoteSubsetResponse` +- `ReleaseRemoteSubsetRequest` +- `ReleaseRemoteSubsetResponse` +- `RemoteSubsetOwner` +- `DuplicateRemoteSubsetOwnerError` +- `TransportedLoadSubsetOptions` +- `RemoteSubsetWirePrimitive` +- `RemoteSubsetWireTypedArray` +- `RemoteSubsetWireRecord` +- `RemoteSubsetWireValue` +- `RemoteSubsetWireExpression` +- `RemoteSubsetWireCompareOptions` +- `RemoteSubsetWireOrderByClause` +- `RemoteSubsetWireCursor` +- `RemoteSubsetWireValueError` +- `toTransportedLoadSubsetOptions(...)` - `ApplyLocalMutationsRequest` - `ApplyLocalMutationsResponse` +- `ApplyCommittedTxRequest` +- `ApplyCommittedTxResponse` - `PullSinceRequest` - `PullSinceResponse` - `CollectionReset` - `PersistedIndexSpec` +- `PersistedRowMetadataMutation` +- `PersistedCollectionMetadataMutation` +- `ReplayableTxDelta` +- `PersistedScannedRow` +- `PersistedRowScanOptions` - `PersistedTx` -- `PersistenceAdapter` +- `PersistenceAdapter` - `SQLiteDriver` - `PersistedCollectionCoordinator` -- `PersistedCollectionPersistence` +- `PersistedCollectionPersistence` - `PersistedCollectionMode` - `PersistedCollectionLeadershipState` - `PersistedCollectionUtils` @@ -64,12 +86,109 @@ and resolves persistence using: This lets runtime wrappers expose one shared persistence instance per database while still handling per-collection schema versions correctly. +### Coordinator contract + +Every `PersistedCollectionCoordinator` must implement +`requestApplyCommittedTx(collectionId, tx)`. A sync transaction with durable +effects is routed in full to the persistence adapter owned by that collection. +The `PersistedTx` includes any truncate, row mutations, row metadata mutations, +collection metadata mutations, and stream position. Multiprocess coordinators +must preserve the complete transaction when they forward it to the elected +writer. This contract does not decide whether an effect-free source commit +should produce a persistence call. + +This is a required invariant, not a capability to detect at commit time. The +TypeScript interface requires the method, and `persistedCollectionOptions(...)` +validates custom coordinators when it configures a collection. An untyped +integration that omits the method throws +`InvalidPersistedCollectionCoordinatorError` before the sync source starts or +publishes rows. Implementations must not fall back to a row-only route such as +`requestApplyLocalMutations(...)`, because that would discard transaction +semantics. + +After a mutating RPC transport failure, replay is allowed only while the +requester can prove the same non-null leader id and term still own the route. +If either value was unknown for the first attempt, or either value changes, +the request rejects with `IndeterminateCommitError` and requires application +reconciliation. Coordinators never retry an indeterminate mutation against an +unknown or replacement leader. + +`SingleProcessCoordinator` also implements the complete route. It is always +leader and has no cross-process transport, but it applies the transaction to +the resolved adapter registered for the specific collection. The Browser and +Electron coordinators provide the corresponding elected-owner route in their +runtime packages. + +An adapter/storage rejection is surfaced as +`PersistedCollectionDurabilityError`, never as `CONFLICT`. The error preserves +the original local cause and its `code` and `path` fields. A transported +`PERSISTENCE_ERROR` response carries the source code/path that can cross the +coordinator boundary. Source writes remain published before their applied +receipt settles; if persistence then fails, the receipt rejects with the named +error and the collection enters its existing `error` lifecycle state. The +collection does not continue, retry, or fall back to a partial writer. + +### Remote subset wire contract + +`requestEnsureRemoteSubset(collectionId, options)` accepts the full live +`LoadSubsetOptions` input. Before choosing a local owner or remote transport, +every built-in coordinator projects it once to the exported +`TransportedLoadSubsetOptions` contract. A registered remote-subset owner +receives that transported type, never live-only fields such as `signal` or +`subscription`. + +Expression values may contain structured-clone-safe primitives, plain records, +arrays (including sparse arrays), `Date`, zero-position `RegExp`, fixed +`ArrayBuffer`, fixed `DataView`, the views listed by the exported +`RemoteSubsetWireTypedArray` union (`Int8Array`, `Uint8Array`, +`Uint8ClampedArray`, `Int16Array`, `Uint16Array`, `Int32Array`, `Uint32Array`, +`Float32Array`, `Float64Array`, `BigInt64Array`, and `BigUint64Array`), `Map`, +and `Set`. Cycles and aliases are preserved. Other view types; shared, +resizable, growable, or detached buffers; accessors; non-enumerable or symbol +keys; native expandos; custom prototypes; promises; weak collections; +functions; and symbols are outside the contract. A rejected value throws the exported +`RemoteSubsetWireValueError` before owner work or transport, with its exact +path in the request. The wire contract does not provide codecs, coercion, or +lossy normalization for unsupported values. + +Remote subset ownership is lease-based. A coordinator gives each accepted +request object a stable acquisition identity scoped to its collection and +requester. Retrying the same acquisition coalesces with the original; a second +request object with equal values is an independent lease. Releasing a lease +calls the registered owner's required `unloadSubset(...)` method with the exact +transported options object that was loaded. Releasing it again is a no-op. +`EnsureRemoteSubsetRequest.acquisitionId` is the wire identity established by +acquisition; `ReleaseRemoteSubsetRequest.acquisitionId` must carry that same +identity so the elected owner releases the matching lease. + +The owner also provides a required `onError(error)` callback. A load or unload +throw, or a runtime thenable rejection from either operation, is reported +through that callback with the original value before the awaited request +rejects. Retirement and disposal observe and report the same failure while +continuing cleanup of sibling leases; they do not retry or silently recover +the failed lifecycle. + +Only one remote subset owner may be registered per collection in a coordinator. +A second live registration throws `DuplicateRemoteSubsetOwnerError` instead of +replacing the first. When a leader loses ownership or is disposed, it unloads +every transferred lease. A requester that still owns a lease replays that same +acquisition when the next leader is observed. There is no adapter fallback or +direct-writer escape hatch for missing ownership. + +The named-error and exact-path guarantee covers values the boundary can +identify with standard JavaScript reflection. A fully transparent `Proxy` is +outside the public wire type and cannot be portably distinguished from its +target. Untyped callers must not rely on Proxy identity, traps, or a +Proxy-specific diagnostic; no non-standard detection is attempted. + ### SQLite core adapter APIs - `SQLiteCoreAdapterOptions` - `SQLitePullSinceResult` -- `SQLiteCorePersistenceAdapter` -- `createSQLiteCorePersistenceAdapter(...)` +- `SQLiteCorePersistenceAdapter` +- `DEFAULT_APPLIED_TX_PRUNE_MAX_ROWS` +- `DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS` +- `createSQLiteCorePersistenceAdapter(...)` ### Error APIs @@ -78,6 +197,12 @@ while still handling per-collection schema versions correctly. - `InvalidSyncConfigError` - `InvalidPersistedCollectionCoordinatorError` - `InvalidPersistenceAdapterError` +- `PersistedCollectionDurabilityErrorOptions` +- `PersistedCollectionDurabilityError` +- `toPersistedCollectionDurabilityError(...)` +- `IndeterminateCommitRequestType` +- `IndeterminateCommitErrorOptions` +- `IndeterminateCommitError` - `InvalidPersistedStorageKeyError` - `InvalidPersistedStorageKeyEncodingError` - `PersistenceUnavailableError` diff --git a/packages/db-sqlite-persistence-core/src/errors.ts b/packages/db-sqlite-persistence-core/src/errors.ts index f0ffc330eb..632f62a074 100644 --- a/packages/db-sqlite-persistence-core/src/errors.ts +++ b/packages/db-sqlite-persistence-core/src/errors.ts @@ -43,6 +43,107 @@ export class InvalidPersistenceAdapterError extends InvalidPersistedCollectionCo } } +export type PersistedCollectionDurabilityErrorOptions = { + cause?: unknown + code?: unknown + path?: unknown +} + +export class PersistedCollectionDurabilityError extends PersistedCollectionCoreError { + override readonly cause: unknown + readonly code: unknown + readonly path: unknown + + constructor( + message: string, + options: PersistedCollectionDurabilityErrorOptions = {}, + ) { + super(message) + this.name = `PersistedCollectionDurabilityError` + this.cause = options.cause + this.code = options.code + this.path = options.path + } +} + +export function toPersistedCollectionDurabilityError( + collectionId: string, + cause: unknown, +): PersistedCollectionDurabilityError { + if (cause instanceof PersistedCollectionDurabilityError) { + return cause + } + + const details = + typeof cause === `object` && cause !== null + ? (cause as Record) + : undefined + const message = cause instanceof Error ? cause.message : String(cause) + return new PersistedCollectionDurabilityError( + `Failed to durably persist collection "${collectionId}": ${message}`, + { + cause, + code: details?.code, + path: details?.path, + }, + ) +} + +export type IndeterminateCommitRequestType = + | `rpc:applyLocalMutations:req` + | `rpc:applyCommittedTx:req` + +export type IndeterminateCommitErrorOptions = { + collectionId: string + requestType: IndeterminateCommitRequestType + previousLeaderId: string | null + previousTerm: number | null + currentLeaderId: string | null + currentTerm: number | null + cause: unknown +} + +export class IndeterminateCommitError extends PersistedCollectionCoreError { + readonly code: `INDETERMINATE_COMMIT` = `INDETERMINATE_COMMIT` + readonly collectionId: string + readonly requestType: IndeterminateCommitRequestType + readonly previousLeaderId: string | null + readonly previousTerm: number | null + readonly currentLeaderId: string | null + readonly currentTerm: number | null + override readonly cause: unknown + + constructor(options: IndeterminateCommitErrorOptions) { + super( + `Commit outcome is indeterminate for collection "${options.collectionId}": ${options.requestType} crossed leadership from ${formatLeader(options.previousLeaderId, options.previousTerm)} to ${formatLeader(options.currentLeaderId, options.currentTerm)}`, + ) + this.name = `IndeterminateCommitError` + this.collectionId = options.collectionId + this.requestType = options.requestType + this.previousLeaderId = options.previousLeaderId + this.previousTerm = options.previousTerm + this.currentLeaderId = options.currentLeaderId + this.currentTerm = options.currentTerm + this.cause = options.cause + } +} + +function formatLeader(leaderId: string | null, term: number | null): string { + return `${leaderId ?? `unknown leader`} (term ${term ?? `unknown`})` +} + +export class DuplicateRemoteSubsetOwnerError extends PersistedCollectionCoreError { + readonly collectionId: string + + constructor(collectionId: string) { + super( + `A remote subset owner is already registered for collection "${collectionId}"`, + ) + this.name = `DuplicateRemoteSubsetOwnerError` + this.collectionId = collectionId + } +} + export class InvalidPersistedStorageKeyError extends InvalidPersistedCollectionConfigError { constructor(key: string | number) { super( diff --git a/packages/db-sqlite-persistence-core/src/index.ts b/packages/db-sqlite-persistence-core/src/index.ts index 9e2bb9faae..963bd84eef 100644 --- a/packages/db-sqlite-persistence-core/src/index.ts +++ b/packages/db-sqlite-persistence-core/src/index.ts @@ -1,5 +1,6 @@ export * from './persisted' export * from './errors' export * from './sqlite-core-adapter' +export * from './remote-subset-wire' // Re-export for use in non-secure browser contexts (see #1541) export { safeRandomUUID } from '@tanstack/db' diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..f47999dfa9 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -6,13 +6,18 @@ import { withCollectionConfigFactory, } from '@tanstack/db' import { + DuplicateRemoteSubsetOwnerError, InvalidPersistedCollectionConfigError, InvalidPersistedCollectionCoordinatorError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, InvalidPersistenceAdapterError, InvalidSyncConfigError, + PersistedCollectionDurabilityError, + toPersistedCollectionDurabilityError, } from './errors' +import { toTransportedLoadSubsetOptions } from './remote-subset-wire' +import type { TransportedLoadSubsetOptions } from './remote-subset-wire' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessageOrDeleteKeyMessage, @@ -38,12 +43,16 @@ export type PersistedMutationEnvelope = type: `insert` key: string | number value: Record + metadata?: unknown + metadataChanged?: boolean } | { mutationId: string type: `update` key: string | number value: Record + metadata?: unknown + metadataChanged?: boolean } | { mutationId: string @@ -96,7 +105,8 @@ export type TxCommitted = { export type EnsureRemoteSubsetRequest = { type: `rpc:ensureRemoteSubset:req` rpcId: string - options: LoadSubsetOptions + acquisitionId: string + options: TransportedLoadSubsetOptions } export type EnsureRemoteSubsetResponse = @@ -104,6 +114,7 @@ export type EnsureRemoteSubsetResponse = type: `rpc:ensureRemoteSubset:res` rpcId: string ok: true + leaderId: string } | { type: `rpc:ensureRemoteSubset:res` @@ -112,6 +123,25 @@ export type EnsureRemoteSubsetResponse = error: string } +export type ReleaseRemoteSubsetRequest = { + type: `rpc:releaseRemoteSubset:req` + rpcId: string + acquisitionId: string +} + +export type ReleaseRemoteSubsetResponse = + | { + type: `rpc:releaseRemoteSubset:res` + rpcId: string + ok: true + } + | { + type: `rpc:releaseRemoteSubset:res` + rpcId: string + ok: false + error: string + } + export type ApplyLocalMutationsRequest = { type: `rpc:applyLocalMutations:req` rpcId: string @@ -136,6 +166,48 @@ export type ApplyLocalMutationsResponse = code: `NOT_LEADER` | `VALIDATION_ERROR` | `CONFLICT` | `TIMEOUT` error: string } + | { + type: `rpc:applyLocalMutations:res` + rpcId: string + ok: false + code: `PERSISTENCE_ERROR` + error: string + sourceCode?: string | number + path?: string | ReadonlyArray + } + +export type ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req` + rpcId: string + envelopeId: string + tx: PersistedTx +} + +export type ApplyCommittedTxResponse = + | { + type: `rpc:applyCommittedTx:res` + rpcId: string + ok: true + term: number + seq: number + latestRowVersion: number + } + | { + type: `rpc:applyCommittedTx:res` + rpcId: string + ok: false + code: `NOT_LEADER` | `CONFLICT` | `TIMEOUT` + error: string + } + | { + type: `rpc:applyCommittedTx:res` + rpcId: string + ok: false + code: `PERSISTENCE_ERROR` + error: string + sourceCode?: string | number + path?: string | ReadonlyArray + } export type PullSinceRequest = { type: `rpc:pullSince:req` @@ -282,6 +354,47 @@ export interface PersistenceAdapter { }> } +export type RemoteSubsetOwner = (( + options: TransportedLoadSubsetOptions, +) => Promise | void) & { + unloadSubset: (options: TransportedLoadSubsetOptions) => void + onError: (error: unknown) => void +} + +function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} + +type SingleProcessRemoteSubsetAcquisition = { + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise +} + export interface SQLiteDriver { exec: (sql: string) => Promise query: ( @@ -306,10 +419,18 @@ export interface PersistedCollectionCoordinator { publish: (collectionId: string, message: ProtocolEnvelope) => void isLeader: (collectionId: string) => boolean ensureLeadership: (collectionId: string) => Promise - requestEnsureRemoteSubset?: ( + requestEnsureRemoteSubset: ( collectionId: string, options: LoadSubsetOptions, ) => Promise + requestReleaseRemoteSubset: ( + collectionId: string, + options: LoadSubsetOptions, + ) => Promise + registerRemoteSubsetOwner: ( + collectionId: string, + owner: RemoteSubsetOwner, + ) => () => void requestEnsurePersistedIndex: ( collectionId: string, signature: string, @@ -319,6 +440,10 @@ export interface PersistedCollectionCoordinator { collectionId: string, mutations: Array, ) => Promise + requestApplyCommittedTx: ( + collectionId: string, + tx: PersistedTx, + ) => Promise pullSince?: ( collectionId: string, fromRowVersion: number, @@ -405,7 +530,11 @@ const REQUIRED_COORDINATOR_METHODS: ReadonlyArray< | `publish` | `isLeader` | `ensureLeadership` + | `requestEnsureRemoteSubset` + | `requestReleaseRemoteSubset` + | `registerRemoteSubsetOwner` | `requestEnsurePersistedIndex` + | `requestApplyCommittedTx` > > = [ `getNodeId`, @@ -413,7 +542,11 @@ const REQUIRED_COORDINATOR_METHODS: ReadonlyArray< `publish`, `isLeader`, `ensureLeadership`, + `requestEnsureRemoteSubset`, + `requestReleaseRemoteSubset`, + `registerRemoteSubsetOwner`, `requestEnsurePersistedIndex`, + `requestApplyCommittedTx`, ] const REQUIRED_ADAPTER_METHODS: ReadonlyArray< @@ -440,6 +573,7 @@ type SyncControlFns = { commit: ((signal?: AbortSignal) => SyncAppliedReceipt) | null truncate: (() => void) | null metadata: SyncMetadataApi | null + markError: ((error: unknown) => void) | null } /** @@ -448,6 +582,12 @@ type SyncControlFns = { */ export class SingleProcessCoordinator implements PersistedCollectionCoordinator { private readonly nodeId: string + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetAcquisitions = new Map< + string, + Map + >() constructor(nodeId: string = safeRandomUUID()) { this.nodeId = nodeId @@ -469,10 +609,128 @@ export class SingleProcessCoordinator implements PersistedCollectionCoordinator public async ensureLeadership(): Promise {} - public async requestEnsureRemoteSubset(): Promise {} + public async requestEnsureRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const transported = toTransportedLoadSubsetOptions(options) + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new InvalidPersistedCollectionConfigError( + `SingleProcessCoordinator has no remote subset owner configured for collection "${collectionId}"`, + ) + } + let acquisitions = this.remoteSubsetAcquisitions.get(collectionId) + if (!acquisitions) { + acquisitions = new Map() + this.remoteSubsetAcquisitions.set(collectionId, acquisitions) + } + const existing = acquisitions.get(options) + if (existing) { + await existing.load + return + } + + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + const load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + acquisitions.set(options, { owner, options: transported, load }) + try { + const ownerLoad = owner(transported) + void Promise.resolve(ownerLoad).then(resolveLoad, (error) => { + reportRemoteSubsetOwnerError(owner, error) + rejectLoad(error) + }) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + rejectLoad(error) + } + await load + } + + public async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const acquisitions = this.remoteSubsetAcquisitions.get(collectionId) + const acquisition = acquisitions?.get(options) + if (!acquisition) return + acquisitions!.delete(options) + if (acquisitions!.size === 0) { + this.remoteSubsetAcquisitions.delete(collectionId) + } + try { + await acquisition.load + } catch { + // Calling the owner transferred the lease even when its load rejected. + } + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + + public registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + const acquisitions = this.remoteSubsetAcquisitions.get(collectionId) + this.remoteSubsetAcquisitions.delete(collectionId) + this.remoteSubsetOwners.delete(collectionId) + for (const acquisition of acquisitions?.values() ?? []) { + void (async () => { + try { + await acquisition.load + } catch { + // Calling the owner transferred the lease even when its load rejected. + } + await unloadRemoteSubsetOwner(owner, acquisition.options) + })().catch(() => undefined) + } + } + } public async requestEnsurePersistedIndex(): Promise {} + public setAdapterForCollection( + collectionId: string, + adapter: PersistenceAdapter, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + public async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const adapter = this.collectionAdapters.get(collectionId) + if (!adapter) { + throw new InvalidPersistedCollectionConfigError( + `SingleProcessCoordinator has no persistence adapter configured for collection "${collectionId}"`, + ) + } + + try { + await adapter.applyCommittedTx(collectionId, tx) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: safeRandomUUID(), + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + } + public pullSince(): Promise { return Promise.resolve({ type: `rpc:pullSince:res`, @@ -509,10 +767,14 @@ function validatePersistenceAdapter(adapter: PersistenceAdapter): void { function resolvePersistence( persistence: PersistedCollectionPersistence, + collectionId: string, ): PersistedResolvedPersistence { validatePersistenceAdapter(persistence.adapter) const coordinator = persistence.coordinator ?? new SingleProcessCoordinator() + if (coordinator instanceof SingleProcessCoordinator) { + coordinator.setAdapterForCollection(collectionId, persistence.adapter) + } validatePersistedCollectionCoordinator(coordinator) return { @@ -524,9 +786,13 @@ function resolvePersistence( function resolvePersistenceForMode( persistence: PersistedCollectionPersistence, mode: PersistedCollectionMode, + collectionId: string, ): PersistedResolvedPersistence { const modeSpecificPersistence = persistence.resolvePersistenceForMode?.(mode) - return resolvePersistence(modeSpecificPersistence ?? persistence) + return resolvePersistence( + modeSpecificPersistence ?? persistence, + collectionId, + ) } function resolvePersistenceForCollection( @@ -540,10 +806,17 @@ function resolvePersistenceForCollection( const collectionSpecificPersistence = persistence.resolvePersistenceForCollection?.(options) if (collectionSpecificPersistence) { - return resolvePersistence(collectionSpecificPersistence) + return resolvePersistence( + collectionSpecificPersistence, + options.collectionId, + ) } - return resolvePersistenceForMode(persistence, options.mode) + return resolvePersistenceForMode( + persistence, + options.mode, + options.collectionId, + ) } function hasOwnSyncKey(options: object): options is { sync: unknown } { @@ -801,6 +1074,7 @@ class PersistedCollectionRuntime< commit: null, truncate: null, metadata: null, + markError: null, } private started = false private startupMetadataPromise: Promise | null = null @@ -809,9 +1083,11 @@ class PersistedCollectionRuntime< private lifecycleGeneration = 0 private internalApplyDepth = 0 private appliedReceiptSequence = 0 + private syncErrorReported = false private readonly pendingAppliedReceipts = new Map>() private hydratingGeneration: number | null = null private coordinatorUnsubscribe: (() => void) | null = null + private remoteSubsetOwnerUnsubscribe: (() => void) | null = null private indexAddedUnsubscribe: (() => void) | null = null private indexRemovedUnsubscribe: (() => void) | null = null private remoteEnsureRetryTimer: ReturnType | null = null @@ -834,6 +1110,7 @@ class PersistedCollectionRuntime< setSyncControls(syncControls: SyncControlFns): void { this.advanceLifecycle() + this.syncErrorReported = false const commit = syncControls.commit this.syncControls = { @@ -844,6 +1121,27 @@ class PersistedCollectionRuntime< } } + reportSyncError(error: unknown): void { + const markError = this.syncControls.markError + if (this.syncErrorReported || !markError) return + + this.syncErrorReported = true + try { + markError(error) + } catch { + // Reporting must not replace the original asynchronous failure. + } + } + + registerRemoteSubsetOwner(owner: RemoteSubsetOwner): void { + this.remoteSubsetOwnerUnsubscribe?.() + this.remoteSubsetOwnerUnsubscribe = + this.persistence.coordinator.registerRemoteSubsetOwner( + this.collectionId, + owner, + ) + } + private trackAppliedReceipt(receipt: SyncAppliedReceipt): SyncAppliedReceipt { const sequence = ++this.appliedReceiptSequence if (receipt === true) { @@ -870,6 +1168,7 @@ class PersistedCollectionRuntime< commit: null, truncate: null, metadata: null, + markError: null, } } @@ -1045,18 +1344,45 @@ class PersistedCollectionRuntime< upstreamLoadSubset?: LoadSubsetFn, ): Promise { const lifecycleGeneration = this.lifecycleGeneration + const routeRemoteDemandThroughCoordinator = + this.mode === `sync-present` && + !(this.persistence.coordinator instanceof SingleProcessCoordinator) this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence await this.applyMutex.run(() => this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, + requestRemoteEnsure: + this.mode === `sync-present` && !routeRemoteDemandThroughCoordinator, lifecycleGeneration, }), ) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) + if (routeRemoteDemandThroughCoordinator) { + try { + await this.persistence.coordinator.requestEnsureRemoteSubset( + this.collectionId, + options, + ) + } catch (error) { + if ( + options.signal?.aborted || + (typeof error === `object` && + error !== null && + `name` in error && + error.name === `AbortError`) + ) { + this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) + throw error + } + this.queueRemoteSubsetEnsure(options) + throw error + } + return + } + if (upstreamLoadSubset) { try { await upstreamLoadSubset(options) @@ -1085,7 +1411,27 @@ class PersistedCollectionRuntime< ): void { this.activeSubsets.delete(this.getSubsetKey(options)) this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) - upstreamUnloadSubset?.(options) + if (this.mode === `sync-present`) { + void this.persistence.coordinator + .requestReleaseRemoteSubset(this.collectionId, options) + .catch((error) => { + this.reportSyncError(error) + }) + } + if (upstreamUnloadSubset) { + try { + const result = ( + upstreamUnloadSubset as unknown as ( + options: LoadSubsetOptions, + ) => unknown + )(options) + void Promise.resolve(result).catch((error) => { + this.reportSyncError(error) + }) + } catch (error) { + this.reportSyncError(error) + } + } } async forceReloadSubset(options: LoadSubsetOptions): Promise { @@ -1216,9 +1562,22 @@ class PersistedCollectionRuntime< cleanup(): void { this.advanceLifecycle() + if (this.mode === `sync-present`) { + for (const options of this.activeSubsets.values()) { + void this.persistence.coordinator + .requestReleaseRemoteSubset(this.collectionId, options) + .catch((error) => { + this.reportSyncError(error) + }) + } + } + this.coordinatorUnsubscribe?.() this.coordinatorUnsubscribe = null + this.remoteSubsetOwnerUnsubscribe?.() + this.remoteSubsetOwnerUnsubscribe = null + this.indexAddedUnsubscribe?.() this.indexAddedUnsubscribe = null @@ -1519,27 +1878,39 @@ class PersistedCollectionRuntime< } const tx = this.createPersistedTxFromOperations(transaction, streamPosition) - - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) - this.publishTxCommittedEvent( - this.createTxCommittedPayload({ - term: tx.term, - seq: tx.seq, - txId: tx.txId, - latestRowVersion: tx.rowVersion, - requiresFullReload: transaction.truncate, - changedRows: transaction.operations - .filter((operation) => operation.type === `update`) - .map((operation) => ({ - key: operation.key, - value: operation.value as Record, - })), - deletedKeys: transaction.operations - .filter((operation) => operation.type === `delete`) - .map((operation) => operation.key), - rowMetadataMutations: tx.rowMetadataMutations, - collectionMetadataMutations: tx.collectionMetadataMutations, - }), + let response: ApplyCommittedTxResponse + try { + response = await this.persistence.coordinator.requestApplyCommittedTx( + this.collectionId, + tx, + ) + } catch (error) { + if (error instanceof PersistedCollectionDurabilityError) { + this.reportSyncError(error) + } + throw error + } + if (!response.ok) { + if (response.code === `PERSISTENCE_ERROR`) { + const error = new PersistedCollectionDurabilityError( + `Failed to durably persist collection "${this.collectionId}": ${response.error}`, + { + cause: response, + code: response.sourceCode ?? response.code, + path: response.path, + }, + ) + this.reportSyncError(error) + throw error + } + throw new Error( + `failed to apply external sync transaction through coordinator: ${response.error}`, + ) + } + this.observeStreamPosition( + response.term, + response.seq, + response.latestRowVersion, ) } @@ -1871,7 +2242,7 @@ class PersistedCollectionRuntime< private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { if ( this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset || + this.persistence.coordinator instanceof SingleProcessCoordinator || this.activeSubsets.get(this.getSubsetKey(options)) !== options ) { return @@ -1884,7 +2255,7 @@ class PersistedCollectionRuntime< private scheduleRemoteEnsureRetry(): void { if ( this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset + this.persistence.coordinator instanceof SingleProcessCoordinator ) { return } @@ -1905,7 +2276,7 @@ class PersistedCollectionRuntime< private async flushPendingRemoteSubsetEnsures(): Promise { if ( this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset + this.persistence.coordinator instanceof SingleProcessCoordinator ) { return } @@ -2337,6 +2708,7 @@ function createWrappedSyncConfig< commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, @@ -2640,6 +3012,25 @@ function createWrappedSyncConfig< sourceResult = normalizeSyncFnResult( sourceSyncConfig.sync(wrappedParams), ) + if (sourceResult.loadSubset) { + const loadSubset = async (options: TransportedLoadSubsetOptions) => { + if (startupState.cleanedUp) { + throw new Error(`persisted sync source is no longer active`) + } + await sourceResult.loadSubset?.( + options as unknown as LoadSubsetOptions, + ) + } + runtime.registerRemoteSubsetOwner( + Object.assign(loadSubset, { + unloadSubset: (options: TransportedLoadSubsetOptions) => + sourceResult.unloadSubset?.( + options as unknown as LoadSubsetOptions, + ), + onError: (error: unknown) => runtime.reportSyncError(error), + }), + ) + } return sourceResult })() @@ -2708,6 +3099,7 @@ function createLoopbackSyncConfig< commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, diff --git a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts new file mode 100644 index 0000000000..407a0534c4 --- /dev/null +++ b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts @@ -0,0 +1,899 @@ +import { IR } from '@tanstack/db' +import type { LoadSubsetOptions } from '@tanstack/db' + +export type RemoteSubsetWirePrimitive = + | undefined + | null + | boolean + | string + | number + | bigint + +export type RemoteSubsetWireTypedArray = + | Int8Array + | Uint8Array + | Uint8ClampedArray + | Int16Array + | Uint16Array + | Int32Array + | Uint32Array + | Float32Array + | Float64Array + | BigInt64Array + | BigUint64Array + +export type RemoteSubsetWireRecord = { + [key: string]: RemoteSubsetWireValue +} + +/** Values whose complete semantics survive a coordinator transport boundary. */ +export type RemoteSubsetWireValue = + | RemoteSubsetWirePrimitive + | Date + | RegExp + | ArrayBuffer + | DataView + | RemoteSubsetWireTypedArray + | Array + | Map + | Set + | RemoteSubsetWireRecord + +export type RemoteSubsetWireExpression = + | { type: `ref`; path: Array } + | { type: `val`; value: RemoteSubsetWireValue } + | { + type: `func` + name: string + args: Array + } + +export type RemoteSubsetWireCompareOptions = { + direction: `asc` | `desc` + nulls: `first` | `last` +} & ( + | { stringSort?: `lexical` } + | { + stringSort?: `locale` + locale?: string + localeOptions?: RemoteSubsetWireRecord + } +) + +export type RemoteSubsetWireOrderByClause = { + expression: RemoteSubsetWireExpression + compareOptions: RemoteSubsetWireCompareOptions +} + +export type RemoteSubsetWireCursor = { + whereFrom: RemoteSubsetWireExpression + whereCurrent: RemoteSubsetWireExpression + lastKey?: string | number +} + +/** The complete data portion transported by a remote-subset request. */ +export type TransportedLoadSubsetOptions = { + where?: RemoteSubsetWireExpression + orderBy?: Array + limit?: number + cursor?: RemoteSubsetWireCursor + offset?: number +} + +export class RemoteSubsetWireValueError extends TypeError { + override readonly name = `RemoteSubsetWireValueError` + + constructor( + readonly path: string, + reason: string, + ) { + super(`Unsupported remote subset wire value at ${path}: ${reason}`) + } +} + +type ProjectionState = { + expressions: WeakMap + expressionArrays: WeakMap> + stringArrays: WeakMap> + orderByArrays: WeakMap> + orderByClauses: WeakMap + compareOptions: WeakMap + cursors: WeakMap + wireValues: WeakMap +} + +type DataProperty = { + present: boolean + value?: unknown +} + +type TypedArrayConstructor = { + readonly prototype: RemoteSubsetWireTypedArray + new ( + buffer: ArrayBuffer, + byteOffset?: number, + length?: number, + ): RemoteSubsetWireTypedArray +} + +const typedArrayConstructors: ReadonlyArray = [ + Int8Array, + Uint8Array, + Uint8ClampedArray, + Int16Array, + Uint16Array, + Int32Array, + Uint32Array, + Float32Array, + Float64Array, + BigInt64Array, + BigUint64Array, +] + +const NativeDate = Date +const identifierPattern = /^[A-Za-z_$][A-Za-z0-9_$]*$/ + +/** + * Projects one live subset request into the exact coordinator wire domain. + * Validation happens before callers choose a local-leader or transported path. + */ +export function toTransportedLoadSubsetOptions( + options: LoadSubsetOptions, +): TransportedLoadSubsetOptions { + assertPlainContainer(options, `options`) + assertAllowedProperties(options, `options`, [ + `where`, + `orderBy`, + `limit`, + `cursor`, + `offset`, + `signal`, + `subscription`, + ]) + + const state: ProjectionState = { + expressions: new WeakMap(), + expressionArrays: new WeakMap(), + stringArrays: new WeakMap(), + orderByArrays: new WeakMap(), + orderByClauses: new WeakMap(), + compareOptions: new WeakMap(), + cursors: new WeakMap(), + wireValues: new WeakMap(), + } + const projected: TransportedLoadSubsetOptions = {} + const where = readDataProperty(options, `where`, `options.where`) + const orderBy = readDataProperty(options, `orderBy`, `options.orderBy`) + const limit = readDataProperty(options, `limit`, `options.limit`) + const cursor = readDataProperty(options, `cursor`, `options.cursor`) + const offset = readDataProperty(options, `offset`, `options.offset`) + + if (where.present && where.value !== undefined) { + projected.where = projectExpression(where.value, `options.where`, state) + } + if (orderBy.present && orderBy.value !== undefined) { + projected.orderBy = projectOrderBy(orderBy.value, `options.orderBy`, state) + } + if (limit.present && limit.value !== undefined) { + projected.limit = projectNumber(limit.value, `options.limit`) + } + if (cursor.present && cursor.value !== undefined) { + projected.cursor = projectCursor(cursor.value, `options.cursor`, state) + } + if (offset.present && offset.value !== undefined) { + projected.offset = projectNumber(offset.value, `options.offset`) + } + + return projected +} + +function projectExpression( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireExpression { + const object = requireObject(value, path) + const existing = state.expressions.get(object) + if (existing) return existing + + const type = readDataProperty(object, `type`, `${path}.type`) + if (!type.present || typeof type.value !== `string`) { + throw new RemoteSubsetWireValueError(`${path}.type`, describe(type.value)) + } + + switch (type.value) { + case `ref`: { + assertExpressionPrototype(object, path, IR.PropRef.prototype) + assertAllowedProperties(object, path, [`type`, `path`]) + const projected = { + type: `ref`, + path: [] as Array, + } satisfies RemoteSubsetWireExpression + state.expressions.set(object, projected) + const sourcePath = readRequiredDataProperty( + object, + `path`, + `${path}.path`, + ) + projected.path = projectStringArray(sourcePath, `${path}.path`, state) + return projected + } + case `val`: { + assertExpressionPrototype(object, path, IR.Value.prototype) + assertAllowedProperties(object, path, [`type`, `value`]) + const projected: Extract = { + type: `val`, + value: undefined, + } + state.expressions.set(object, projected) + const sourceValue = readRequiredDataProperty( + object, + `value`, + `${path}.value`, + ) + projected.value = projectWireValue(sourceValue, `${path}.value`, state) + return projected + } + case `func`: { + assertExpressionPrototype(object, path, IR.Func.prototype) + assertAllowedProperties(object, path, [`type`, `name`, `args`]) + const projected = { + type: `func`, + name: ``, + args: [] as Array, + } satisfies RemoteSubsetWireExpression + state.expressions.set(object, projected) + const name = readRequiredDataProperty(object, `name`, `${path}.name`) + if (typeof name !== `string`) { + throw new RemoteSubsetWireValueError(`${path}.name`, describe(name)) + } + projected.name = name + const args = readRequiredDataProperty(object, `args`, `${path}.args`) + projected.args = projectExpressionArray(args, `${path}.args`, state) + return projected + } + default: + throw new RemoteSubsetWireValueError(`${path}.type`, String(type.value)) + } +} + +function projectExpressionArray( + value: unknown, + path: string, + state: ProjectionState, +): Array { + const array = requireArray(value, path) + const existing = state.expressionArrays.get(array) + if (existing) return existing + assertArrayShape(array, path) + const projected = new Array(array.length) + state.expressionArrays.set(array, projected) + for (let index = 0; index < array.length; index++) { + if (!(index in array)) { + throw new RemoteSubsetWireValueError( + `${path}[${index}]`, + `missing expression`, + ) + } + projected[index] = projectExpression( + array[index], + `${path}[${index}]`, + state, + ) + } + return projected +} + +function projectStringArray( + value: unknown, + path: string, + state: ProjectionState, +): Array { + const array = requireArray(value, path) + const existing = state.stringArrays.get(array) + if (existing) return existing + assertArrayShape(array, path) + const projected = new Array(array.length) + state.stringArrays.set(array, projected) + for (let index = 0; index < array.length; index++) { + const entry = array[index] + if (!(index in array) || typeof entry !== `string`) { + throw new RemoteSubsetWireValueError(`${path}[${index}]`, describe(entry)) + } + projected[index] = entry + } + return projected +} + +function projectOrderBy( + value: unknown, + path: string, + state: ProjectionState, +): Array { + const array = requireArray(value, path) + const existing = state.orderByArrays.get(array) + if (existing) return existing + assertArrayShape(array, path) + const projected = new Array(array.length) + state.orderByArrays.set(array, projected) + + for (let index = 0; index < array.length; index++) { + const clausePath = `${path}[${index}]` + const clause = requirePlainRecord(array[index], clausePath) + const existingClause = state.orderByClauses.get(clause) + if (existingClause) { + projected[index] = existingClause + continue + } + assertAllowedProperties(clause, clausePath, [ + `expression`, + `compareOptions`, + ]) + const projectedClause = { + expression: undefined as unknown as RemoteSubsetWireExpression, + compareOptions: undefined as unknown as RemoteSubsetWireCompareOptions, + } + projected[index] = projectedClause + state.orderByClauses.set(clause, projectedClause) + projectedClause.expression = projectExpression( + readRequiredDataProperty( + clause, + `expression`, + `${clausePath}.expression`, + ), + `${clausePath}.expression`, + state, + ) + projectedClause.compareOptions = projectCompareOptions( + readRequiredDataProperty( + clause, + `compareOptions`, + `${clausePath}.compareOptions`, + ), + `${clausePath}.compareOptions`, + state, + ) + } + return projected +} + +function projectCompareOptions( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireCompareOptions { + const source = requirePlainRecord(value, path) + const existing = state.compareOptions.get(source) + if (existing) return existing + assertAllowedProperties(source, path, [ + `direction`, + `nulls`, + `stringSort`, + `locale`, + `localeOptions`, + ]) + const projected: { + direction?: `asc` | `desc` + nulls?: `first` | `last` + stringSort?: `lexical` | `locale` + locale?: string + localeOptions?: RemoteSubsetWireRecord + } = {} + + const direction = readRequiredDataProperty( + source, + `direction`, + `${path}.direction`, + ) + if (direction !== `asc` && direction !== `desc`) { + throw new RemoteSubsetWireValueError( + `${path}.direction`, + describe(direction), + ) + } + projected.direction = direction + + const nulls = readRequiredDataProperty(source, `nulls`, `${path}.nulls`) + if (nulls !== `first` && nulls !== `last`) { + throw new RemoteSubsetWireValueError(`${path}.nulls`, describe(nulls)) + } + projected.nulls = nulls + + const stringSort = readDataProperty( + source, + `stringSort`, + `${path}.stringSort`, + ) + if (stringSort.present && stringSort.value !== undefined) { + if (stringSort.value !== `lexical` && stringSort.value !== `locale`) { + throw new RemoteSubsetWireValueError( + `${path}.stringSort`, + describe(stringSort.value), + ) + } + projected.stringSort = stringSort.value + } + const locale = readDataProperty(source, `locale`, `${path}.locale`) + if (locale.present && locale.value !== undefined) { + if (typeof locale.value !== `string`) { + throw new RemoteSubsetWireValueError( + `${path}.locale`, + describe(locale.value), + ) + } + projected.locale = locale.value + } + const localeOptions = readDataProperty( + source, + `localeOptions`, + `${path}.localeOptions`, + ) + if (localeOptions.present && localeOptions.value !== undefined) { + projected.localeOptions = projectWireRecord( + localeOptions.value, + `${path}.localeOptions`, + state, + ) + } + const result = projected as RemoteSubsetWireCompareOptions + state.compareOptions.set(source, result) + return result +} + +function projectCursor( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireCursor { + const source = requirePlainRecord(value, path) + const existing = state.cursors.get(source) + if (existing) return existing + assertAllowedProperties(source, path, [ + `whereFrom`, + `whereCurrent`, + `lastKey`, + ]) + const projected: RemoteSubsetWireCursor = { + whereFrom: undefined as unknown as RemoteSubsetWireExpression, + whereCurrent: undefined as unknown as RemoteSubsetWireExpression, + } + state.cursors.set(source, projected) + projected.whereFrom = projectExpression( + readRequiredDataProperty(source, `whereFrom`, `${path}.whereFrom`), + `${path}.whereFrom`, + state, + ) + projected.whereCurrent = projectExpression( + readRequiredDataProperty(source, `whereCurrent`, `${path}.whereCurrent`), + `${path}.whereCurrent`, + state, + ) + const lastKey = readDataProperty(source, `lastKey`, `${path}.lastKey`) + if (lastKey.present && lastKey.value !== undefined) { + if ( + typeof lastKey.value !== `string` && + typeof lastKey.value !== `number` + ) { + throw new RemoteSubsetWireValueError( + `${path}.lastKey`, + describe(lastKey.value), + ) + } + projected.lastKey = lastKey.value + } + return projected +} + +function projectWireValue( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireValue { + if ( + value === undefined || + value === null || + typeof value === `boolean` || + typeof value === `string` || + typeof value === `number` || + typeof value === `bigint` + ) { + return value + } + if (typeof value !== `object`) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + + const existing = state.wireValues.get(value) + if (existing) return existing + + if ( + typeof SharedArrayBuffer !== `undefined` && + value instanceof SharedArrayBuffer + ) { + throw new RemoteSubsetWireValueError(path, `SharedArrayBuffer`) + } + if (value instanceof NativeDate) { + assertExactPrototype(value, NativeDate.prototype, path) + assertNoOwnProperties(value, path) + const projected = new NativeDate(value.getTime()) + state.wireValues.set(value, projected) + return projected + } + if (value instanceof RegExp) { + assertExactPrototype(value, RegExp.prototype, path) + assertRegExpShape(value, path) + const projected = new RegExp(value.source, value.flags) + state.wireValues.set(value, projected) + return projected + } + if (value instanceof ArrayBuffer) { + assertExactPrototype(value, ArrayBuffer.prototype, path) + assertNoOwnProperties(value, path) + assertFixedArrayBuffer(value, path) + let projected: ArrayBuffer + try { + projected = value.slice(0) + } catch { + throw new RemoteSubsetWireValueError(path, `detached ArrayBuffer`) + } + state.wireValues.set(value, projected) + return projected + } + if (value instanceof DataView) { + assertExactPrototype(value, DataView.prototype, path) + assertNoOwnProperties(value, path) + let byteOffset: number + let byteLength: number + let buffer: ArrayBufferLike + try { + byteOffset = value.byteOffset + byteLength = value.byteLength + buffer = value.buffer + } catch { + throw new RemoteSubsetWireValueError(path, `detached DataView`) + } + const projectedBuffer = projectWireValue(buffer, path, state) + if (!(projectedBuffer instanceof ArrayBuffer)) { + throw new RemoteSubsetWireValueError(path, `shared DataView buffer`) + } + const projected = new DataView(projectedBuffer, byteOffset, byteLength) + state.wireValues.set(value, projected) + return projected + } + + const typedArrayConstructor = getTypedArrayConstructor(value) + if (typedArrayConstructor) { + const typedArray = value as RemoteSubsetWireTypedArray + assertTypedArrayShape(typedArray, path) + let byteOffset: number + let length: number + let buffer: ArrayBufferLike + try { + byteOffset = typedArray.byteOffset + length = typedArray.length + buffer = typedArray.buffer + } catch { + throw new RemoteSubsetWireValueError(path, `detached typed array`) + } + if (!(buffer instanceof ArrayBuffer)) { + throw new RemoteSubsetWireValueError(path, `shared typed-array buffer`) + } + const projectedBuffer = projectWireValue(buffer, path, state) + if (!(projectedBuffer instanceof ArrayBuffer)) { + throw new RemoteSubsetWireValueError(path, `shared typed-array buffer`) + } + let projected: RemoteSubsetWireTypedArray + try { + projected = new typedArrayConstructor(projectedBuffer, byteOffset, length) + } catch { + throw new RemoteSubsetWireValueError(path, `detached typed array`) + } + state.wireValues.set(value, projected) + return projected + } + if (ArrayBuffer.isView(value)) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + if (Array.isArray(value)) return projectWireArray(value, path, state) + if (value instanceof Map) return projectWireMap(value, path, state) + if (value instanceof Set) return projectWireSet(value, path, state) + return projectWireRecord(value, path, state) +} + +function projectWireArray( + value: Array, + path: string, + state: ProjectionState, +): Array { + assertArrayShape(value, path) + const projected = new Array(value.length) + state.wireValues.set(value, projected) + for (let index = 0; index < value.length; index++) { + if (index in value) { + projected[index] = projectWireValue( + value[index], + `${path}[${index}]`, + state, + ) + } + } + return projected +} + +function projectWireMap( + value: Map, + path: string, + state: ProjectionState, +): Map { + assertExactPrototype(value, Map.prototype, path) + assertNoOwnProperties(value, path) + const projected = new Map() + state.wireValues.set(value, projected) + let index = 0 + for (const [key, entry] of value) { + projected.set( + projectWireValue(key, `${path}.entries[${index}].key`, state), + projectWireValue(entry, `${path}.entries[${index}].value`, state), + ) + index++ + } + return projected +} + +function projectWireSet( + value: Set, + path: string, + state: ProjectionState, +): Set { + assertExactPrototype(value, Set.prototype, path) + assertNoOwnProperties(value, path) + const projected = new Set() + state.wireValues.set(value, projected) + let index = 0 + for (const entry of value) { + projected.add(projectWireValue(entry, `${path}.values[${index}]`, state)) + index++ + } + return projected +} + +function projectWireRecord( + value: unknown, + path: string, + state: ProjectionState, +): RemoteSubsetWireRecord { + const source = requirePlainRecord(value, path) + const existing = state.wireValues.get(source) + if (existing) return existing as RemoteSubsetWireRecord + const projected: RemoteSubsetWireRecord = {} + state.wireValues.set(source, projected) + + for (const key of ownKeys(source, path)) { + const childPath = propertyPath(path, key) + if (typeof key === `symbol`) { + throw new RemoteSubsetWireValueError(childPath, `symbol property key`) + } + const descriptor = Object.getOwnPropertyDescriptor(source, key) + if (!descriptor || !descriptor.enumerable || !(`value` in descriptor)) { + throw new RemoteSubsetWireValueError(childPath, `non-data property`) + } + Object.defineProperty(projected, key, { + value: projectWireValue(descriptor.value, childPath, state), + enumerable: true, + writable: true, + configurable: true, + }) + } + return projected +} + +function projectNumber(value: unknown, path: string): number { + if (typeof value !== `number`) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + return value +} + +function readRequiredDataProperty( + value: object, + key: string, + path: string, +): unknown { + const property = readDataProperty(value, key, path) + if (!property.present) { + throw new RemoteSubsetWireValueError(path, `missing property`) + } + return property.value +} + +function readDataProperty( + value: object, + key: string, + path: string, +): DataProperty { + let descriptor: PropertyDescriptor | undefined + try { + descriptor = Object.getOwnPropertyDescriptor(value, key) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable property`) + } + if (!descriptor) return { present: false } + if (!descriptor.enumerable || !(`value` in descriptor)) { + throw new RemoteSubsetWireValueError(path, `non-data property`) + } + return { present: true, value: descriptor.value } +} + +function assertAllowedProperties( + value: object, + path: string, + allowed: ReadonlyArray, +): void { + const allowedSet = new Set(allowed) + for (const key of ownKeys(value, path)) { + if (typeof key === `symbol` || !allowedSet.has(key)) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + readDataProperty(value, key, propertyPath(path, key)) + } +} + +function assertPlainContainer(value: object, path: string): void { + assertExactPrototype(value, Object.prototype, path) +} + +function assertExpressionPrototype( + value: object, + path: string, + expressionPrototype: object, +): void { + let prototype: object | null + try { + prototype = Object.getPrototypeOf(value) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable prototype`) + } + if (prototype !== Object.prototype && prototype !== expressionPrototype) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } +} + +function assertExactPrototype( + value: object, + expected: object, + path: string, +): void { + let prototype: object | null + try { + prototype = Object.getPrototypeOf(value) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable prototype`) + } + if (prototype !== expected) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } +} + +function assertNoOwnProperties(value: object, path: string): void { + const keys = ownKeys(value, path) + if (keys.length > 0) { + throw new RemoteSubsetWireValueError( + propertyPath(path, keys[0]!), + `property`, + ) + } +} + +function assertFixedArrayBuffer(value: ArrayBuffer, path: string): void { + if ((value as ArrayBuffer & { readonly resizable?: boolean }).resizable) { + throw new RemoteSubsetWireValueError(path, `resizable ArrayBuffer`) + } +} + +function assertRegExpShape(value: RegExp, path: string): void { + const keys = ownKeys(value, path) + for (const key of keys) { + if (key !== `lastIndex`) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + } + const descriptor = Object.getOwnPropertyDescriptor(value, `lastIndex`) + if (!descriptor || !(`value` in descriptor) || descriptor.value !== 0) { + throw new RemoteSubsetWireValueError( + `${path}.lastIndex`, + `RegExp lastIndex must be 0`, + ) + } +} + +function assertTypedArrayShape( + value: RemoteSubsetWireTypedArray, + path: string, +): void { + const keys = ownKeys(value, path) + for (const key of keys) { + if (typeof key !== `string` || !isCanonicalArrayIndex(key, value.length)) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + } +} + +function assertArrayShape(value: Array, path: string): void { + assertExactPrototype(value, Array.prototype, path) + const keys = ownKeys(value, path) + for (const key of keys) { + if ( + key !== `length` && + (typeof key !== `string` || !isCanonicalArrayIndex(key, value.length)) + ) { + throw new RemoteSubsetWireValueError(propertyPath(path, key), `property`) + } + if (key !== `length`) { + readDataProperty(value, key, propertyPath(path, key)) + } + } +} + +function isCanonicalArrayIndex(key: string, length: number): boolean { + if (!/^(0|[1-9][0-9]*)$/.test(key)) return false + const index = Number(key) + return Number.isSafeInteger(index) && index >= 0 && index < length +} + +function getTypedArrayConstructor( + value: object, +): TypedArrayConstructor | undefined { + let prototype: object | null + try { + prototype = Object.getPrototypeOf(value) + } catch { + return undefined + } + return typedArrayConstructors.find( + (constructor) => prototype === constructor.prototype, + ) +} + +function requireArray(value: unknown, path: string): Array { + if (!Array.isArray(value)) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + return value +} + +function requireObject(value: unknown, path: string): object { + if (value === null || typeof value !== `object`) { + throw new RemoteSubsetWireValueError(path, describe(value)) + } + return value +} + +function requirePlainRecord( + value: unknown, + path: string, +): Record { + const object = requireObject(value, path) + assertExactPrototype(object, Object.prototype, path) + return object as Record +} + +function ownKeys(value: object, path: string): Array { + try { + return Reflect.ownKeys(value) + } catch { + throw new RemoteSubsetWireValueError(path, `unreadable properties`) + } +} + +function propertyPath(path: string, key: string | symbol): string { + if (typeof key === `symbol`) return `${path}[${String(key)}]` + return identifierPattern.test(key) + ? `${path}.${key}` + : `${path}[${JSON.stringify(key)}]` +} + +function describe(value: unknown): string { + if (value === null) return `null` + if (typeof value !== `object`) return typeof value + return `object` +} diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 69f29fc604..5cd5305911 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1884,10 +1884,15 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } } - private async ensureCollectionReadyInternal( - collectionId: string, - ): Promise { - const existingRows = await this.driver.query<{ + private async loadCollectionRegistration(collectionId: string): Promise< + | { + table_name: string + tombstone_table_name: string + schema_version: number + } + | undefined + > { + const rows = await this.driver.query<{ table_name: string tombstone_table_name: string schema_version: number @@ -1899,25 +1904,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { [collectionId], ) - let tableName: string - let tombstoneTableName: string + return rows[0] + } - if (existingRows.length > 0) { - tableName = existingRows[0]!.table_name - tombstoneTableName = existingRows[0]!.tombstone_table_name + private async ensureCollectionReadyInternal( + collectionId: string, + ): Promise { + let registration = await this.loadCollectionRegistration(collectionId) - if (existingRows[0]!.schema_version !== this.schemaVersion) { - await this.handleSchemaMismatch( - collectionId, - existingRows[0]!.schema_version, - this.schemaVersion, - tableName, - tombstoneTableName, - ) - } - } else { - tableName = createPersistedTableName(collectionId, `c`) - tombstoneTableName = createPersistedTableName(collectionId, `t`) + if (!registration) { + const tableName = createPersistedTableName(collectionId, `c`) + const tombstoneTableName = createPersistedTableName(collectionId, `t`) await this.driver.run( `INSERT INTO collection_registry ( collection_id, @@ -1926,9 +1923,29 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { schema_version, updated_at ) - VALUES (?, ?, ?, ?, CAST(strftime('%s', 'now') AS INTEGER))`, + VALUES (?, ?, ?, ?, CAST(strftime('%s', 'now') AS INTEGER)) + ON CONFLICT DO NOTHING`, [collectionId, tableName, tombstoneTableName, this.schemaVersion], ) + + registration = await this.loadCollectionRegistration(collectionId) + if (!registration) { + throw new InvalidPersistedCollectionConfigError( + `Unable to register persistence tables for collection "${collectionId}"`, + ) + } + } + + const tableName = registration.table_name + const tombstoneTableName = registration.tombstone_table_name + if (registration.schema_version !== this.schemaVersion) { + await this.handleSchemaMismatch( + collectionId, + registration.schema_version, + this.schemaVersion, + tableName, + tombstoneTableName, + ) } const collectionTableSql = quoteIdentifier(tableName) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts index b07f9f787d..1305fd9274 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test-d.ts @@ -1,7 +1,22 @@ import { describe, expectTypeOf, it } from 'vitest' import { createCollection } from '@tanstack/db' -import { persistedCollectionOptions } from '../src' -import type { PersistedCollectionUtils, PersistenceAdapter } from '../src' +import { + IndeterminateCommitError, + PersistedCollectionDurabilityError, + persistedCollectionOptions, +} from '../src' +import type { + ApplyCommittedTxResponse, + ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + IndeterminateCommitRequestType, + PersistedCollectionCoordinator, + PersistedCollectionUtils, + PersistenceAdapter, + RemoteSubsetWireValue, + TransportedLoadSubsetOptions, +} from '../src' import type { SyncConfig, UtilsRecord } from '@tanstack/db' type Todo = { @@ -24,6 +39,212 @@ const adapter: PersistenceAdapter = { } describe(`persisted collection types`, () => { + it(`exports the indeterminate commit reconciliation contract`, () => { + const requestType: IndeterminateCommitRequestType = `rpc:applyLocalMutations:req` + const error = new IndeterminateCommitError({ + collectionId: `todos`, + requestType, + previousLeaderId: `leader-a`, + previousTerm: 1, + currentLeaderId: null, + currentTerm: null, + cause: new Error(`channel closed`), + }) + + expectTypeOf(error.code).toEqualTypeOf<`INDETERMINATE_COMMIT`>() + expectTypeOf( + error.requestType, + ).toEqualTypeOf() + expectTypeOf(error.previousLeaderId).toEqualTypeOf() + expectTypeOf(error.previousTerm).toEqualTypeOf() + expectTypeOf(error.currentLeaderId).toEqualTypeOf() + expectTypeOf(error.currentTerm).toEqualTypeOf() + expectTypeOf(error.cause).toEqualTypeOf() + }) + + it(`exports the durability error and persistence response details`, () => { + const cause = Object.assign(new Error(`disk failed`), { + code: `SQLITE_FULL`, + path: `todos.sqlite`, + }) + const error = new PersistedCollectionDurabilityError(`durability failed`, { + cause, + code: cause.code, + path: cause.path, + }) + const response: Extract< + ApplyCommittedTxResponse, + { ok: false; code: `PERSISTENCE_ERROR` } + > = { + type: `rpc:applyCommittedTx:res`, + rpcId: `durability`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: `disk failed`, + sourceCode: `SQLITE_FULL`, + path: [`database`, `todos.sqlite`], + } + const localMutationResponse: Extract< + ApplyLocalMutationsResponse, + { ok: false; code: `PERSISTENCE_ERROR` } + > = { + type: `rpc:applyLocalMutations:res`, + rpcId: `local-durability`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: `disk failed`, + sourceCode: `SQLITE_FULL`, + path: [`database`, `todos.sqlite`], + } + + expectTypeOf(error.cause).toEqualTypeOf() + expectTypeOf(error.code).toEqualTypeOf() + expectTypeOf(error.path).toEqualTypeOf() + expectTypeOf(response.sourceCode).toEqualTypeOf< + string | number | undefined + >() + expectTypeOf(response.path).toEqualTypeOf< + string | ReadonlyArray | undefined + >() + expectTypeOf(localMutationResponse.sourceCode).toEqualTypeOf< + string | number | undefined + >() + expectTypeOf(localMutationResponse.path).toEqualTypeOf< + string | ReadonlyArray | undefined + >() + }) + + it(`exports the exact remote-subset transport domain`, () => { + type WireRecord = { [key: string]: RemoteSubsetWireValue } + const cyclic: WireRecord = {} + cyclic.self = cyclic + const supported: TransportedLoadSubsetOptions = { + where: { + type: `func`, + name: `eq`, + args: [ + { type: `ref`, path: [`todos`, `payload`] }, + { + type: `val`, + value: { + missing: undefined, + precise: 9_007_199_254_740_993n, + date: new Date(Number.NaN), + regexp: /wire/giu, + buffer: new ArrayBuffer(4), + view: new DataView(new ArrayBuffer(4)), + typed: new Uint8Array([1, 2]), + map: new Map([ + [{ key: `object` }, cyclic], + ]), + set: new Set([cyclic]), + }, + }, + ], + }, + orderBy: [ + { + expression: { type: `ref`, path: [`todos`, `title`] }, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }, + }, + ], + cursor: { + whereFrom: { type: `val`, value: `from` }, + whereCurrent: { type: `val`, value: `current` }, + lastKey: `key`, + }, + limit: 10, + offset: 2, + } + const request: EnsureRemoteSubsetRequest = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `wire`, + acquisitionId: `wire-acquisition`, + options: supported, + } + + // @ts-expect-error every transported acquisition has a stable identity + const requestWithoutAcquisition: EnsureRemoteSubsetRequest = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `wire-without-acquisition`, + options: supported, + } + + expectTypeOf(request.options).toEqualTypeOf() + expectTypeOf(request.acquisitionId).toEqualTypeOf() + expectTypeOf< + Extract + >().toHaveProperty(`leaderId`) + expectTypeOf( + requestWithoutAcquisition, + ).toEqualTypeOf() + type RegisteredOwner = Parameters< + NonNullable + >[1] + expectTypeOf() + .parameter(0) + .toEqualTypeOf() + + // @ts-expect-error functions are not remote-subset wire values + const functionValue: RemoteSubsetWireValue = () => {} + // @ts-expect-error symbols are not remote-subset wire values + const symbolValue: RemoteSubsetWireValue = Symbol(`unsupported`) + // @ts-expect-error promises are not remote-subset wire values + const promiseValue: RemoteSubsetWireValue = Promise.resolve() + // @ts-expect-error weak collections are not remote-subset wire values + const weakValue: RemoteSubsetWireValue = new WeakMap() + // @ts-expect-error custom prototypes are not remote-subset wire values + const customValue: RemoteSubsetWireValue = new (class Custom { + value = `custom` + })() + // @ts-expect-error shared buffers are not remote-subset wire values + const sharedValue: RemoteSubsetWireValue = new SharedArrayBuffer(4) + const signalOptions: TransportedLoadSubsetOptions = { + // @ts-expect-error live signals never cross the coordinator wire + signal: new AbortController().signal, + } + const subscriptionOptions: TransportedLoadSubsetOptions = { + // @ts-expect-error live subscriptions never cross the coordinator wire + subscription: {}, + } + + expectTypeOf(functionValue).toEqualTypeOf() + expectTypeOf(symbolValue).toEqualTypeOf() + expectTypeOf(promiseValue).toEqualTypeOf() + expectTypeOf(weakValue).toEqualTypeOf() + expectTypeOf(customValue).toEqualTypeOf() + expectTypeOf(sharedValue).toEqualTypeOf() + expectTypeOf(signalOptions).toEqualTypeOf() + expectTypeOf( + subscriptionOptions, + ).toEqualTypeOf() + }) + + it(`requires complete committed-transaction routing from coordinators`, () => { + // @ts-expect-error requestApplyCommittedTx is required for every coordinator + const incompleteCoordinator: PersistedCollectionCoordinator = { + getNodeId: () => `incomplete`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => true, + ensureLeadership: () => Promise.resolve(), + requestEnsureRemoteSubset: () => Promise.resolve(), + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestEnsurePersistedIndex: () => Promise.resolve(), + } + + expectTypeOf( + incompleteCoordinator, + ).toMatchTypeOf() + }) + it(`adds persisted utils in sync-absent mode`, () => { const options = persistedCollectionOptions< Todo, diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0895e52375..0862daa2ef 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -8,6 +8,7 @@ import { createTransaction, } from '@tanstack/db' import { + IndeterminateCommitError, InvalidPersistedCollectionCoordinatorError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, @@ -25,9 +26,10 @@ import type { PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, + RemoteSubsetOwner, TxCommitted, } from '../src' -import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { Collection, LoadSubsetOptions, SyncConfig } from '@tanstack/db' type Todo = { id: string @@ -218,6 +220,18 @@ function createCoordinatorHarness(): CoordinatorHarness { ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, requestEnsureRemoteSubset: async () => {}, + // Remote subset ownership is outside this protocol-delivery harness. + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), pullSince: () => { harness.pullSinceCalls++ return Promise.resolve(pullSinceResponse) @@ -259,6 +273,34 @@ async function flushAsyncWork(delayMs: number = 0): Promise { } describe(`persistedCollectionOptions`, () => { + it(`preserves exact reconciliation context for an indeterminate commit`, () => { + const cause = new Error(`response channel closed`) + const error = new IndeterminateCommitError({ + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: `leader-a`, + previousTerm: 4, + currentLeaderId: `leader-b`, + currentTerm: 5, + cause, + }) + + expect(error).toMatchObject({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: `leader-a`, + previousTerm: 4, + currentLeaderId: `leader-b`, + currentTerm: 5, + cause, + }) + expect(error.message).toContain( + `rpc:applyCommittedTx:req crossed leadership from leader-a (term 4) to leader-b (term 5)`, + ) + }) + it(`provides a sync-absent loopback configuration with persisted utils`, async () => { const adapter = createRecordingAdapter() const collection = createCollection( @@ -701,6 +743,326 @@ describe(`persistedCollectionOptions`, () => { ) }) + it(`coalesces duplicate single-process subset acknowledgements until owner work finishes`, async () => { + const coordinator = new SingleProcessCoordinator(`single-coalesced`) + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + const owner = Object.assign( + vi.fn(() => loadGate), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await Promise.resolve() + const duplicate = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + duplicateSettled = true + }) + await Promise.resolve() + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicate]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`coalesces same-stack single-process subset reentry until owner work finishes`, async () => { + const coordinator = new SingleProcessCoordinator(`single-reentrant`) + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + const options: LoadSubsetOptions = { limit: 1 } + let duplicate: Promise | undefined + let didReenter = false + const owner = Object.assign( + vi.fn(() => { + if (!didReenter) { + didReenter = true + duplicate = coordinator.requestEnsureRemoteSubset(`todos`, options) + } + return loadGate + }), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await Promise.resolve() + const duplicateResult = duplicate!.then(() => { + duplicateSettled = true + }) + await Promise.resolve() + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicateResult]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`waits for a pending single-process subset load before unloading it`, async () => { + const coordinator = new SingleProcessCoordinator(`single-release-order`) + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + const events: Array = [] + const owner = Object.assign( + vi.fn(async () => { + events.push(`load-start`) + await loadGate + events.push(`load-finish`) + }), + { + unloadSubset: vi.fn(() => { + events.push(`unload`) + }), + onError: vi.fn(), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + const load = coordinator.requestEnsureRemoteSubset(`todos`, options) + await Promise.resolve() + let releaseSettled = false + const release = coordinator + .requestReleaseRemoteSubset(`todos`, options) + .then(() => { + releaseSettled = true + }) + await Promise.resolve() + + const beforeLoadRelease = { events: [...events], releaseSettled } + expect(beforeLoadRelease).toEqual({ + events: [`load-start`], + releaseSettled: false, + }) + + releaseLoad() + await Promise.all([load, release]) + expect(events).toEqual([`load-start`, `load-finish`, `unload`]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`rejects unsupported single-process subset values before owner work`, async () => { + const coordinator = new SingleProcessCoordinator(`single-wire-domain`) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`in`, [ + new IR.PropRef([`todos`, `status`]), + new IR.Value([`kept`, () => {}]), + ]), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + } + }) + + it(`rejects a sparse function argument at its exact expression path`, async () => { + const coordinator = new SingleProcessCoordinator(`single-wire-hole`) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const args = [new IR.PropRef([`todos`, `status`])] + args.length = 2 + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`eq`, args), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + } + }) + + it(`reports and rethrows a single-process owner unload rejection`, async () => { + const coordinator = new SingleProcessCoordinator(`single-unload-error`) + const unloadError = new Error(`single-process owner unload failed`) + const ownerErrors: Array = [] + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(() => + Promise.reject(unloadError), + ) as unknown as RemoteSubsetOwner[`unloadSubset`], + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await expect( + coordinator.requestReleaseRemoteSubset(`todos`, options), + ).rejects.toBe(unloadError) + expect(ownerErrors).toEqual([unloadError]) + } finally { + unregisterOwner() + } + }) + + it(`reports one fail-stop lifecycle error when a single-process async unload rejects`, async () => { + const unloadError = new Error(`remote owner unload failed`) + let unloadThenCalls = 0 + const rejectingThenable = { + then: ( + _resolve: (value?: void) => void, + reject: (error: unknown) => void, + ) => { + unloadThenCalls++ + queueMicrotask(() => reject(unloadError)) + }, + } + const sourceUnloadMock = vi.fn((options: LoadSubsetOptions) => + options.limit === 1 ? rejectingThenable : undefined, + ) + const sourceUnload = sourceUnloadMock as unknown as ( + options: LoadSubsetOptions, + ) => void + const collection = createCollection( + persistedCollectionOptions({ + id: `owner-unload-error`, + getKey: (todo) => todo.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: sourceUnload, + } + }, + }, + persistence: { + adapter: createRecordingAdapter(), + }, + }), + ) + const failing: LoadSubsetOptions = { limit: 1 } + const sibling: LoadSubsetOptions = { limit: 2 } + const markError = vi.spyOn(collection._lifecycle, `markError`) + const unhandled: Array = [] + const captureUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, captureUnhandled) + + try { + collection.startSyncImmediate() + await collection._sync.loadSubset(failing) + await collection._sync.loadSubset(sibling) + collection._sync.unloadSubset(failing) + collection._sync.unloadSubset(sibling) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + status: collection.status, + error: collection._lifecycle.getSyncError(), + unloadThenCalls, + unloadOptions: sourceUnloadMock.mock.calls.map(([options]) => options), + lifecycleErrors: markError.mock.calls.map(([error]) => error), + unhandled, + }).toEqual({ + status: `error`, + error: unloadError, + unloadThenCalls: 1, + unloadOptions: [failing, sibling], + lifecycleErrors: [unloadError], + unhandled: [], + }) + + await flushAsyncWork() + expect(sourceUnloadMock).toHaveBeenCalledTimes(2) + expect(markError).toHaveBeenCalledTimes(1) + } finally { + process.off(`unhandledRejection`, captureUnhandled) + await collection.cleanup() + } + }) + it(`resolves persistence per collection and forwards schemaVersion`, () => { const baseAdapter = createNoopAdapter() const syncAdapter = createNoopAdapter() @@ -778,6 +1140,114 @@ describe(`persistedCollectionOptions`, () => { ).toThrow(InvalidPersistedCollectionCoordinatorError) }) + it(`rejects an incomplete coordinator before starting a rich sync source`, async () => { + type SourceParams = Parameters[`sync`]>[0] + + const adapter = createRecordingAdapter() + const applyCommittedTx = adapter.applyCommittedTx + let adapterApplies = 0 + adapter.applyCommittedTx = (...args) => { + adapterApplies++ + return applyCommittedTx(...args) + } + let sourceCalls = 0 + let legacyCalls = 0 + let sourceParams: SourceParams | undefined + let collectionConstructed = false + let visible = false + let configurationError: { name: string; message: string } | null = null + let collection: Collection | undefined + + const incompleteCoordinator = { + getNodeId: () => `incomplete-rich-sync`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => false, + ensureLeadership: () => Promise.resolve(), + requestEnsurePersistedIndex: () => Promise.resolve(), + // Remote subset routing is complete so this fixture isolates the + // required committed-transaction method named by the assertion. + requestEnsureRemoteSubset: () => Promise.resolve(), + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestApplyLocalMutations: () => { + legacyCalls++ + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: `legacy`, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [], + }) + }, + } as unknown as PersistedCollectionCoordinator + + try { + try { + const options = persistedCollectionOptions({ + id: `incomplete-rich-sync`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceCalls++ + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator: incompleteCoordinator }, + }) + collection = createCollection(options) + collectionConstructed = true + } catch (error) { + const resolvedError = + error instanceof Error ? error : new Error(String(error)) + configurationError = { + name: resolvedError.name, + message: resolvedError.message, + } + } + + if (collection) { + collection.startSyncImmediate() + await vi.waitFor(() => expect(sourceParams).toBeDefined()) + sourceParams!.begin() + sourceParams!.metadata?.collection.set(`resume`, { offset: 7 }) + sourceParams!.truncate() + sourceParams!.write({ + type: `insert`, + value: { id: `rich`, title: `Visible before rejection` }, + metadata: { source: `remote` }, + }) + await Promise.resolve(sourceParams!.commit()).catch(() => undefined) + visible = collection.has(`rich`) + } + } finally { + await collection?.cleanup() + } + + expect({ + configurationError, + sourceCalls, + collectionConstructed, + visible, + adapterApplies, + legacyCalls, + }).toEqual({ + configurationError: { + name: `InvalidPersistedCollectionCoordinatorError`, + message: + 'Invalid persisted collection coordinator: missing required "requestApplyCommittedTx" method', + }, + sourceCalls: 0, + collectionConstructed: false, + visible: false, + adapterApplies: 0, + legacyCalls: 0, + }) + }) + it(`preserves valid sync config in sync-present mode`, async () => { const adapter = createRecordingAdapter() const sync: SyncConfig = { @@ -943,10 +1413,17 @@ describe(`persistedCollectionOptions`, () => { } }) - it(`rejects a wrapped sync receipt when persistence fails`, async () => { + it(`publishes before persistence settles, then fails with the named durability error`, async () => { const adapter = createRecordingAdapter() - const persistenceError = new Error(`persistence failed`) - adapter.applyCommittedTx = () => Promise.reject(persistenceError) + const persistenceError = Object.assign(new Error(`persistence failed`), { + code: `SQLITE_IOERR_WRITE`, + path: `todos.sqlite-wal`, + }) + let rejectPersistence: ((error: unknown) => void) | undefined + adapter.applyCommittedTx = () => + new Promise((_resolve, reject) => { + rejectPersistence = reject + }) let remoteBegin: (() => void) | undefined let remoteWrite: | ((message: { type: `insert`; value: Todo }) => void) @@ -978,15 +1455,209 @@ describe(`persistedCollectionOptions`, () => { type: `insert`, value: { id: `failed`, title: `Not durable` }, }) + const receipt = Promise.resolve(remoteCommit?.()) + let settled = false + void receipt.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) - await expect(Promise.resolve(remoteCommit?.())).rejects.toBe( - persistenceError, + expect(stripVirtualProps(collection.get(`failed`))).toEqual({ + id: `failed`, + title: `Not durable`, + }) + expect(settled).toBe(false) + await vi.waitFor(() => expect(rejectPersistence).toBeTypeOf(`function`)) + + rejectPersistence!(persistenceError) + const rejection = await receipt.then( + () => undefined, + (error: unknown) => error, + ) + + expect(rejection).not.toBe(persistenceError) + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR_WRITE`, + path: `todos.sqlite-wal`, + cause: persistenceError, + }) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) + } finally { + rejectPersistence?.(persistenceError) + await collection.cleanup() + } + }) + + it(`preserves a coordinator persistence classification without converting it to conflict`, async () => { + const coordinator = createCoordinatorHarness() + const persistenceResponse = { + type: `rpc:applyCommittedTx:res` as const, + rpcId: `persistence-response`, + ok: false as const, + code: `PERSISTENCE_ERROR` as const, + error: `disk write failed`, + sourceCode: `SQLITE_IOERR_FSYNC`, + path: [`database`, `wal`] as const, + } + coordinator.requestApplyCommittedTx = () => + Promise.resolve(persistenceResponse) + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `coordinator-persistence-error`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter: createRecordingAdapter(), coordinator }, + }), + ) + + try { + await collection.stateWhenReady() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `remote-failure`, title: `Published once` }, + }) + const rejection = await Promise.resolve(remoteCommit?.()).then( + () => undefined, + (error: unknown) => error, ) + + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR_FSYNC`, + path: [`database`, `wal`], + cause: persistenceResponse, + }) + expect(String(rejection)).not.toContain(`CONFLICT`) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) } finally { await collection.cleanup() } }) + it(`replays a hydration-buffered write before its named durability rejection`, async () => { + const adapter = createRecordingAdapter() + let releaseHydration: (() => void) | undefined + adapter.loadSubset = () => + new Promise>((resolve) => { + releaseHydration = () => resolve([]) + }) + const persistenceError = Object.assign( + new Error(`buffered persistence failed`), + { + code: `SQLITE_FULL`, + path: `todos.sqlite`, + }, + ) + let rejectPersistence: ((error: unknown) => void) | undefined + adapter.applyCommittedTx = () => + new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const collection = createCollection( + persistedCollectionOptions({ + id: `hydration-buffered-persistence-error`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + + try { + void collection.stateWhenReady().catch(() => undefined) + await vi.waitFor(() => { + expect(releaseHydration).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + }) + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `buffered`, title: `Published before failure` }, + }) + const receipt = Promise.resolve(remoteCommit?.()) + let settled = false + void receipt.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + + expect(collection.get(`buffered`)).toBeUndefined() + expect(settled).toBe(false) + releaseHydration!() + await vi.waitFor(() => expect(rejectPersistence).toBeTypeOf(`function`)) + + expect(stripVirtualProps(collection.get(`buffered`))).toEqual({ + id: `buffered`, + title: `Published before failure`, + }) + expect(settled).toBe(false) + + rejectPersistence!(persistenceError) + const rejection = await receipt.then( + () => undefined, + (error: unknown) => error, + ) + + expect(rejection).not.toBe(persistenceError) + expect(rejection).toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_FULL`, + path: `todos.sqlite`, + cause: persistenceError, + }) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(rejection) + } finally { + releaseHydration?.() + rejectPersistence?.(persistenceError) + await collection.cleanup() + warning.mockRestore() + } + }) + it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { const adapter = createRecordingAdapter() const ownership = { queryCollection: { owners: [`gc:q1`] } } @@ -2181,6 +2852,107 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`hydrates a multiprocess follower locally while only the elected owner acquires remote demand`, async () => { + const adapter = createRecordingAdapter([ + { id: `persisted`, title: `Persisted follower row` }, + ]) + let releaseRemoteLoad = (): void => {} + const remoteLoadGate = new Promise((resolve) => { + releaseRemoteLoad = resolve + }) + let registeredOwner: RemoteSubsetOwner | undefined + const ensureRemote = vi.fn(() => remoteLoadGate) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `follower-node`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => false, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + requestEnsureRemoteSubset: ensureRemote, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: (_collectionId, owner) => { + registeredOwner = owner + return () => { + if (registeredOwner === owner) registeredOwner = undefined + } + }, + } + const followerUpstreamLoad = vi.fn(() => Promise.resolve()) + const collection = createCollection( + persistedCollectionOptions({ + id: `multiprocess-follower-subset`, + getKey: (todo) => todo.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: followerUpstreamLoad, + unloadSubset: vi.fn(), + } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const options: LoadSubsetOptions = { limit: 1 } + + try { + collection.startSyncImmediate() + let settled = false + const load = Promise.resolve(collection._sync.loadSubset(options)).then( + () => { + settled = true + }, + ) + + await vi.waitFor(() => { + expect(adapter.loadSubsetCalls.length).toBeGreaterThan(0) + expect(ensureRemote).toHaveBeenCalledWith( + `multiprocess-follower-subset`, + options, + ) + }) + const persistedRow = collection.get(`persisted`) + expect({ + persistedRow: persistedRow + ? { id: persistedRow.id, title: persistedRow.title } + : undefined, + followerUpstreamCalls: followerUpstreamLoad.mock.calls.length, + registeredOwner: registeredOwner !== undefined, + settled, + }).toEqual({ + persistedRow: { id: `persisted`, title: `Persisted follower row` }, + followerUpstreamCalls: 0, + registeredOwner: true, + settled: false, + }) + + releaseRemoteLoad() + await load + expect({ + routedEnsures: ensureRemote.mock.calls.length, + followerUpstreamCalls: followerUpstreamLoad.mock.calls.length, + }).toEqual({ + routedEnsures: 1, + followerUpstreamCalls: 0, + }) + } finally { + releaseRemoteLoad() + await collection.cleanup() + } + }) + it.each([`abort`, `release`, `offline`] as const)( `handles remote ensure after %s without resurrecting cancelled demand`, async (action) => { @@ -2190,7 +2962,10 @@ describe(`persistedCollectionOptions`, () => { name: action === `abort` ? `AbortError` : `Error`, }) const ensure = vi.fn(async () => { - throw new Error(`offline`) + throw failure + }) + const followerUpstreamLoad = vi.fn(async () => { + throw new Error(`follower-local source must not own remote demand`) }) const coordinator: PersistedCollectionCoordinator = { getNodeId: () => `cancel-ensure`, @@ -2199,7 +2974,19 @@ describe(`persistedCollectionOptions`, () => { isLeader: () => true, ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), requestEnsureRemoteSubset: ensure, + // This fixture isolates ensure retry/cancellation, not ownership. + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, } const collection = createCollection( persistedCollectionOptions({ @@ -2210,9 +2997,7 @@ describe(`persistedCollectionOptions`, () => { sync: ({ markReady }) => { markReady() return { - loadSubset: async () => { - throw failure - }, + loadSubset: followerUpstreamLoad, } }, }, @@ -2231,11 +3016,11 @@ describe(`persistedCollectionOptions`, () => { if (action === `release`) collection._sync.unloadSubset(options) const callsBeforeRetry = ensure.mock.calls.length await vi.advanceTimersByTimeAsync(200) + expect(result).toBe(failure) + expect(followerUpstreamLoad).not.toHaveBeenCalled() if (action === `offline`) { - expect(result).toBe(failure) expect(ensure.mock.calls.length).toBeGreaterThan(callsBeforeRetry) } else { - if (action === `abort`) expect(result).toBe(failure) expect(ensure).toHaveBeenCalledTimes(callsBeforeRetry) } } finally { @@ -2249,6 +3034,7 @@ describe(`persistedCollectionOptions`, () => { it(`retries queued remote subset ensure after transient failures`, async () => { const adapter = createRecordingAdapter() let ensureCalls = 0 + const offline = new Error(`offline`) const coordinator: PersistedCollectionCoordinator = { getNodeId: () => `retry-node`, @@ -2257,12 +3043,24 @@ describe(`persistedCollectionOptions`, () => { isLeader: () => true, ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), requestEnsureRemoteSubset: async () => { ensureCalls++ if (ensureCalls === 1) { - throw new Error(`offline`) + throw offline } }, + // This fixture isolates ensure retry behavior, not ownership. + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, } const collection = createCollection( @@ -2286,9 +3084,15 @@ describe(`persistedCollectionOptions`, () => { collection.startSyncImmediate() await flushAsyncWork() - await (collection as any)._sync.loadSubset({ limit: 1 }) + const firstResult = await Promise.resolve( + (collection as any)._sync.loadSubset({ limit: 1 }), + ).then( + () => `ready`, + (error: unknown) => error, + ) await flushAsyncWork(120) + expect(firstResult).toBe(offline) expect(ensureCalls).toBeGreaterThanOrEqual(2) }) @@ -2301,6 +3105,19 @@ describe(`persistedCollectionOptions`, () => { isLeader: () => false, ensureLeadership: async () => {}, requestEnsurePersistedIndex: async () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + // This fixture isolates follower transaction acknowledgements. + requestEnsureRemoteSubset: async () => {}, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, requestApplyLocalMutations: async () => ({ type: `rpc:applyLocalMutations:res`, rpcId: `ack-1`, diff --git a/packages/db/skills/db-core/persistence/SKILL.md b/packages/db/skills/db-core/persistence/SKILL.md index 8a5f657af0..66330ffcc5 100644 --- a/packages/db/skills/db-core/persistence/SKILL.md +++ b/packages/db/skills/db-core/persistence/SKILL.md @@ -110,11 +110,24 @@ This works with any adapter: `electricCollectionOptions`, `queryCollectionOption Coordinators handle leader election and cross-instance communication so only one tab/process owns the database writer. -| Platform | Coordinator | Mechanism | -| ------------------------------------- | ------------------------------- | ---------------------------- | -| Browser | `BrowserCollectionCoordinator` | BroadcastChannel + Web Locks | -| Electron | `ElectronCollectionCoordinator` | BroadcastChannel + Web Locks | -| Single-process (RN, Expo, Node, etc.) | `SingleProcessCoordinator` | No-op (always leader) | +| Platform | Coordinator | Mechanism | +| ------------------------------------- | ------------------------------- | ---------------------------------------------------- | +| Browser | `BrowserCollectionCoordinator` | BroadcastChannel + Web Locks | +| Electron | `ElectronCollectionCoordinator` | BroadcastChannel + Web Locks | +| Single-process (RN, Expo, Node, etc.) | `SingleProcessCoordinator` | Always leader; direct per-collection adapter routing | + +Every coordinator must implement +`requestApplyCommittedTx(collectionId, tx)`. The method routes a complete +committed transaction to the collection's supported writer, including +truncate, rows, row metadata, collection metadata, and stream position. This +is a required contract, not an optional capability. Do not feature-detect it, +fall back to row-only mutation routing, or bypass the coordinator after source +publication. Untyped custom coordinators that omit the method fail during +collection configuration. + +`SingleProcessCoordinator` has no election or cross-process communication, but +it is not a no-op persistence owner. It registers the resolved adapter for each +collection and applies complete committed transactions through that adapter. Browser persistence uses single-process semantics by default. That is correct when the app runs in one tab at a time or each tab has its own database. Pass a @@ -163,7 +176,9 @@ const persistence = createElectronSQLitePersistence({ Electron persistence calls cross the renderer/main boundary through IPC. The `ElectronCollectionCoordinator` separately coordinates renderer instances with -`BroadcastChannel` and Web Locks. +`BroadcastChannel` and Web Locks. It elects an owner per collection and routes +the complete committed transaction through that collection's resolved renderer +adapter to the main-process persistence owner. ## Schema Versioning diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 00b3fc09b4..eed134f1f0 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -171,6 +171,7 @@ function fixture( } return { collection, + adapter, rows, metadata, commits, @@ -239,6 +240,7 @@ describe(`persisted Electric recovery laws`, () => { function externalPublisher() { let receive: ((message: ProtocolEnvelope) => void) | undefined + let ownerAdapter: PersistenceAdapter | undefined let id = `` let term = 100 const coordinator: PersistedCollectionCoordinator = { @@ -255,9 +257,29 @@ describe(`persisted Electric recovery laws`, () => { ensureLeadership: () => Promise.resolve(), requestEnsurePersistedIndex: () => Promise.resolve(), requestEnsureRemoteSubset: () => Promise.resolve(), + // This fixture owns external publication/recovery, not subset leases. + requestReleaseRemoteSubset: () => Promise.resolve(), + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: async (collectionId, tx) => { + if (!ownerAdapter) { + throw new Error(`external publisher has no persistence owner`) + } + await ownerAdapter.applyCommittedTx(collectionId, tx) + return { + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + }, } return { coordinator, + bindAdapter: (adapter: PersistenceAdapter) => { + ownerAdapter = adapter + }, publish: ( row: Item, deleted: boolean, @@ -300,6 +322,7 @@ describe(`persisted Electric recovery laws`, () => { async ({ syncMode, fullReload }) => { const peer = externalPublisher() const f = fixture(syncMode, peer.coordinator) + peer.bindAdapter(f.adapter) try { f.start() await vi.waitFor(() => expect(subscribers).toHaveLength(1)) @@ -369,6 +392,7 @@ describe(`persisted Electric recovery laws`, () => { subscribers.length = 0 const peer = externalPublisher() const f = fixture(`on-demand`, peer.coordinator) + peer.bindAdapter(f.adapter) const expected = new Map([[oldRow.id, structuredClone(oldRow)]]) const expectedRows = () => structuredClone([...expected.values()].sort((a, b) => a.id - b.id)) diff --git a/packages/electron-db-sqlite-persistence/README.md b/packages/electron-db-sqlite-persistence/README.md index 8f288dab3b..c53dd7493e 100644 --- a/packages/electron-db-sqlite-persistence/README.md +++ b/packages/electron-db-sqlite-persistence/README.md @@ -6,6 +6,7 @@ Thin Electron bridge for TanStack DB SQLite persistence. - `exposeElectronSQLitePersistence(...)` (main process) - `createElectronSQLitePersistence(...)` (renderer process) +- `ElectronCollectionCoordinator` - `persistedCollectionOptions(...)` (re-exported from core) Use `@tanstack/electron-db-sqlite-persistence/main` and @@ -64,8 +65,72 @@ export const todosCollection = createCollection( ) ``` +## Multi-renderer ownership + +Pass an `ElectronCollectionCoordinator` when multiple renderers share the same +database. Coordinators using the same `dbName` elect a writer independently for +each collection. + +```ts +import { + ElectronCollectionCoordinator, + createElectronSQLitePersistence, +} from '@tanstack/electron-db-sqlite-persistence' + +const coordinator = new ElectronCollectionCoordinator({ + dbName: `tanstack-db`, +}) + +const persistence = createElectronSQLitePersistence({ + ipcRenderer, + coordinator, +}) + +// On teardown: +// coordinator.dispose() +``` + +The persisted sync wrapper routes every source transaction with durable effects +through the coordinator's required +`requestApplyCommittedTx(collectionId, tx)` method. The elected renderer applies +the complete `PersistedTx` through its resolved per-collection renderer adapter, +which forwards it over IPC to the main-process persistence owner. Resolution +remains specific to the collection id, mode, and `schemaVersion`. + +Truncation, row changes, row metadata, collection metadata, and stream position +stay in the same transaction. Electron does not feature-detect a partial route +or fall back to row-only mutation RPC. A custom coordinator that omits +`requestApplyCommittedTx` is rejected while the collection is configured, +before its sync source can publish rows. + +If a mutating RPC loses its response, Electron coordination replays it only +while the requester still knows the same non-null leader id and term. An +unknown initial route or any leader/term change rejects with +`IndeterminateCommitError`; the application must reconcile the outcome. The +coordinator does not retry that mutation against an unknown or replacement +leader. + +Remote-subset requests use the same crash-only boundary. The coordinator +projects a live `LoadSubsetOptions` input to the exported +`TransportedLoadSubsetOptions` type before either local completion or renderer +transport. Live `signal` and `subscription` fields do not cross the boundary. +Unsupported nested values throw `RemoteSubsetWireValueError` with the exact +value path before publication; Electron does not add codecs, coercion, or a +fallback representation. + +Electron uses the same explicit lease lifecycle as Browser coordination. The +same request object reuses one acquisition identity, distinct equal objects +remain independent, and release unloads the exact transported options once. +Leadership loss unloads the retiring renderer's leases; still-live requester +leases replay against the next leader. A second owner registration for one +collection throws `DuplicateRemoteSubsetOwnerError`, and there is no adapter +fallback for a missing owner. + ## Notes - The renderer API mirrors other runtimes: one shared `create...Persistence`. - Collection mode (`sync-present` vs `sync-absent`) and `schemaVersion` are resolved per collection and forwarded across IPC automatically. +- Without an `ElectronCollectionCoordinator`, single-renderer mode uses + `SingleProcessCoordinator`: it is always leader, but still routes complete + transactions to the collection's resolved IPC adapter. diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index a4c6bb7fe8..1c3ea73f0a 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -1,12 +1,31 @@ -import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' +import { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + safeRandomUUID, + toPersistedCollectionDurabilityError, + toTransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' import type { + ApplyCommittedTxRequest, + ApplyCommittedTxResponse, ApplyLocalMutationsResponse, + EnsureRemoteSubsetRequest, + EnsureRemoteSubsetResponse, + IndeterminateCommitRequestType, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, + PersistedRowMetadataMutation, + PersistedTx, PersistenceAdapter, ProtocolEnvelope, PullSinceResponse, + ReleaseRemoteSubsetRequest, + ReleaseRemoteSubsetResponse, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, + TxCommitted, } from '@tanstack/db-sqlite-persistence-core' import type { LoadSubsetOptions } from '@tanstack/db' @@ -26,11 +45,8 @@ const WRITER_LOCK_MAX_RETRIES = 20 // --------------------------------------------------------------------------- type RPCRequest = - | { - type: `rpc:ensureRemoteSubset:req` - rpcId: string - options: LoadSubsetOptions - } + | EnsureRemoteSubsetRequest + | ReleaseRemoteSubsetRequest | { type: `rpc:ensurePersistedIndex:req` rpcId: string @@ -43,6 +59,7 @@ type RPCRequest = envelopeId: string mutations: Array } + | ApplyCommittedTxRequest | { type: `rpc:pullSince:req` rpcId: string @@ -50,12 +67,8 @@ type RPCRequest = } type RPCResponse = - | { - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - } + | EnsureRemoteSubsetResponse + | ReleaseRemoteSubsetResponse | { type: `rpc:ensurePersistedIndex:res` rpcId: string @@ -63,6 +76,7 @@ type RPCResponse = error?: string } | ApplyLocalMutationsResponse + | ApplyCommittedTxResponse | PullSinceResponse type PendingRPC = { @@ -73,6 +87,7 @@ type PendingRPC = { type CollectionState = { isLeader: boolean + leaderId: string | null lockAbortController: AbortController | null heartbeatTimer: ReturnType | null latestTerm: number @@ -81,6 +96,47 @@ type CollectionState = { subscribers: Set<(message: ProtocolEnvelope) => void> } +type ActiveRemoteSubsetAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + owner: RemoteSubsetOwner + options: TransportedLoadSubsetOptions + load: Promise + transferred: boolean + released: boolean + terminalRelease: boolean + release: Promise | null +} + +type AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: string + requesterId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + released: true + awaitingOwner: true +} + +type RemoteSubsetAcquisition = + | ActiveRemoteSubsetAcquisition + | AwaitingRemoteSubsetOwnerAcquisition + | { + collectionId: string + requesterId: string + acquisitionId: string + released: true + } + +type OutboundRemoteSubsetAcquisition = { + collectionId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + inFlight: Promise | null + forceReplay: boolean +} + // Adapter with pullSince support type AdapterWithPullSince = PersistenceAdapter & { pullSince?: ( @@ -121,11 +177,40 @@ export type ElectronCollectionCoordinatorOptions = { export class ElectronCollectionCoordinator implements PersistedCollectionCoordinator { private readonly nodeId = safeRandomUUID() private readonly dbName: string - private adapter: AdapterWithPullSince | null + private defaultAdapter: AdapterWithPullSince | null + private readonly collectionAdapters = new Map() + private readonly remoteSubsetOwners = new Map() + private readonly remoteSubsetIds = new Map< + string, + WeakMap + >() + private readonly outboundRemoteSubsetAcquisitions = new Map< + string, + OutboundRemoteSubsetAcquisition + >() + private readonly inboundRemoteSubsetAcquisitions = new Map< + string, + RemoteSubsetAcquisition + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopeIds = new Map< + string, + { appliedAt: number; response: ApplyLocalMutationsResponse } + >() + private readonly inFlightLocalMutationEnvelopes = new Map< + string, + Promise + >() + private readonly appliedCommittedTxEnvelopes = new Map< + string, + { appliedAt: number; response: ApplyCommittedTxResponse } + >() + private readonly inFlightCommittedTxEnvelopes = new Map< + string, + Promise + >() private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -133,18 +218,20 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin return this.disposed } - private requireAdapter(): AdapterWithPullSince { - if (!this.adapter) { + private requireAdapter(collectionId: string): AdapterWithPullSince { + const adapter = + this.collectionAdapters.get(collectionId) ?? this.defaultAdapter + if (!adapter) { throw new Error( - `ElectronCollectionCoordinator: adapter not set. Call setAdapter() before using leader-side operations.`, + `ElectronCollectionCoordinator: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, ) } - return this.adapter + return adapter } constructor(options: ElectronCollectionCoordinatorOptions) { this.dbName = options.dbName - this.adapter = options.adapter ?? null + this.defaultAdapter = options.adapter ?? null this.channel = new BroadcastChannel(`tsdb:coord:${this.dbName}`) this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) @@ -157,7 +244,36 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin * adapter into the coordinator. */ setAdapter(adapter: AdapterWithPullSince): void { - this.adapter = adapter + this.defaultAdapter = adapter + } + + setAdapterForCollection( + collectionId: string, + adapter: AdapterWithPullSince, + ): void { + this.collectionAdapters.set(collectionId, adapter) + } + + registerRemoteSubsetOwner( + collectionId: string, + owner: RemoteSubsetOwner, + ): () => void { + if (this.remoteSubsetOwners.has(collectionId)) { + throw new DuplicateRemoteSubsetOwnerError(collectionId) + } + this.remoteSubsetOwners.set(collectionId, owner) + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if (acquisition.collectionId !== collectionId) continue + acquisition.acquiredLeaderId = null + acquisition.forceReplay = true + } + void this.replayRemoteSubsetAcquisitions(collectionId) + this.rebindRemoteInboundSubsetAcquisitions(collectionId, owner) + return () => { + if (this.remoteSubsetOwners.get(collectionId) !== owner) return + this.remoteSubsetOwners.delete(collectionId) + this.releaseInboundRemoteSubsetAcquisitions(collectionId, owner) + } } // ----------------------------------------------------------------------- @@ -197,23 +313,138 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin collectionId: string, options: LoadSubsetOptions, ): Promise { - if (this.isLeader(collectionId)) return + const transportedOptions = toTransportedLoadSubsetOptions(options) + let collectionIds = this.remoteSubsetIds.get(collectionId) + if (!collectionIds) { + collectionIds = new WeakMap() + this.remoteSubsetIds.set(collectionId, collectionIds) + } + let acquisitionId = collectionIds.get(options) + let acquisition = acquisitionId + ? this.outboundRemoteSubsetAcquisitions.get( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + ) + : undefined + if (!acquisition) { + acquisitionId = safeRandomUUID() + collectionIds.set(options, acquisitionId) + acquisition = { + collectionId, + acquisitionId, + options: transportedOptions, + acquiredLeaderId: null, + inFlight: null, + forceReplay: false, + } + this.outboundRemoteSubsetAcquisitions.set( + remoteSubsetAcquisitionKey(collectionId, acquisitionId), + acquisition, + ) + } - const response = await this.sendRPC<{ - type: `rpc:ensureRemoteSubset:res` - rpcId: string - ok: boolean - error?: string - }>(collectionId, { - type: `rpc:ensureRemoteSubset:req`, + await this.acquireRemoteSubset(acquisition) + } + + async requestReleaseRemoteSubset( + collectionId: string, + options: LoadSubsetOptions, + ): Promise { + const collectionIds = this.remoteSubsetIds.get(collectionId) + const acquisitionId = collectionIds?.get(options) + if (!acquisitionId) return + const key = remoteSubsetAcquisitionKey(collectionId, acquisitionId) + if (!this.outboundRemoteSubsetAcquisitions.delete(key)) return + collectionIds!.delete(options) + + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, rpcId: safeRandomUUID(), - options, - }) + acquisitionId, + } + const response = this.isLeader(collectionId) + ? await this.handleReleaseRemoteSubset(collectionId, request, this.nodeId) + : await this.sendRPC(collectionId, request) if (!response.ok) { - throw new Error( - `ensureRemoteSubset failed: ${response.error ?? `unknown error`}`, + throw new Error(`releaseRemoteSubset failed: ${response.error}`) + } + } + + private async acquireRemoteSubset( + acquisition: OutboundRemoteSubsetAcquisition, + ): Promise { + if (acquisition.inFlight) return acquisition.inFlight + + const routedToLocalOwner = this.isLeader(acquisition.collectionId) + let resolveWork!: () => void + let rejectWork!: (error: unknown) => void + const work = new Promise((resolve, reject) => { + resolveWork = resolve + rejectWork = reject + }) + acquisition.inFlight = work + const run = async (): Promise => { + const request: Extract< + RPCRequest, + { type: `rpc:ensureRemoteSubset:req` } + > = { + type: `rpc:ensureRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + } + const response = routedToLocalOwner + ? await this.handleEnsureRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ) + : await this.sendRPC( + acquisition.collectionId, + request, + ) + + if (!response.ok) { + throw new Error(`ensureRemoteSubset failed: ${response.error}`) + } + acquisition.acquiredLeaderId = response.leaderId + } + void run().then(resolveWork, rejectWork) + let acquired = false + try { + await work + acquired = true + } catch (error) { + if (!routedToLocalOwner) { + const owner = this.remoteSubsetOwners.get(acquisition.collectionId) + if (owner) reportRemoteSubsetOwnerError(owner, error) + } + throw error + } finally { + if (acquisition.inFlight === work) acquisition.inFlight = null + const current = this.collections.get(acquisition.collectionId) + const currentLeaderId = current?.isLeader + ? this.nodeId + : (current?.leaderId ?? null) + const key = remoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.acquisitionId, ) + if ( + acquired && + this.outboundRemoteSubsetAcquisitions.get(key) === acquisition && + (acquisition.forceReplay || + (currentLeaderId !== null && + acquisition.acquiredLeaderId !== currentLeaderId)) + ) { + acquisition.forceReplay = false + void this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }) + } } } @@ -223,7 +454,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin spec: PersistedIndexSpec, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter().ensureIndex(collectionId, signature, spec) + await this.requireAdapter(collectionId).ensureIndex( + collectionId, + signature, + spec, + ) return } @@ -267,6 +502,23 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }) } + async requestApplyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { + const request: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: safeRandomUUID(), + envelopeId: safeRandomUUID(), + tx, + } + if (this.isLeader(collectionId)) { + return this.handleApplyCommittedTx(collectionId, request) + } + + return this.sendRPC(collectionId, request) + } + async pullSince( collectionId: string, fromRowVersion: number, @@ -293,6 +545,12 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin dispose(): void { this.disposed = true + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + this.postRemoteSubsetRelease(acquisition) + } + this.outboundRemoteSubsetAcquisitions.clear() + this.remoteSubsetIds.clear() + for (const [collectionId, state] of this.collections) { this.releaseLeadership(collectionId, state) } @@ -305,6 +563,16 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.channel.close() this.collections.clear() + this.collectionAdapters.clear() + for (const collectionId of this.remoteSubsetOwners.keys()) { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) + } + this.remoteSubsetOwners.clear() + this.inboundRemoteSubsetAcquisitions.clear() + this.appliedEnvelopeIds.clear() + this.inFlightLocalMutationEnvelopes.clear() + this.appliedCommittedTxEnvelopes.clear() + this.inFlightCommittedTxEnvelopes.clear() } // ----------------------------------------------------------------------- @@ -316,6 +584,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin if (!state) { state = { isLeader: false, + leaderId: null, lockAbortController: null, heartbeatTimer: null, latestTerm: 0, @@ -348,7 +617,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (adapter.getStreamPosition) { const pos = await adapter.getStreamPosition(collectionId) state.latestTerm = pos.latestTerm @@ -358,8 +627,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.latestTerm++ state.isLeader = true + state.leaderId = this.nodeId this.emitHeartbeat(collectionId, state) + void this.replayRemoteSubsetAcquisitions(collectionId) state.heartbeatTimer = setInterval(() => { this.emitHeartbeat(collectionId, state) }, HEARTBEAT_INTERVAL_MS) @@ -377,7 +648,9 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin abortController.signal.addEventListener(`abort`, onAbort) }) } finally { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) state.isLeader = false + state.leaderId = null if (state.heartbeatTimer) { clearInterval(state.heartbeatTimer) state.heartbeatTimer = null @@ -399,9 +672,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } private releaseLeadership( - _collectionId: string, + collectionId: string, state: CollectionState, ): void { + this.releaseInboundRemoteSubsetAcquisitions(collectionId) if (state.lockAbortController) { state.lockAbortController.abort() state.lockAbortController = null @@ -411,6 +685,68 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin state.heartbeatTimer = null } state.isLeader = false + state.leaderId = null + } + + private postRemoteSubsetRelease( + acquisition: OutboundRemoteSubsetAcquisition, + ): void { + const request: Extract< + RPCRequest, + { type: `rpc:releaseRemoteSubset:req` } + > = { + type: `rpc:releaseRemoteSubset:req`, + rpcId: safeRandomUUID(), + acquisitionId: acquisition.acquisitionId, + } + if (this.isLeader(acquisition.collectionId)) { + void this.handleReleaseRemoteSubset( + acquisition.collectionId, + request, + this.nodeId, + ).catch(() => { + // The owner already received the exact unload failure through onError. + }) + return + } + this.channel.postMessage({ + v: 1, + dbName: this.dbName, + collectionId: acquisition.collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: request, + } satisfies ProtocolEnvelope) + } + + private async replayRemoteSubsetAcquisitions( + collectionId: string, + ): Promise { + if (this.isDisposed()) return + const state = this.collections.get(collectionId) + const leaderId = state?.isLeader ? this.nodeId : state?.leaderId + if (!leaderId) return + + const replays: Array> = [] + for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { + if ( + acquisition.collectionId !== collectionId || + (!acquisition.forceReplay && acquisition.acquiredLeaderId === leaderId) + ) { + continue + } + if (acquisition.inFlight) { + acquisition.forceReplay = true + continue + } + acquisition.forceReplay = false + replays.push( + this.acquireRemoteSubset(acquisition).catch(() => { + // Failure is already reported; only new demand or ownership change retries. + }), + ) + } + await Promise.all(replays) } private emitHeartbeat(collectionId: string, state: CollectionState): void { @@ -448,6 +784,35 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const type = (payload as Record).type as string | undefined + if (type === `leader:heartbeat`) { + const heartbeat = payload as { + leaderId?: unknown + term?: unknown + latestSeq?: unknown + latestRowVersion?: unknown + } + if ( + typeof heartbeat.leaderId === `string` && + typeof heartbeat.term === `number` && + typeof heartbeat.latestSeq === `number` && + typeof heartbeat.latestRowVersion === `number` + ) { + const state = this.ensureCollectionState(envelope.collectionId) + if (heartbeat.term < state.latestTerm) return + const changedLeader = state.leaderId !== heartbeat.leaderId + state.leaderId = heartbeat.leaderId + state.latestTerm = Math.max(state.latestTerm, heartbeat.term) + state.latestSeq = Math.max(state.latestSeq, heartbeat.latestSeq) + state.latestRowVersion = Math.max( + state.latestRowVersion, + heartbeat.latestRowVersion, + ) + if (changedLeader) { + void this.replayRemoteSubsetAcquisitions(envelope.collectionId) + } + } + } + // Handle RPC responses (for pending outbound RPCs) if (type && type.endsWith(`:res`)) { const rpcId = (payload as { rpcId?: string }).rpcId @@ -464,7 +829,11 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin if (type && type.endsWith(`:req`)) { const collectionId = envelope.collectionId if (this.isLeader(collectionId)) { - void this.handleRPCRequest(collectionId, payload as RPCRequest) + void this.handleRPCRequest( + collectionId, + payload as RPCRequest, + envelope.senderId, + ) } return } @@ -487,15 +856,53 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin request: RPCRequest, ): Promise { let lastError: Error | undefined + let firstTransportCause: unknown + const mutationRequestType = isMutatingRPCRequest(request) + ? request.type + : undefined + const mutationRoute = mutationRequestType + ? this.captureMutationRoute(collectionId) + : undefined for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { await sleep(RPC_RETRY_DELAY_MS * attempt) } + if ( + mutationRoute && + mutationRequestType && + firstTransportCause !== undefined + ) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } + + if (this.isLeader(collectionId)) { + return (await this.dispatchRPCRequest( + collectionId, + request, + this.nodeId, + )) as T + } + try { return await this.sendRPCOnce(collectionId, request) } catch (error) { + if (this.isDisposed()) throw error + firstTransportCause ??= error + if (mutationRoute && mutationRequestType) { + this.assertMutationRouteUnchanged( + collectionId, + mutationRequestType, + mutationRoute, + firstTransportCause, + ) + } lastError = error instanceof Error ? error : new Error(String(error)) } } @@ -503,6 +910,43 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin throw lastError ?? new Error(`RPC failed after retries`) } + private captureMutationRoute(collectionId: string): { + leaderId: string | null + term: number | null + } { + const state = this.collections.get(collectionId) + return { + leaderId: state?.isLeader ? this.nodeId : (state?.leaderId ?? null), + term: state?.latestTerm ?? null, + } + } + + private assertMutationRouteUnchanged( + collectionId: string, + requestType: IndeterminateCommitRequestType, + previous: { leaderId: string | null; term: number | null }, + cause: unknown, + ): void { + const current = this.captureMutationRoute(collectionId) + if ( + previous.leaderId !== null && + previous.term !== null && + current.leaderId === previous.leaderId && + current.term === previous.term + ) { + return + } + throw new IndeterminateCommitError({ + collectionId, + requestType, + previousLeaderId: previous.leaderId, + previousTerm: previous.term, + currentLeaderId: current.leaderId, + currentTerm: current.term, + cause, + }) + } + private sendRPCOnce( collectionId: string, request: RPCRequest, @@ -542,38 +986,22 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private async handleRPCRequest( collectionId: string, request: RPCRequest, + requesterId: string, ): Promise { let response: RPCResponse try { - switch (request.type) { - case `rpc:ensureRemoteSubset:req`: - response = await this.handleEnsureRemoteSubset(collectionId, request) - break - case `rpc:ensurePersistedIndex:req`: - response = await this.handleEnsurePersistedIndex( - collectionId, - request, - ) - break - case `rpc:applyLocalMutations:req`: - response = await this.handleApplyLocalMutations(collectionId, request) - break - case `rpc:pullSince:req`: - response = await this.handlePullSince(collectionId, request) - break - default: - return - } + response = await this.dispatchRPCRequest( + collectionId, + request, + requesterId, + ) } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error) - response = { - type: request.type.replace(`:req`, `:res`) as RPCResponse[`type`], - rpcId: request.rpcId, - ok: false, - error: errorMessage, - } as RPCResponse + response = createRPCErrorResponse(request, error) + } + + if (this.isDisposed()) { + return } const envelope: ProtocolEnvelope = { @@ -587,16 +1015,357 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin this.channel.postMessage(envelope) } - private handleEnsureRemoteSubset( - _collectionId: string, - request: { type: `rpc:ensureRemoteSubset:req`; rpcId: string }, - ): RPCResponse { - // Leader doesn't need to do anything special — the remote subset - // is ensured by the leader's own sync connection + private dispatchRPCRequest( + collectionId: string, + request: RPCRequest, + requesterId: string, + ): Promise { + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return this.handleEnsureRemoteSubset(collectionId, request, requesterId) + case `rpc:releaseRemoteSubset:req`: + return this.handleReleaseRemoteSubset( + collectionId, + request, + requesterId, + ) + case `rpc:ensurePersistedIndex:req`: + return this.handleEnsurePersistedIndex(collectionId, request) + case `rpc:applyLocalMutations:req`: + return this.handleApplyLocalMutations(collectionId, request) + case `rpc:applyCommittedTx:req`: + return this.handleApplyCommittedTx(collectionId, request) + case `rpc:pullSince:req`: + return this.handlePullSince(collectionId, request) + } + } + + private async handleEnsureRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const existing = this.inboundRemoteSubsetAcquisitions.get(key) + const awaitingOwner = + existing && `awaitingOwner` in existing ? existing : undefined + if (existing) { + if (`owner` in existing) { + if (!existing.released) { + await existing.load + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + await existing.release + if (existing.terminalRelease) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + if (this.inboundRemoteSubsetAcquisitions.get(key) === existing) { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } else if (!(`awaitingOwner` in existing)) { + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + leaderId: this.nodeId, + } + } + } + + const owner = this.remoteSubsetOwners.get(collectionId) + if (!owner) { + throw new Error( + `ElectronCollectionCoordinator: no remote subset owner registered for collection "${collectionId}"`, + ) + } + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + owner, + options: awaitingOwner?.options ?? request.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + if (awaitingOwner) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } else { + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + reportRemoteSubsetOwnerError(owner, error) + throw error + } return { type: `rpc:ensureRemoteSubset:res`, rpcId: request.rpcId, ok: true, + leaderId: this.nodeId, + } + } + + private async handleReleaseRemoteSubset( + collectionId: string, + request: Extract, + requesterId: string, + ): Promise { + const key = inboundRemoteSubsetAcquisitionKey( + collectionId, + requesterId, + request.acquisitionId, + ) + const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) + if (!acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } else if (`owner` in acquisition) { + acquisition.terminalRelease = true + await this.releaseRemoteSubsetAcquisition(acquisition) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + } else if (`awaitingOwner` in acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId, + requesterId, + acquisitionId: request.acquisitionId, + released: true, + }) + } + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: true, + } + } + + private releaseRemoteSubsetAcquisition( + acquisition: ActiveRemoteSubsetAcquisition, + ): Promise { + if (acquisition.release) return acquisition.release + acquisition.released = true + acquisition.release = (async () => { + try { + await acquisition.load + } catch { + // A returned promise transfers the lease even when initial loading fails. + } + try { + if (acquisition.transferred) { + await unloadRemoteSubsetOwner(acquisition.owner, acquisition.options) + } + } finally { + if (!acquisition.terminalRelease) { + const key = inboundRemoteSubsetAcquisitionKey( + acquisition.collectionId, + acquisition.requesterId, + acquisition.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { + this.inboundRemoteSubsetAcquisitions.set(key, { + collectionId: acquisition.collectionId, + requesterId: acquisition.requesterId, + acquisitionId: acquisition.acquisitionId, + options: acquisition.options, + released: true, + awaitingOwner: true, + }) + } + } + } + })() + return acquisition.release + } + + private releaseInboundRemoteSubsetAcquisitions( + collectionId: string, + owner?: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + (owner && acquisition.owner !== owner) + ) { + continue + } + void this.releaseRemoteSubsetAcquisition(acquisition).catch( + () => undefined, + ) + } + } + + private rebindRemoteInboundSubsetAcquisitions( + collectionId: string, + owner: RemoteSubsetOwner, + ): void { + for (const acquisition of this.inboundRemoteSubsetAcquisitions.values()) { + if (`awaitingOwner` in acquisition) { + if ( + acquisition.collectionId === collectionId && + acquisition.requesterId !== this.nodeId + ) { + void this.bindAwaitingRemoteSubsetAcquisition( + acquisition, + owner, + ).catch(() => { + // The owner receives the exact load failure through onError. + }) + } + continue + } + if ( + !(`owner` in acquisition) || + acquisition.collectionId !== collectionId || + acquisition.requesterId === this.nodeId || + !acquisition.released || + acquisition.terminalRelease + ) { + continue + } + + void this.rebindRemoteInboundSubsetAcquisition(acquisition, owner).catch( + () => undefined, + ) + } + } + + private async rebindRemoteInboundSubsetAcquisition( + previous: ActiveRemoteSubsetAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + await previous.release + if ( + previous.terminalRelease || + this.remoteSubsetOwners.get(previous.collectionId) !== owner + ) { + return + } + + const key = inboundRemoteSubsetAcquisitionKey( + previous.collectionId, + previous.requesterId, + previous.acquisitionId, + ) + const current = this.inboundRemoteSubsetAcquisitions.get(key) + if (current && current !== previous) { + if (`awaitingOwner` in current) { + await this.bindAwaitingRemoteSubsetAcquisition(current, owner) + } + return + } + + const awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition = { + collectionId: previous.collectionId, + requesterId: previous.requesterId, + acquisitionId: previous.acquisitionId, + options: previous.options, + released: true, + awaitingOwner: true, + } + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + await this.bindAwaitingRemoteSubsetAcquisition(awaitingOwner, owner) + } + + private async bindAwaitingRemoteSubsetAcquisition( + awaitingOwner: AwaitingRemoteSubsetOwnerAcquisition, + owner: RemoteSubsetOwner, + ): Promise { + if (this.remoteSubsetOwners.get(awaitingOwner.collectionId) !== owner) { + return + } + const key = inboundRemoteSubsetAcquisitionKey( + awaitingOwner.collectionId, + awaitingOwner.requesterId, + awaitingOwner.acquisitionId, + ) + if (this.inboundRemoteSubsetAcquisitions.get(key) !== awaitingOwner) return + + const acquisition: ActiveRemoteSubsetAcquisition = { + collectionId: awaitingOwner.collectionId, + requesterId: awaitingOwner.requesterId, + acquisitionId: awaitingOwner.acquisitionId, + owner, + options: awaitingOwner.options, + load: Promise.resolve(), + transferred: false, + released: false, + terminalRelease: false, + release: null, + } + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + let resolveLoad!: () => void + let rejectLoad!: (error: unknown) => void + acquisition.load = new Promise((resolve, reject) => { + resolveLoad = resolve + rejectLoad = reject + }) + try { + const load = owner(acquisition.options) + acquisition.transferred = true + void Promise.resolve(load).then(resolveLoad, rejectLoad) + } catch (error) { + rejectLoad(error) + } + try { + await acquisition.load + } catch (error) { + if ( + !acquisition.transferred && + this.inboundRemoteSubsetAcquisitions.get(key) === acquisition + ) { + this.inboundRemoteSubsetAcquisitions.set(key, awaitingOwner) + } + reportRemoteSubsetOwnerError(owner, error) + throw error } } @@ -610,7 +1379,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }, ): Promise { await this.withWriterLock(() => - this.requireAdapter().ensureIndex( + this.requireAdapter(collectionId).ensureIndex( collectionId, request.signature, request.spec, @@ -632,17 +1401,39 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin mutations: Array }, ): Promise { - // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedEnvelopeIds.get(envelopeKey) + if (appliedEnvelope) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + + const inFlightEnvelope = + this.inFlightLocalMutationEnvelopes.get(envelopeKey) + if (inFlightEnvelope) { + const response = await inFlightEnvelope + return { ...response, rpcId: request.rpcId } + } + + const response = this.applyLocalMutationsOnce(collectionId, request) + this.inFlightLocalMutationEnvelopes.set(envelopeKey, response) + try { + return await response + } finally { + if (this.inFlightLocalMutationEnvelopes.get(envelopeKey) === response) { + this.inFlightLocalMutationEnvelopes.delete(envelopeKey) } } + } + private async applyLocalMutationsOnce( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array + }, + ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { return { @@ -663,6 +1454,21 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const rowVersion = state.latestRowVersion // Build and apply the persisted transaction + const rowMetadataMutations: Array = [] + for (const mutation of request.mutations) { + if (!(`metadataChanged` in mutation) || !mutation.metadataChanged) { + continue + } + rowMetadataMutations.push( + mutation.metadata === undefined + ? { type: `delete`, key: mutation.key } + : { + type: `set`, + key: mutation.key, + value: mutation.metadata, + }, + ) + } const tx = { txId: safeRandomUUID(), term, @@ -672,15 +1478,38 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin type: m.type, key: m.key, value: m.value, + ...(`metadataChanged` in m + ? { metadata: m.metadata, metadataChanged: m.metadataChanged } + : {}), })), + rowMetadataMutations, } - await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), - ) + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) + const response: ApplyLocalMutationsResponse = { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term, + seq, + latestRowVersion: rowVersion, + acceptedMutationIds: request.mutations.map((m) => m.mutationId), + } + if (this.isDisposed()) { + return response + } + + this.appliedEnvelopeIds.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { appliedAt: Date.now(), response }, + ) this.pruneAppliedEnvelopeIds() // Broadcast tx:committed to all tabs @@ -706,6 +1535,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin requiresFullReload: false, changedRows, deletedKeys, + rowMetadataMutations, }, } this.channel.postMessage(txCommitted) @@ -715,15 +1545,123 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin subscriber(txCommitted) } - return { - type: `rpc:applyLocalMutations:res`, + return response + } + + private async handleApplyCommittedTx( + collectionId: string, + request: ApplyCommittedTxRequest, + ): Promise { + const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) + const appliedEnvelope = this.appliedCommittedTxEnvelopes.get(envelopeKey) + if (appliedEnvelope) { + return { ...appliedEnvelope.response, rpcId: request.rpcId } + } + + const inFlightEnvelope = this.inFlightCommittedTxEnvelopes.get(envelopeKey) + if (inFlightEnvelope) { + const response = await inFlightEnvelope + return { ...response, rpcId: request.rpcId } + } + + const response = this.applyCommittedTxOnce(collectionId, request) + this.inFlightCommittedTxEnvelopes.set(envelopeKey, response) + try { + return await response + } finally { + if (this.inFlightCommittedTxEnvelopes.get(envelopeKey) === response) { + this.inFlightCommittedTxEnvelopes.delete(envelopeKey) + } + } + } + + private async applyCommittedTxOnce( + collectionId: string, + request: ApplyCommittedTxRequest, + ): Promise { + const state = this.collections.get(collectionId) + if (!state || !state.isLeader) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, + } + } + + state.latestSeq++ + state.latestRowVersion++ + const tx: PersistedTx = { + ...request.tx, + term: state.latestTerm, + seq: state.latestSeq, + rowVersion: state.latestRowVersion, + } + + try { + await this.withWriterLock(() => + this.requireAdapter(collectionId).applyCommittedTx(collectionId, tx), + ) + } catch (error) { + throw toPersistedCollectionDurabilityError(collectionId, error) + } + + const response: ApplyCommittedTxResponse = { + type: `rpc:applyCommittedTx:res`, rpcId: request.rpcId, ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, } + if (this.isDisposed()) { + return response + } + this.appliedCommittedTxEnvelopes.set( + appliedEnvelopeKey(collectionId, request.envelopeId), + { appliedAt: Date.now(), response }, + ) + this.pruneAppliedEnvelopeIds() + + const committedBase = { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + } + const committedPayload: TxCommitted = tx.truncate + ? { ...committedBase, requiresFullReload: true } + : { + ...committedBase, + requiresFullReload: false, + changedRows: tx.mutations + .filter((mutation) => mutation.type !== `delete`) + .map((mutation) => ({ + key: mutation.key, + value: mutation.value, + })), + deletedKeys: tx.mutations + .filter((mutation) => mutation.type === `delete`) + .map((mutation) => mutation.key), + rowMetadataMutations: tx.rowMetadataMutations, + collectionMetadataMutations: tx.collectionMetadataMutations, + } + const committed: ProtocolEnvelope = { + v: 1, + dbName: this.dbName, + collectionId, + senderId: this.nodeId, + ts: Date.now(), + payload: committedPayload, + } + this.channel.postMessage(committed) + for (const subscriber of state.subscribers) { + subscriber(committed) + } + + return response } private async handlePullSince( @@ -736,7 +1674,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter() + const adapter = this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -784,9 +1722,16 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const lockName = `tsdb:writer:${this.dbName}` for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const callbackState = { entered: false } try { - return await navigator.locks.request(lockName, async () => fn()) + return await navigator.locks.request(lockName, async () => { + callbackState.entered = true + return fn() + }) } catch (error) { + if (callbackState.entered) { + throw error + } if (error instanceof DOMException && error.name === `AbortError`) { throw error } @@ -811,11 +1756,16 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { + for (const [id, envelope] of this.appliedEnvelopeIds) { + if (envelope.appliedAt < cutoff) { this.appliedEnvelopeIds.delete(id) } } + for (const [key, envelope] of this.appliedCommittedTxEnvelopes) { + if (envelope.appliedAt < cutoff) { + this.appliedCommittedTxEnvelopes.delete(key) + } + } } } @@ -838,3 +1788,157 @@ function isProtocolEnvelope(data: unknown): data is ProtocolEnvelope { function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } + +async function unloadRemoteSubsetOwner( + owner: RemoteSubsetOwner, + options: TransportedLoadSubsetOptions, +): Promise { + try { + const result = ( + owner.unloadSubset as unknown as ( + options: TransportedLoadSubsetOptions, + ) => unknown + )(options) + await Promise.resolve(result) + } catch (error) { + reportRemoteSubsetOwnerError(owner, error) + throw error + } +} + +function reportRemoteSubsetOwnerError( + owner: RemoteSubsetOwner, + error: unknown, +): void { + try { + owner.onError(error) + } catch { + // Reporting must not replace the original owner failure. + } +} + +function appliedEnvelopeKey(collectionId: string, envelopeId: string): string { + return JSON.stringify([collectionId, envelopeId]) +} + +function remoteSubsetAcquisitionKey( + collectionId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, acquisitionId]) +} + +function inboundRemoteSubsetAcquisitionKey( + collectionId: string, + requesterId: string, + acquisitionId: string, +): string { + return JSON.stringify([collectionId, requesterId, acquisitionId]) +} + +function createRPCErrorResponse( + request: RPCRequest, + cause: unknown, +): RPCResponse { + const error = cause instanceof Error ? cause.message : String(cause) + switch (request.type) { + case `rpc:ensureRemoteSubset:req`: + return { + type: `rpc:ensureRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:releaseRemoteSubset:req`: + return { + type: `rpc:releaseRemoteSubset:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:ensurePersistedIndex:req`: + return { + type: `rpc:ensurePersistedIndex:res`, + rpcId: request.rpcId, + ok: false, + error, + } + case `rpc:applyLocalMutations:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:applyCommittedTx:req`: + if (cause instanceof PersistedCollectionDurabilityError) { + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `PERSISTENCE_ERROR`, + error, + ...toSafeDurabilityDetails(cause), + } + } + return { + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: false, + code: `CONFLICT`, + error, + } + case `rpc:pullSince:req`: + return { + type: `rpc:pullSince:res`, + rpcId: request.rpcId, + ok: false, + error, + } + } +} + +function isMutatingRPCRequest(request: RPCRequest): request is Extract< + RPCRequest, + { + type: IndeterminateCommitRequestType + } +> { + return ( + request.type === `rpc:applyLocalMutations:req` || + request.type === `rpc:applyCommittedTx:req` + ) +} + +function toSafeDurabilityDetails(error: PersistedCollectionDurabilityError): { + sourceCode?: string | number + path?: string | ReadonlyArray +} { + const sourceCode = + typeof error.code === `string` || typeof error.code === `number` + ? error.code + : undefined + const path = + typeof error.path === `string` || + (Array.isArray(error.path) && + error.path.every( + (part) => typeof part === `string` || typeof part === `number`, + )) + ? (error.path as string | ReadonlyArray) + : undefined + return { + ...(sourceCode === undefined ? {} : { sourceCode }), + ...(path === undefined ? {} : { path }), + } +} diff --git a/packages/electron-db-sqlite-persistence/src/index.ts b/packages/electron-db-sqlite-persistence/src/index.ts index 65c5717261..ab41419fed 100644 --- a/packages/electron-db-sqlite-persistence/src/index.ts +++ b/packages/electron-db-sqlite-persistence/src/index.ts @@ -10,8 +10,19 @@ export type { } from './renderer' export { ElectronCollectionCoordinator } from './electron-coordinator' export type { ElectronCollectionCoordinatorOptions } from './electron-coordinator' -export { persistedCollectionOptions } from '@tanstack/db-sqlite-persistence-core' +export { + DuplicateRemoteSubsetOwnerError, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + RemoteSubsetWireValueError, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' export type { PersistedCollectionCoordinator, PersistedCollectionPersistence, + IndeterminateCommitRequestType, + RemoteSubsetOwner, + RemoteSubsetWireExpression, + RemoteSubsetWireValue, + TransportedLoadSubsetOptions, } from '@tanstack/db-sqlite-persistence-core' diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index 2ac7203000..50e459cc95 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -360,22 +360,24 @@ export function createElectronSQLitePersistence( }) adapterCache.set(cacheKey, adapter) - // Wire the adapter into the coordinator so it can handle - // leader-side RPCs (applyCommittedTx, pullSince, getStreamPosition, etc.) - if (coordinator instanceof ElectronCollectionCoordinator) { - coordinator.setAdapter(adapter) - } - return adapter } const createCollectionPersistence = ( mode: PersistedCollectionMode, schemaVersion: number | undefined, - ): PersistedCollectionPersistence => ({ - adapter: getAdapterForCollection(mode, schemaVersion), - coordinator, - }) + collectionId?: string, + ): PersistedCollectionPersistence => { + const adapter = getAdapterForCollection(mode, schemaVersion) + if (coordinator instanceof ElectronCollectionCoordinator) { + if (collectionId === undefined) { + coordinator.setAdapter(adapter) + } else { + coordinator.setAdapterForCollection(collectionId, adapter) + } + } + return { adapter, coordinator } + } const defaultPersistence = createCollectionPersistence( `sync-absent`, @@ -384,8 +386,8 @@ export function createElectronSQLitePersistence( return { ...defaultPersistence, - resolvePersistenceForCollection: ({ mode, schemaVersion }) => - createCollectionPersistence(mode, schemaVersion), + resolvePersistenceForCollection: ({ collectionId, mode, schemaVersion }) => + createCollectionPersistence(mode, schemaVersion, collectionId), // Backward compatible fallback for older callers. resolvePersistenceForMode: (mode) => createCollectionPersistence(mode, undefined), diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index d1c330fb6f..c2913a0833 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -1,11 +1,16 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterEach, describe, expect, it } from 'vitest' -import { InvalidPersistedCollectionConfigError } from '@tanstack/db-sqlite-persistence-core' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + InvalidPersistedCollectionConfigError, + persistedCollectionOptions, +} from '@tanstack/db-sqlite-persistence-core' import { createNodeSQLitePersistence } from '@tanstack/node-db-sqlite-persistence' +import { IR, createCollection } from '../../db/src' import { BetterSqlite3SQLiteDriver } from '../../node-db-sqlite-persistence/src/node-driver' import { + ElectronCollectionCoordinator, createElectronSQLitePersistence, exposeElectronSQLitePersistence, } from '../src' @@ -17,7 +22,18 @@ import { createElectronRuntimeBridgeInvoke, isElectronFullE2EEnabled, } from './e2e/electron-process-client' -import type { PersistedCollectionPersistence } from '@tanstack/db-sqlite-persistence-core' +import type { + ApplyCommittedTxRequest, + ApplyLocalMutationsRequest, + IndeterminateCommitError, + PersistedCollectionDurabilityError, + PersistedCollectionPersistence, + PersistedTx, + PersistenceAdapter, + RemoteSubsetOwner, + TransportedLoadSubsetOptions, +} from '@tanstack/db-sqlite-persistence-core' +import type { LoadSubsetOptions, Subscription, SyncConfig } from '../../db/src' import type { ElectronPersistenceInvoke, ElectronPersistenceRequestEnvelope, @@ -29,6 +45,15 @@ type InvokeHarness = { close: () => void } +type CoordinatorInvokeHarness = { + coordinator: ElectronCollectionCoordinator + persistence: PersistedCollectionPersistence + appliedTransactions: Array + committedPayloads: Array> + start: () => void + close: () => void +} + type ElectronMainPersistence = PersistedCollectionPersistence const electronRuntimeBridgeTimeoutMs = isElectronFullE2EEnabled() @@ -151,6 +176,17 @@ function createInvokeHarness( const activeCleanupFns: Array<() => void> = [] +function registerCleanup(cleanupFn: () => void): () => void { + let cleanupPending = true + const cleanupOnce = () => { + if (!cleanupPending) return + cleanupPending = false + cleanupFn() + } + activeCleanupFns.push(cleanupOnce) + return cleanupOnce +} + afterEach(() => { while (activeCleanupFns.length > 0) { const cleanupFn = activeCleanupFns.pop() @@ -167,6 +203,117 @@ function createTempDbPath(): string { return dbPath } +function installImmediatelyGrantedWebLocks(): () => void { + const originalNavigator = Object.getOwnPropertyDescriptor( + globalThis, + `navigator`, + ) + const navigatorValue = globalThis.navigator + + Object.defineProperty(globalThis, `navigator`, { + value: { + ...navigatorValue, + locks: { + request: async ( + name: string, + optionsOrCallback: + | { signal?: AbortSignal } + | ((lock: { name: string }) => Promise), + maybeCallback?: (lock: { name: string }) => Promise, + ): Promise => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + return callback({ name }) + }, + }, + }, + writable: true, + configurable: true, + }) + + return () => { + if (originalNavigator) { + Object.defineProperty(globalThis, `navigator`, originalNavigator) + } else { + Reflect.deleteProperty(globalThis, `navigator`) + } + } +} + +async function waitForLeadership( + coordinator: ElectronCollectionCoordinator, + collectionId: string, +): Promise { + const deadline = Date.now() + electronRuntimeBridgeTimeoutMs + while (Date.now() < deadline) { + if (coordinator.isLeader(collectionId)) return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`Electron coordinator did not acquire leadership`) +} + +function electronSubsetWithNestedValue(value: unknown): LoadSubsetOptions { + return { + where: new IR.Func(`in`, [ + new IR.PropRef([`todos`, `status`]), + new IR.Value([`kept`, value]), + ]), + } +} + +function createElectronCoordinatorTestAdapter(): PersistenceAdapter { + return { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => Promise.resolve(), + ensureIndex: () => Promise.resolve(), + } +} + +function createCoordinatorInvokeHarness( + dbPath: string, + dbName: string, +): CoordinatorInvokeHarness { + const invokeHarness = createInvokeHarness(dbPath, `todos`) + const coordinator = new ElectronCollectionCoordinator({ dbName }) + const appliedTransactions: Array = [] + const committedPayloads: Array> = [] + const persistence = createElectronSQLitePersistence({ + coordinator, + invoke: async (channel, request) => { + if (request.method === `applyCommittedTx`) { + appliedTransactions.push(structuredClone(request.payload.tx)) + } + return invokeHarness.invoke(channel, request) + }, + timeoutMs: electronRuntimeBridgeTimeoutMs, + }) + const close = registerCleanup(() => { + try { + coordinator.dispose() + } finally { + invokeHarness.close() + } + }) + + return { + coordinator, + persistence, + appliedTransactions, + committedPayloads, + start: () => { + coordinator.subscribe(`todos`, (message) => { + const payload = message.payload as Record + if (payload.type === `tx:committed`) { + committedPayloads.push(payload) + } + }) + }, + close, + } +} + describe(`electron sqlite persistence bridge`, () => { it(`round-trips reads and writes through main process`, async () => { const dbPath = createTempDbPath() @@ -283,6 +430,2841 @@ describe(`electron sqlite persistence bridge`, () => { expect(rows[0]?.value.title).toBe(`Survives restart`) }) + it(`preserves coordinator row metadata updates across restarts`, async () => { + const dbPath = createTempDbPath() + registerCleanup(installImmediatelyGrantedWebLocks()) + const metadataSetHarness = createCoordinatorInvokeHarness( + dbPath, + `electron-metadata-set`, + ) + + await metadataSetHarness.persistence.adapter.applyCommittedTx(`todos`, { + txId: `seed-row`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `todo-1`, + value: { id: `todo-1`, title: `Before` }, + }, + ], + }) + metadataSetHarness.appliedTransactions.length = 0 + metadataSetHarness.start() + await waitForLeadership(metadataSetHarness.coordinator, `todos`) + + const setResponse = + await metadataSetHarness.coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `metadata-set`, + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadataChanged: true, + metadata: { source: `local`, revision: 2 }, + }, + ]) + const setScannedRows = + await metadataSetHarness.persistence.adapter.scanRows?.(`todos`) + const setAppliedTransaction = metadataSetHarness.appliedTransactions[0] + const setCommittedPayload = metadataSetHarness.committedPayloads[0] + + expect({ + response: + setResponse.ok === true + ? { + ok: setResponse.ok, + acceptedMutationIds: setResponse.acceptedMutationIds, + } + : setResponse, + appliedTransactionCount: metadataSetHarness.appliedTransactions.length, + appliedMutation: setAppliedTransaction?.mutations[0], + appliedRowMetadataMutations: + setAppliedTransaction?.rowMetadataMutations ?? null, + committedPayloadCount: metadataSetHarness.committedPayloads.length, + committedType: setCommittedPayload?.type, + committedRowMetadataMutations: + setCommittedPayload?.rowMetadataMutations ?? null, + committedChangedRows: setCommittedPayload?.changedRows, + committedDeletedKeys: setCommittedPayload?.deletedKeys, + scannedRows: setScannedRows, + }).toEqual({ + response: { + ok: true, + acceptedMutationIds: [`metadata-set`], + }, + appliedTransactionCount: 1, + appliedMutation: { + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadataChanged: true, + metadata: { source: `local`, revision: 2 }, + }, + appliedRowMetadataMutations: [ + { + type: `set`, + key: `todo-1`, + value: { source: `local`, revision: 2 }, + }, + ], + committedPayloadCount: 1, + committedType: `tx:committed`, + committedRowMetadataMutations: [ + { + type: `set`, + key: `todo-1`, + value: { source: `local`, revision: 2 }, + }, + ], + committedChangedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + }, + ], + committedDeletedKeys: [], + scannedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadata: { source: `local`, revision: 2 }, + }, + ], + }) + + metadataSetHarness.close() + + const metadataDeleteHarness = createCoordinatorInvokeHarness( + dbPath, + `electron-metadata-delete`, + ) + expect( + await metadataDeleteHarness.persistence.adapter.scanRows?.(`todos`), + ).toEqual([ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After set` }, + metadata: { source: `local`, revision: 2 }, + }, + ]) + + metadataDeleteHarness.start() + await waitForLeadership(metadataDeleteHarness.coordinator, `todos`) + const deleteResponse = + await metadataDeleteHarness.coordinator.requestApplyLocalMutations( + `todos`, + [ + { + mutationId: `metadata-delete`, + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadataChanged: true, + }, + ], + ) + const deleteScannedRows = + await metadataDeleteHarness.persistence.adapter.scanRows?.(`todos`) + const deleteAppliedTransaction = + metadataDeleteHarness.appliedTransactions[0] + const deleteCommittedPayload = metadataDeleteHarness.committedPayloads[0] + + expect({ + response: + deleteResponse.ok === true + ? { + ok: deleteResponse.ok, + acceptedMutationIds: deleteResponse.acceptedMutationIds, + } + : deleteResponse, + appliedTransactionCount: metadataDeleteHarness.appliedTransactions.length, + appliedMutation: deleteAppliedTransaction?.mutations[0], + appliedRowMetadataMutations: + deleteAppliedTransaction?.rowMetadataMutations ?? null, + committedPayloadCount: metadataDeleteHarness.committedPayloads.length, + committedType: deleteCommittedPayload?.type, + committedRowMetadataMutations: + deleteCommittedPayload?.rowMetadataMutations ?? null, + committedChangedRows: deleteCommittedPayload?.changedRows, + committedDeletedKeys: deleteCommittedPayload?.deletedKeys, + scannedRows: deleteScannedRows, + }).toEqual({ + response: { + ok: true, + acceptedMutationIds: [`metadata-delete`], + }, + appliedTransactionCount: 1, + appliedMutation: { + type: `update`, + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadataChanged: true, + metadata: undefined, + }, + appliedRowMetadataMutations: [{ type: `delete`, key: `todo-1` }], + committedPayloadCount: 1, + committedType: `tx:committed`, + committedRowMetadataMutations: [{ type: `delete`, key: `todo-1` }], + committedChangedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + }, + ], + committedDeletedKeys: [], + scannedRows: [ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadata: undefined, + }, + ], + }) + + metadataDeleteHarness.close() + + const reopenedInvokeHarness = createInvokeHarness(dbPath, `todos`) + const closeReopenedHarness = registerCleanup(reopenedInvokeHarness.close) + const reopenedPersistence = createElectronSQLitePersistence({ + invoke: reopenedInvokeHarness.invoke, + timeoutMs: electronRuntimeBridgeTimeoutMs, + }) + expect(await reopenedPersistence.adapter.scanRows?.(`todos`)).toEqual([ + { + key: `todo-1`, + value: { id: `todo-1`, title: `After delete` }, + metadata: undefined, + }, + ]) + closeReopenedHarness() + }) + + it(`routes a rich source transaction through the Electron owner and durable reopen`, async () => { + type Todo = { id: string; title: string } + type SourceParams = Parameters[`sync`]>[0] + + const dbPath = createTempDbPath() + registerCleanup(installImmediatelyGrantedWebLocks()) + const ownerHarness = createCoordinatorInvokeHarness( + dbPath, + `electron-rich-source`, + ) + + await ownerHarness.persistence.adapter.applyCommittedTx(`todos`, { + txId: `stale-seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `stale`, + value: { id: `stale`, title: `Must be truncated` }, + }, + ], + }) + ownerHarness.appliedTransactions.length = 0 + ownerHarness.start() + await waitForLeadership(ownerHarness.coordinator, `todos`) + + let sourceParams: SourceParams | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `todos`, + getKey: (todo) => todo.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: ownerHarness.persistence, + }), + ) + + try { + await collection.stateWhenReady() + sourceParams!.begin() + sourceParams!.metadata?.collection.set(`resume`, { offset: 9 }) + sourceParams!.truncate() + sourceParams!.write({ + type: `insert`, + value: { id: `fresh`, title: `Durable through owner` }, + metadata: { source: `electron-sync` }, + }) + const receipt = sourceParams!.commit() + if (receipt !== true) await receipt + + expect(collection.has(`stale`)).toBe(false) + expect(collection.get(`fresh`)).toMatchObject({ + id: `fresh`, + title: `Durable through owner`, + }) + expect(ownerHarness.appliedTransactions).toHaveLength(1) + expect(ownerHarness.appliedTransactions[0]).toMatchObject({ + truncate: true, + mutations: [ + { + type: `update`, + key: `fresh`, + value: { id: `fresh`, title: `Durable through owner` }, + }, + ], + rowMetadataMutations: [ + { + type: `set`, + key: `fresh`, + value: { source: `electron-sync` }, + }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `resume`, value: { offset: 9 } }, + ], + }) + expect(ownerHarness.committedPayloads).toHaveLength(1) + expect(ownerHarness.committedPayloads[0]).toMatchObject({ + type: `tx:committed`, + requiresFullReload: true, + }) + } finally { + await collection.cleanup() + } + + ownerHarness.close() + + const reopenedInvokeHarness = createInvokeHarness(dbPath, `todos`) + const closeReopenedHarness = registerCleanup(reopenedInvokeHarness.close) + const reopenedPersistence = createElectronSQLitePersistence({ + invoke: reopenedInvokeHarness.invoke, + timeoutMs: electronRuntimeBridgeTimeoutMs, + }) + expect(await reopenedPersistence.adapter.scanRows?.(`todos`)).toEqual([ + { + key: `fresh`, + value: { id: `fresh`, title: `Durable through owner` }, + metadata: { source: `electron-sync` }, + }, + ]) + expect( + await reopenedPersistence.adapter.loadCollectionMetadata?.(`todos`), + ).toEqual([{ key: `resume`, value: { offset: 9 } }]) + closeReopenedHarness() + }) + + it( + `does not retry a committed transaction after the writer callback starts`, + { timeout: 10_000 }, + async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const persistenceError = new Error(`irreversible apply failed`) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-writer-callback-failure`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => { + applyCalls++ + return Promise.reject(persistenceError) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + await expect( + coordinator.requestApplyCommittedTx(`todos`, { + txId: `irreversible-effect`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + cause: persistenceError, + } satisfies Partial) + expect(applyCalls).toBe(1) + }, + ) + + it(`fails indeterminate instead of retrying a committed mutation across leaders`, async () => { + const bEffects: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-requester-takeover`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: (_collectionId, tx) => { + bEffects.push(tx) + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + const internals = coordinator as unknown as { + nodeId: string + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + } + internals.acquireLeadership = async () => {} + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName: `electron-requester-takeover`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + internals.onChannelMessage(heartbeat(`electron-leader-a`, 4)) + const transportError = new Error(`leader A response was lost`) + const aEffects: Array = [] + let aPublications = 0 + let bPublications = 0 + let transportCalls = 0 + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyCommittedTxRequest) => { + transportCalls++ + if (transportCalls === 1) { + aEffects.push(structuredClone(request.tx)) + aPublications++ + internals.onChannelMessage(heartbeat(`electron-leader-b`, 5)) + return Promise.reject(transportError) + } + bEffects.push(structuredClone(request.tx)) + bPublications++ + return Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: request.rpcId, + ok: true, + term: 5, + seq: 1, + latestRowVersion: 1, + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + vi.useFakeTimers() + try { + const outcomePromise = coordinator + .requestApplyCommittedTx(`todos`, { + txId: `requester-takeover`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `takeover`, + value: { id: `takeover` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + await vi.advanceTimersByTimeAsync(1_000) + const outcome = await outcomePromise + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: `electron-leader-a`, + previousTerm: 4, + currentLeaderId: `electron-leader-b`, + currentTerm: 5, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(aEffects).toHaveLength(1) + expect(aPublications).toBe(1) + expect(aEffects[0]?.txId).toBe(`requester-takeover`) + expect(bEffects).toEqual([]) + expect(bPublications).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it(`fails indeterminate instead of retrying a committed transaction without an initial leader route`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-unknown-leader-committed`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + const transportError = new Error(`unknown leader response was lost`) + const attemptedTransactions: Array = [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyCommittedTxRequest) => { + attemptedTransactions.push(structuredClone(request.tx)) + if (attemptedTransactions.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyCommittedTx:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + const outcome = await coordinator + .requestApplyCommittedTx(`todos`, { + txId: `electron-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `electron-unknown-leader-committed`, + value: { id: `electron-unknown-leader-committed` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyCommittedTx:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedTransactions).toEqual([ + { + txId: `electron-unknown-leader-committed`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [ + { + type: `insert`, + key: `electron-unknown-leader-committed`, + value: { id: `electron-unknown-leader-committed` }, + }, + ], + }, + ]) + }) + + it(`fails indeterminate instead of retrying local mutations across leaders`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-local-requester-takeover`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + } + internals.acquireLeadership = async () => {} + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName: `electron-local-requester-takeover`, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + internals.onChannelMessage(heartbeat(`electron-local-leader-a`, 7)) + const transportError = new Error(`leader A local response was lost`) + const aEffects: Array = [] + const bEffects: Array = [] + let aPublications = 0 + let bPublications = 0 + let transportCalls = 0 + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + transportCalls++ + if (transportCalls === 1) { + aEffects.push(structuredClone(request.mutations)) + aPublications++ + internals.onChannelMessage(heartbeat(`electron-local-leader-b`, 8)) + return Promise.reject(transportError) + } + bEffects.push(structuredClone(request.mutations)) + bPublications++ + return Promise.resolve({ + type: `rpc:applyLocalMutations:res`, + rpcId: request.rpcId, + ok: true, + term: 8, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `electron-local-takeover`, + type: `insert`, + key: `electron-local-takeover`, + value: { id: `electron-local-takeover` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: `electron-local-leader-a`, + previousTerm: 7, + currentLeaderId: `electron-local-leader-b`, + currentTerm: 8, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(aEffects).toHaveLength(1) + expect(aPublications).toBe(1) + expect(bEffects).toEqual([]) + expect(bPublications).toBe(0) + }) + + it(`fails indeterminate instead of retrying local mutations without an initial leader route`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-unknown-leader-local`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + const transportError = new Error(`unknown leader response was lost`) + const attemptedMutations: Array = + [] + const failedTransport = vi.fn( + (_collectionId: string, request: ApplyLocalMutationsRequest) => { + attemptedMutations.push(structuredClone(request.mutations)) + if (attemptedMutations.length === 1) { + return Promise.reject(transportError) + } + return Promise.resolve({ + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + }) + }, + ) + Object.defineProperty(coordinator, `sendRPCOnce`, { + value: failedTransport, + configurable: true, + }) + + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `electron-unknown-leader-local`, + type: `insert`, + key: `electron-unknown-leader-local`, + value: { id: `electron-unknown-leader-local` }, + }, + ]) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + expect(outcome).toEqual({ + error: expect.objectContaining({ + name: `IndeterminateCommitError`, + code: `INDETERMINATE_COMMIT`, + collectionId: `todos`, + requestType: `rpc:applyLocalMutations:req`, + previousLeaderId: null, + previousTerm: null, + currentLeaderId: null, + currentTerm: null, + cause: transportError, + } satisfies Partial), + }) + expect(failedTransport).toHaveBeenCalledTimes(1) + expect(attemptedMutations).toEqual([ + [ + { + mutationId: `electron-unknown-leader-local`, + type: `insert`, + key: `electron-unknown-leader-local`, + value: { id: `electron-unknown-leader-local` }, + }, + ], + ]) + }) + + it(`binds committed transaction owners to each resolved collection adapter`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const requests: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-per-collection-adapters`, + }) + registerCleanup(() => coordinator.dispose()) + const persistence = createElectronSQLitePersistence({ + coordinator, + invoke: (_channel, request) => { + requests.push(structuredClone(request)) + if (request.method === `getStreamPosition`) { + return Promise.resolve({ + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: request.requestId, + method: request.method, + ok: true, + result: { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + } + if (request.method === `applyCommittedTx`) { + return Promise.resolve({ + v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, + requestId: request.requestId, + method: request.method, + ok: true, + result: null, + }) + } + throw new Error(`unexpected method ${request.method}`) + }, + }) + + persistence.resolvePersistenceForCollection?.({ + collectionId: `alpha`, + mode: `sync-present`, + schemaVersion: 1, + }) + persistence.resolvePersistenceForCollection?.({ + collectionId: `beta`, + mode: `sync-present`, + schemaVersion: 2, + }) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + await waitForLeadership(coordinator, `alpha`) + await waitForLeadership(coordinator, `beta`) + + await coordinator.requestApplyCommittedTx(`alpha`, { + txId: `alpha-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + await coordinator.requestApplyCommittedTx(`beta`, { + txId: `beta-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }) + + expect( + requests + .filter((request) => request.method === `applyCommittedTx`) + .map((request) => ({ + collectionId: request.collectionId, + resolution: request.resolution, + txId: request.payload.tx.txId, + })), + ).toEqual([ + { + collectionId: `alpha`, + resolution: { mode: `sync-present`, schemaVersion: 1 }, + txId: `alpha-tx`, + }, + { + collectionId: `beta`, + resolution: { mode: `sync-present`, schemaVersion: 2 }, + txId: `beta-tx`, + }, + ]) + }) + + it(`coalesces and replays the exact committed success on the same leader`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-committed-envelope-replay`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async () => { + applyCalls++ + await applyGate + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const handleApplyCommittedTx = ( + coordinator as unknown as { + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + ).handleApplyCommittedTx.bind(coordinator) + const request: ApplyCommittedTxRequest = { + type: `rpc:applyCommittedTx:req`, + rpcId: `first-rpc`, + envelopeId: `stable-envelope`, + tx: { + txId: `stable-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + } + + const first = handleApplyCommittedTx(`todos`, request) + const duplicate = handleApplyCommittedTx(`todos`, { + ...request, + rpcId: `duplicate-rpc`, + }) + await Promise.resolve() + expect(applyCalls).toBe(1) + releaseApply() + expect(await Promise.all([first, duplicate])).toMatchObject([ + { ok: true, rpcId: `first-rpc` }, + { ok: true, rpcId: `duplicate-rpc` }, + ]) + + await expect( + handleApplyCommittedTx(`todos`, { + ...request, + rpcId: `replay-rpc`, + }), + ).resolves.toMatchObject({ ok: true, rpcId: `replay-rpc` }) + expect(applyCalls).toBe(1) + }) + + it(`replays the exact local-mutation success on the same leader`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyCalls = 0 + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-local-envelope-replay`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => { + applyCalls++ + return Promise.resolve() + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const handleApplyLocalMutations = ( + coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: ApplyLocalMutationsRequest, + ) => Promise + } + ).handleApplyLocalMutations.bind(coordinator) + const request: ApplyLocalMutationsRequest = { + type: `rpc:applyLocalMutations:req`, + rpcId: `first-local-rpc`, + envelopeId: `stable-local-envelope`, + mutations: [ + { + mutationId: `stable-local-mutation`, + type: `insert`, + key: `stable-local-row`, + value: { id: `stable-local-row` }, + }, + ], + } + + const first = await handleApplyLocalMutations(`todos`, request) + const replay = await handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `replayed-local-rpc`, + }) + + expect([first, replay]).toEqual([ + { + type: `rpc:applyLocalMutations:res`, + rpcId: `first-local-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`stable-local-mutation`], + }, + { + type: `rpc:applyLocalMutations:res`, + rpcId: `replayed-local-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`stable-local-mutation`], + }, + ]) + expect(applyCalls).toBe(1) + }) + + it(`classifies Electron durability failures on local and follower routes`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const persistenceError = Object.assign(new Error(`electron disk failed`), { + code: `SQLITE_IOERR`, + path: [`electron`, `todos`], + }) + const adapter = createElectronCoordinatorTestAdapter() + adapter.applyCommittedTx = vi.fn().mockRejectedValue(persistenceError) + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-durability-classification`, + adapter, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const responses: Array = [] + const internals = coordinator as unknown as { + channel: { postMessage: (message: unknown) => void } + handleRPCRequest: ( + collectionId: string, + request: ApplyLocalMutationsRequest | ApplyCommittedTxRequest, + requesterId: string, + ) => Promise + } + internals.channel.postMessage = (message) => { + responses.push((message as { payload: unknown }).payload) + } + + await internals.handleRPCRequest( + `todos`, + { + type: `rpc:applyLocalMutations:req`, + rpcId: `remote-local-failure-rpc`, + envelopeId: `remote-local-failure-envelope`, + mutations: [], + }, + `remote-electron-follower`, + ) + await internals.handleRPCRequest( + `todos`, + { + type: `rpc:applyCommittedTx:req`, + rpcId: `remote-committed-failure-rpc`, + envelopeId: `remote-committed-failure-envelope`, + tx: { + txId: `remote-committed-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }, + `remote-electron-follower`, + ) + + expect(responses).toEqual([ + { + type: `rpc:applyLocalMutations:res`, + rpcId: `remote-local-failure-rpc`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(persistenceError.message), + sourceCode: `SQLITE_IOERR`, + path: [`electron`, `todos`], + }, + { + type: `rpc:applyCommittedTx:res`, + rpcId: `remote-committed-failure-rpc`, + ok: false, + code: `PERSISTENCE_ERROR`, + error: expect.stringContaining(persistenceError.message), + sourceCode: `SQLITE_IOERR`, + path: [`electron`, `todos`], + }, + ]) + + await expect( + coordinator.requestApplyLocalMutations(`todos`, []), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR`, + path: [`electron`, `todos`], + cause: persistenceError, + } satisfies Partial) + await expect( + coordinator.requestApplyCommittedTx(`todos`, { + txId: `local-committed-failure`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }), + ).rejects.toMatchObject({ + name: `PersistedCollectionDurabilityError`, + code: `SQLITE_IOERR`, + path: [`electron`, `todos`], + cause: persistenceError, + } satisfies Partial) + expect(adapter.applyCommittedTx).toHaveBeenCalledTimes(4) + }) + + it(`preserves a held committed apply without retaining or publishing after disposal`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + const durableTransactions: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-committed`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async (_collectionId, tx) => { + applyEntered = true + await applyGate + durableTransactions.push(structuredClone(tx)) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const internals = coordinator as unknown as { + handleApplyCommittedTx: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + appliedCommittedTxEnvelopes: Map + inFlightCommittedTxEnvelopes: Map + } + const outcomePromise = internals + .handleApplyCommittedTx(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-committed-rpc`, + envelopeId: `held-committed-envelope`, + tx: { + txId: `held-committed-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + try { + await vi.waitFor(() => expect(applyEntered).toBe(true)) + coordinator.dispose() + releaseApply() + const outcome = await outcomePromise + + expect(outcome).toEqual({ + response: { + type: `rpc:applyCommittedTx:res`, + rpcId: `held-committed-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + }, + }) + expect(durableTransactions).toHaveLength(1) + expect(durableTransactions[0]?.txId).toBe(`held-committed-tx`) + expect({ + completed: internals.appliedCommittedTxEnvelopes.size, + inFlight: internals.inFlightCommittedTxEnvelopes.size, + }).toEqual({ completed: 0, inFlight: 0 }) + } finally { + releaseApply() + coordinator.dispose() + } + }) + + it(`preserves a held local apply without retaining or publishing after disposal`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + const durableTransactions: Array = [] + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-local`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async (_collectionId, tx) => { + applyEntered = true + await applyGate + durableTransactions.push(structuredClone(tx)) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.subscribe(`todos`, () => {}) + await waitForLeadership(coordinator, `todos`) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: ApplyLocalMutationsRequest, + ) => Promise + appliedEnvelopeIds: Map + } + const outcomePromise = internals + .handleApplyLocalMutations(`todos`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `held-local-rpc`, + envelopeId: `held-local-envelope`, + mutations: [ + { + mutationId: `held-local-mutation`, + type: `insert`, + key: `held-local-row`, + value: { id: `held-local-row` }, + }, + ], + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + try { + await vi.waitFor(() => expect(applyEntered).toBe(true)) + coordinator.dispose() + releaseApply() + const outcome = await outcomePromise + + expect(outcome).toEqual({ + response: { + type: `rpc:applyLocalMutations:res`, + rpcId: `held-local-rpc`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`held-local-mutation`], + }, + }) + expect(durableTransactions).toHaveLength(1) + expect(durableTransactions[0]?.mutations).toEqual([ + { + type: `insert`, + key: `held-local-row`, + value: { id: `held-local-row` }, + }, + ]) + expect(internals.appliedEnvelopeIds.size).toBe(0) + } finally { + releaseApply() + coordinator.dispose() + } + }) + + it(`validates remote-subset values before Electron leader completion`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const leader = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-leader`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => Promise.resolve(), + ensureIndex: () => Promise.resolve(), + }, + }) + registerCleanup(() => leader.dispose()) + leader.subscribe(`todos`, () => {}) + await waitForLeadership(leader, `todos`) + + const invalid = electronSubsetWithNestedValue(() => {}) + await expect( + leader.requestEnsureRemoteSubset(`todos`, invalid), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + }) + }) + + it(`validates remote-subset values before Electron follower transport`, async () => { + const follower = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-follower`, + }) + registerCleanup(() => follower.dispose()) + follower.isLeader = () => false + const followerInternals = follower as unknown as { + channel: { postMessage: (message: unknown) => void } + } + let subsetPosts = 0 + followerInternals.channel.postMessage = (message) => { + const payload = (message as { payload?: { type?: string } }).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) subsetPosts++ + structuredClone(message) + } + const outcome = await follower + .requestEnsureRemoteSubset( + `todos`, + electronSubsetWithNestedValue(() => {}), + ) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ) + expect({ outcome, subsetPosts }).toMatchObject({ + outcome: { + ok: false, + error: { + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1].value[1]`, + }, + }, + subsetPosts: 0, + }) + }) + + it(`rejects a sparse function argument at its exact Electron wire path`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-hole`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const args = [new IR.PropRef([`todos`, `status`])] + args.length = 2 + + try { + await expect( + coordinator.requestEnsureRemoteSubset(`todos`, { + where: new IR.Func(`eq`, args), + }), + ).rejects.toMatchObject({ + name: `RemoteSubsetWireValueError`, + path: `options.where.args[1]`, + }) + expect(owner).not.toHaveBeenCalled() + } finally { + unregisterOwner() + } + }) + + it(`holds same-stack Electron subset reentry behind the original owner load`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-same-stack`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let duplicate: Promise | undefined + let didReenter = false + const owner = Object.assign( + vi.fn(() => { + if (!didReenter) { + didReenter = true + duplicate = coordinator.requestEnsureRemoteSubset(`todos`, options) + } + return loadGate + }), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + let firstSettled = false + let duplicateSettled = false + const first = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(duplicate).toBeDefined()) + const duplicateResult = duplicate!.then(() => { + duplicateSettled = true + }) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + ownerCalls: owner.mock.calls.length, + firstSettled, + duplicateSettled, + }).toEqual({ + ownerCalls: 1, + firstSettled: false, + duplicateSettled: false, + }) + + releaseLoad() + await Promise.all([first, duplicateResult]) + } finally { + releaseLoad() + unregisterOwner() + } + }) + + it(`compacts a terminal same-stack Electron release after the real owner load finishes`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-terminal-release`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const options: LoadSubsetOptions = { limit: 1 } + let releaseLoad = (): void => {} + const loadGate = new Promise((resolve) => { + releaseLoad = resolve + }) + let release: Promise | undefined + const events: Array = [] + const owner = Object.assign( + vi.fn(() => { + events.push(`load`) + release = coordinator.requestReleaseRemoteSubset(`todos`, options) + return loadGate + }), + { + unloadSubset: vi.fn(() => { + events.push(`unload`) + }), + onError: vi.fn(), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + } + + try { + let releaseSettled = false + const load = coordinator.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(release).toBeDefined()) + const terminalRelease = release!.then(() => { + releaseSettled = true + }) + await Promise.resolve() + expect({ events: [...events], releaseSettled }).toEqual({ + events: [`load`], + releaseSettled: false, + }) + + releaseLoad() + await Promise.all([load, terminalRelease]) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + expect({ + events, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + events: [`load`, `unload`], + outbound: 0, + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + releaseLoad() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`releases a transferred Electron lease whose initial load rejected`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-rejected-transfer`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const loadError = new Error(`electron transferred load failed`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => + Promise.reject(loadError), + ), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 20 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + const ensureError = await coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => undefined, + (error: unknown) => error, + ) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + await new Promise((resolve) => setTimeout(resolve, 0)) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(ensureError).toBe(loadError) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset.mock.calls[0]?.[0]).toBe( + owner.mock.calls[0]?.[0], + ) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`keeps an Electron release tombstone when a transferred load rejects concurrently`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-concurrent-rejected-transfer`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const loadError = new Error(`electron concurrent transferred load failed`) + let rejectLoad = (_error: unknown): void => {} + const loadGate = new Promise((_resolve, reject) => { + rejectLoad = reject + }) + const ownerErrors: Array = [] + const owner = Object.assign( + vi.fn((_options: TransportedLoadSubsetOptions) => loadGate), + { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { offset: 21 } + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + nodeId: string + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map> + handleEnsureRemoteSubset: ( + collectionId: string, + request: { + type: `rpc:ensureRemoteSubset:req` + rpcId: string + acquisitionId: string + options: TransportedLoadSubsetOptions + }, + requesterId: string, + ) => Promise<{ ok: boolean }> + } + + try { + const ensure = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + const [outbound] = internals.outboundRemoteSubsetAcquisitions.values() + const release = coordinator + .requestReleaseRemoteSubset(`todos`, options) + .then( + () => ({ status: `fulfilled` as const }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + rejectLoad(loadError) + const outcomes = await Promise.all([ensure, release]) + const duplicate = await internals + .handleEnsureRemoteSubset( + `todos`, + { + type: `rpc:ensureRemoteSubset:req`, + rpcId: `delayed-electron-duplicate`, + acquisitionId: outbound!.acquisitionId, + options: owner.mock.calls[0]![0], + }, + internals.nodeId, + ) + .then( + (response) => ({ status: `fulfilled` as const, response }), + (error: unknown) => ({ status: `rejected` as const, error }), + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + const [terminal] = internals.inboundRemoteSubsetAcquisitions.values() + + expect(outcomes).toEqual([ + { status: `rejected`, error: loadError }, + { status: `fulfilled` }, + ]) + expect(duplicate).toEqual({ + status: `fulfilled`, + response: expect.objectContaining({ ok: true }), + }) + expect(owner).toHaveBeenCalledTimes(1) + expect(owner.unloadSubset).toHaveBeenCalledTimes(1) + expect(ownerErrors).toEqual([loadError]) + expect(unhandled).toEqual([]) + expect({ + inbound: internals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + rejectLoad(loadError) + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`posts only the validated remote-subset wire domain from Electron`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-wire-projection`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => false + + const shared = { label: `shared` } + const cycle: Record = { label: `cycle` } + cycle.self = cycle + const options: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([`todos`, `payload`]), + new IR.Value({ + date: new Date(`2026-09-16T12:34:56.000Z`), + regexp: /wire/giu, + bytes: new Uint8Array([0, 127, 255]), + map: new Map([[shared, cycle]]), + set: new Set([shared, cycle]), + first: shared, + second: shared, + cycle, + }), + ]), + signal: new AbortController().signal, + subscription: { + on: () => () => {}, + } as unknown as Subscription, + } + const internals = coordinator as unknown as { + channel: { postMessage: (message: unknown) => void } + } + let posted: unknown + internals.channel.postMessage = (message) => { + posted = structuredClone(message) + } + + const pending = coordinator + .requestEnsureRemoteSubset(`todos`, options) + .catch((error: unknown) => error) + await Promise.resolve() + + const wireOptions = ( + posted as + | { + payload?: { + type?: string + options?: LoadSubsetOptions + } + } + | undefined + )?.payload?.options + expect( + (posted as { payload?: { type?: string } } | undefined)?.payload?.type, + ).toBe(`rpc:ensureRemoteSubset:req`) + expect(wireOptions).not.toHaveProperty(`signal`) + expect(wireOptions).not.toHaveProperty(`subscription`) + const wireValue = ( + wireOptions?.where as unknown as + | { + args: Array<{ + value?: { + date: Date + regexp: RegExp + bytes: Uint8Array + map: Map + set: Set + first: unknown + second: unknown + cycle: Record + } + }> + } + | undefined + )?.args[1]?.value + expect(wireValue?.date.getTime()).toBe( + new Date(`2026-09-16T12:34:56.000Z`).getTime(), + ) + expect(wireValue?.regexp).toEqual(/wire/giu) + expect(Array.from(wireValue?.bytes ?? [])).toEqual([0, 127, 255]) + expect(wireValue?.first).toBe(wireValue?.second) + expect([...wireValue!.map.keys()]).toEqual([wireValue?.first]) + expect(wireValue?.map.get(wireValue.first)).toBe(wireValue?.cycle) + expect(wireValue?.set.has(wireValue.first)).toBe(true) + expect(wireValue?.cycle.self).toBe(wireValue?.cycle) + + coordinator.dispose() + await pending + }) + + it(`loads and releases exact Electron remote-subset acquisitions`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-routing`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + + const loads: Array = [] + const unloads: Array = [] + const owner = Object.assign( + vi.fn((options: TransportedLoadSubsetOptions) => { + loads.push(options) + }), + { + unloadSubset: vi.fn((options: TransportedLoadSubsetOptions) => { + unloads.push(options) + }), + onError: vi.fn(), + }, + ) satisfies RemoteSubsetOwner + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const first = { limit: 2, offset: 1 } + const sibling = { limit: 2, offset: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, first) + await coordinator.requestEnsureRemoteSubset(`todos`, first) + await coordinator.requestEnsureRemoteSubset(`todos`, sibling) + + expect(loads).toEqual([ + { limit: 2, offset: 1 }, + { limit: 2, offset: 1 }, + ]) + + await coordinator.requestReleaseRemoteSubset(`todos`, first) + await coordinator.requestReleaseRemoteSubset(`todos`, first) + expect(unloads).toEqual([{ limit: 2, offset: 1 }]) + + await coordinator.requestReleaseRemoteSubset(`todos`, sibling) + expect(unloads).toEqual([ + { limit: 2, offset: 1 }, + { limit: 2, offset: 1 }, + ]) + } finally { + unregisterOwner() + } + }) + + it(`scopes a reused subset options object to each Electron collection`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-collection-identity`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const alpha = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const beta = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterAlpha = coordinator.registerRemoteSubsetOwner( + `alpha`, + alpha, + ) + const unregisterBeta = coordinator.registerRemoteSubsetOwner(`beta`, beta) + const shared: LoadSubsetOptions = { limit: 2, offset: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + await coordinator.requestEnsureRemoteSubset(`beta`, shared) + await coordinator.requestEnsureRemoteSubset(`alpha`, shared) + + const afterAcquire = { + alphaLoads: alpha.mock.calls.length, + betaLoads: beta.mock.calls.length, + } + await coordinator.requestReleaseRemoteSubset(`alpha`, shared) + await coordinator.requestReleaseRemoteSubset(`beta`, shared) + + expect({ + afterAcquire, + alphaUnloads: alpha.unloadSubset.mock.calls.length, + betaUnloads: beta.unloadSubset.mock.calls.length, + }).toEqual({ + afterAcquire: { alphaLoads: 1, betaLoads: 1 }, + alphaUnloads: 1, + betaUnloads: 1, + }) + } finally { + unregisterAlpha() + unregisterBeta() + } + }) + + it(`replays after an earlier Electron leader responds behind a new heartbeat`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + const dbName = `electron-subset-stale-response-${process.pid}` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: () => Promise.resolve(), + ensureIndex: () => Promise.resolve(), + }, + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + }) + ;( + follower as unknown as { acquireLeadership: () => Promise } + ).acquireLeadership = async () => {} + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + let releaseOwner = (): void => {} + const owner = Object.assign( + vi.fn( + () => + new Promise((resolve) => { + releaseOwner = resolve + }), + ), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) satisfies RemoteSubsetOwner + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + leader.subscribe(`todos`, () => {}) + await waitForLeadership(leader, `todos`) + + const leaderInternals = leader as unknown as { + nodeId: string + channel: BroadcastChannel + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + channel: BroadcastChannel + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message: unknown) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensureRemoteSubset:res`) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + const originalFollowerPost = followerInternals.channel.postMessage.bind( + followerInternals.channel, + ) + let requestPosts = 0 + followerInternals.channel.postMessage = (message: unknown) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + originalFollowerPost(message) + } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, { limit: 1 }) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + followerInternals.onChannelMessage({ + v: 1, + dbName, + collectionId: `todos`, + senderId: `replacement-electron-leader`, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term: 2, + leaderId: `replacement-electron-leader`, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + releaseOwner() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + leader.isLeader = () => false + followerInternals.onChannelMessage(heldResponse) + await request + await new Promise((resolve) => setTimeout(resolve, 0)) + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + acquiredLeaderId: leaderInternals.nodeId, + }) + } finally { + releaseOwner() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`replays a held Electron lease after an A to B to A leader cycle`, async () => { + const dbName = `electron-subset-a-b-a` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.isLeader = () => true + follower.isLeader = () => false + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const owner = Object.assign( + vi.fn().mockImplementationOnce(() => firstLoadGate), + { unloadSubset: vi.fn(), onError: vi.fn() }, + ) + const unregisterOwner = leader.registerRemoteSubsetOwner(`todos`, owner) + const leaderInternals = leader as unknown as { + nodeId: string + channel: BroadcastChannel + inboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const followerInternals = follower as unknown as { + onChannelMessage: (message: unknown) => void + channel: BroadcastChannel + acquireLeadership: () => Promise + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null } + > + } + followerInternals.acquireLeadership = async () => {} + const originalLeaderPost = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let heldResponse: unknown + leaderInternals.channel.postMessage = (message) => { + const type = (message as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensureRemoteSubset:res` && heldResponse === undefined) { + heldResponse = structuredClone(message) + return + } + originalLeaderPost(message) + } + const originalFollowerPost = followerInternals.channel.postMessage.bind( + followerInternals.channel, + ) + let requestPosts = 0 + const wireRequests: Array<{ + acquisitionId: string + rpcId: string + }> = [] + followerInternals.channel.postMessage = (message) => { + const payload = ( + message as { + payload?: { + type?: string + acquisitionId?: string + rpcId?: string + } + } + ).payload + if (payload?.type === `rpc:ensureRemoteSubset:req`) { + requestPosts++ + wireRequests.push({ + acquisitionId: payload.acquisitionId!, + rpcId: payload.rpcId!, + }) + } + originalFollowerPost(message) + } + const heartbeat = (leaderId: string, term: number) => ({ + v: 1, + dbName, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + const request = follower.requestEnsureRemoteSubset(`todos`, options) + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(1)) + followerInternals.onChannelMessage( + heartbeat(`replacement-electron-leader`, 2), + ) + leaderInternals.releaseInboundRemoteSubsetAcquisitions(`todos`) + followerInternals.onChannelMessage(heartbeat(leaderInternals.nodeId, 3)) + releaseFirstLoad() + await vi.waitFor(() => expect(heldResponse).toBeDefined()) + followerInternals.onChannelMessage(heldResponse) + await request + await vi.waitFor(() => expect(owner).toHaveBeenCalledTimes(2)) + + const [acquisition] = + followerInternals.outboundRemoteSubsetAcquisitions.values() + expect({ + requestPosts, + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + acquiredLeaderId: acquisition?.acquiredLeaderId, + }).toEqual({ + requestPosts: 2, + loads: 2, + unloads: 1, + inbound: 1, + acquiredLeaderId: leaderInternals.nodeId, + }) + expect(wireRequests).toHaveLength(2) + expect(wireRequests[0]!.acquisitionId).not.toBe(``) + expect(wireRequests[1]!.acquisitionId).toBe( + wireRequests[0]!.acquisitionId, + ) + expect(wireRequests[1]!.rpcId).not.toBe(wireRequests[0]!.rpcId) + + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + releaseFirstLoad() + unregisterOwner() + leader.dispose() + follower.dispose() + } + }) + + it(`rebinds a live Electron acquisition when its owner is replaced`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-replacement`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn( + (_options: TransportedLoadSubsetOptions) => undefined, + ), + onError: vi.fn(), + }) + const unregisterFirst = coordinator.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + + try { + const initialAcquire = coordinator.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = coordinator.registerRemoteSubsetOwner( + `todos`, + secondOwner, + ) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + coordinator.dispose() + } + }) + + it(`rebinds a live remote Electron follower lease when the same leader replaces its owner`, async () => { + const dbName = `electron-remote-owner-replacement` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.isLeader = () => true + follower.isLeader = () => false + let releaseFirstLoad = (): void => {} + const firstLoadGate = new Promise((resolve) => { + releaseFirstLoad = resolve + }) + const firstOwner = Object.assign( + vi.fn(() => firstLoadGate), + { + unloadSubset: vi.fn(), + onError: vi.fn(), + }, + ) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { limit: 1 } + const followerInternals = follower as unknown as { + channel: BroadcastChannel + } + const originalFollowerPost = followerInternals.channel.postMessage.bind( + followerInternals.channel, + ) + let requestPosts = 0 + followerInternals.channel.postMessage = (message) => { + if ( + (message as { payload?: { type?: string } }).payload?.type === + `rpc:ensureRemoteSubset:req` + ) { + requestPosts++ + } + originalFollowerPost(message) + } + + try { + const initialAcquire = follower.requestEnsureRemoteSubset( + `todos`, + options, + ) + await vi.waitFor(() => expect(firstOwner).toHaveBeenCalledTimes(1)) + unregisterFirst() + unregisterSecond = leader.registerRemoteSubsetOwner(`todos`, secondOwner) + releaseFirstLoad() + await initialAcquire + await vi.waitFor(() => expect(secondOwner).toHaveBeenCalledTimes(1)) + + expect({ + requestPosts, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + }).toEqual({ requestPosts: 1, firstUnloads: 1 }) + await follower.requestReleaseRemoteSubset(`todos`, options) + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + } finally { + releaseFirstLoad() + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`rebinds remote Electron demand when its replacement owner registers after release settlement`, async () => { + const dbName = `electron-remote-owner-settled-replacement` + const leader = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + const follower = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.isLeader = () => true + follower.isLeader = () => false + const firstOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const secondOwner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterFirst = leader.registerRemoteSubsetOwner( + `todos`, + firstOwner, + ) + let unregisterSecond: (() => void) | undefined + const options: LoadSubsetOptions = { offset: 22 } + const leaderInternals = leader as unknown as { + inboundRemoteSubsetAcquisitions: Map> + } + + try { + await follower.requestEnsureRemoteSubset(`todos`, options) + unregisterFirst() + await vi.waitFor(() => + expect(firstOwner.unloadSubset).toHaveBeenCalledTimes(1), + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + + unregisterSecond = leader.registerRemoteSubsetOwner(`todos`, secondOwner) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + firstLoads: firstOwner.mock.calls.length, + firstUnloads: firstOwner.unloadSubset.mock.calls.length, + secondLoads: secondOwner.mock.calls.length, + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ + firstLoads: 1, + firstUnloads: 1, + secondLoads: 1, + inbound: 1, + }) + + await follower.requestReleaseRemoteSubset(`todos`, options) + const [terminal] = + leaderInternals.inboundRemoteSubsetAcquisitions.values() + expect(secondOwner.unloadSubset).toHaveBeenCalledTimes(1) + expect(secondOwner.unloadSubset.mock.calls[0]?.[0]).toBe( + secondOwner.mock.calls[0]?.[0], + ) + expect({ + inbound: leaderInternals.inboundRemoteSubsetAcquisitions.size, + terminalKeys: Object.keys(terminal ?? {}).sort(), + }).toEqual({ + inbound: 1, + terminalKeys: [ + `acquisitionId`, + `collectionId`, + `released`, + `requesterId`, + ], + }) + } finally { + unregisterFirst() + unregisterSecond?.() + leader.dispose() + follower.dispose() + } + }) + + it(`reacquires the same Electron lease after leadership retirement`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-leadership-return`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const internals = coordinator as unknown as { + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => + expect(owner.unloadSubset).toHaveBeenCalledTimes(1), + ) + + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(owner).toHaveBeenCalledTimes(2) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + expect(owner.unloadSubset).toHaveBeenCalledTimes(2) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`starts independent Electron lease replays without sibling head-of-line blocking`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-parallel-replay`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + + type Acquisition = { + collectionId: string + options: TransportedLoadSubsetOptions + acquiredLeaderId: string | null + } + const internals = coordinator as unknown as { + nodeId: string + collections: Map + outboundRemoteSubsetAcquisitions: Map + acquireRemoteSubset: (acquisition: Acquisition) => Promise + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.collections.set(`todos`, { + isLeader: true, + leaderId: internals.nodeId, + }) + const acquisitions = [ + ...internals.outboundRemoteSubsetAcquisitions.values(), + ] + for (const acquisition of acquisitions) { + acquisition.acquiredLeaderId = `retired-electron-leader` + } + let releaseFirstReplay = (): void => {} + const firstReplayGate = new Promise((resolve) => { + releaseFirstReplay = resolve + }) + const starts: Array = [] + internals.acquireRemoteSubset = async (acquisition) => { + starts.push(acquisition.options.offset) + if (acquisition.options.offset === 1) await firstReplayGate + } + + try { + const replay = internals.replayRemoteSubsetAcquisitions(`todos`) + await Promise.resolve() + const startsBeforeFirstFinished = [...starts] + expect(startsBeforeFirstFinished).toEqual([1, 2]) + releaseFirstReplay() + await replay + } finally { + releaseFirstReplay() + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports a failed Electron replay once without self-retrying`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-replay-error`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const replayError = new Error(`replacement Electron owner is not ready`) + const ownerErrors: Array = [] + const owner = Object.assign( + vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(replayError), + { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }, + ) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { limit: 1 } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + type Acquisition = { + acquiredLeaderId: string | null + inFlight: Promise | null + } + const internals = coordinator as unknown as { + nodeId: string + collections: Map + outboundRemoteSubsetAcquisitions: Map + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.collections.set(`todos`, { + isLeader: true, + leaderId: internals.nodeId, + }) + const [acquisition] = internals.outboundRemoteSubsetAcquisitions.values() + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + await vi.waitFor(() => expect(owner.unloadSubset).toHaveBeenCalledTimes(1)) + acquisition!.acquiredLeaderId = `retired-electron-leader` + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + + try { + await internals.replayRemoteSubsetAcquisitions(`todos`) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect({ + attempts: owner.mock.calls.length - 1, + ownerErrors, + acquiredLeaderId: acquisition!.acquiredLeaderId, + inFlight: acquisition!.inFlight, + unhandled, + }).toEqual({ + attempts: 1, + ownerErrors: [replayError], + acquiredLeaderId: `retired-electron-leader`, + inFlight: null, + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports a failed remote Electron follower replay to its local owner once`, async () => { + const dbName = `electron-remote-replay-error` + const coordinator = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => false + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + sendRPC: (collectionId: string, request: unknown) => Promise + outboundRemoteSubsetAcquisitions: Map< + string, + { acquiredLeaderId: string | null; inFlight: Promise | null } + > + } + internals.acquireLeadership = async () => {} + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + heartbeat(`remote-electron-a`, 1) + const ownerErrors: Array = [] + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + internals.sendRPC = vi.fn().mockResolvedValue({ + type: `rpc:ensureRemoteSubset:res`, + rpcId: `initial-electron`, + ok: true, + leaderId: `remote-electron-a`, + }) + const options: LoadSubsetOptions = { limit: 1 } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + const replayError = new Error(`remote Electron replay transport failed`) + const replayRPC = vi.fn().mockRejectedValue(replayError) + internals.sendRPC = replayRPC + heartbeat(`remote-electron-b`, 2) + await vi.waitFor(() => expect(replayRPC).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + const [acquisition] = + internals.outboundRemoteSubsetAcquisitions.values() + expect(acquisition?.inFlight).toBeNull() + }) + + const [acquisition] = internals.outboundRemoteSubsetAcquisitions.values() + expect({ + ownerErrors, + attempts: replayRPC.mock.calls.length, + acquiredLeaderId: acquisition?.acquiredLeaderId, + retained: internals.outboundRemoteSubsetAcquisitions.size, + }).toEqual({ + ownerErrors: [replayError], + attempts: 1, + acquiredLeaderId: `remote-electron-a`, + retained: 1, + }) + } finally { + unregisterOwner() + coordinator.dispose() + } + }) + + it(`ignores a lower-term Electron heartbeat without replaying to its stale leader`, () => { + const dbName = `electron-stale-heartbeat` + const coordinator = new ElectronCollectionCoordinator({ + dbName, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => false + const internals = coordinator as unknown as { + onChannelMessage: (message: unknown) => void + acquireLeadership: () => Promise + collections: Map + replayRemoteSubsetAcquisitions: (collectionId: string) => Promise + } + internals.acquireLeadership = async () => {} + const replay = vi.fn(() => Promise.resolve()) + internals.replayRemoteSubsetAcquisitions = replay + const heartbeat = (leaderId: string, term: number) => + internals.onChannelMessage({ + v: 1, + dbName, + collectionId: `todos`, + senderId: leaderId, + ts: Date.now(), + payload: { + type: `leader:heartbeat`, + term, + leaderId, + latestSeq: 0, + latestRowVersion: 0, + }, + }) + + heartbeat(`remote-electron-b`, 2) + replay.mockClear() + heartbeat(`retired-electron-a`, 1) + const state = internals.collections.get(`todos`) + + expect({ + leaderId: state?.leaderId, + latestTerm: state?.latestTerm, + replayCalls: replay.mock.calls.length, + }).toEqual({ + leaderId: `remote-electron-b`, + latestTerm: 2, + replayCalls: 0, + }) + }) + + it(`reports owner unload rejection while completing sibling Electron cleanup`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-unload-error`, + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const unloadError = new Error(`electron owner unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 1 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 1 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 2 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map< + string, + { release: Promise | null } + > + releaseInboundRemoteSubsetAcquisitions: (collectionId: string) => void + } + + try { + internals.releaseInboundRemoteSubsetAcquisitions(`todos`) + const releases = [ + ...internals.inboundRemoteSubsetAcquisitions.values(), + ].map((acquisition) => acquisition.release!) + const outcomes = await Promise.allSettled(releases) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + outcomes: outcomes.map((outcome) => + outcome.status === `rejected` ? outcome.reason : outcome.status, + ), + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + }).toEqual({ + outcomes: [unloadError, `fulfilled`], + unloadOffsets: [1, 2], + ownerErrors: [unloadError], + unhandled: [], + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`reports local Electron disposal unload rejection once and completes cleanup`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-disposal-unload-error`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const unloadError = new Error(`electron disposal unload failed`) + const ownerErrors: Array = [] + const unloadSubset = vi.fn((options: TransportedLoadSubsetOptions) => + options.offset === 23 ? Promise.reject(unloadError) : undefined, + ) + const owner = Object.assign(vi.fn(), { + unloadSubset, + onError: (error: unknown) => ownerErrors.push(error), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 23 }) + await coordinator.requestEnsureRemoteSubset(`todos`, { offset: 24 }) + const unhandled: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const internals = coordinator as unknown as { + collections: Map + pendingRPCs: Map + remoteSubsetOwners: Map + outboundRemoteSubsetAcquisitions: Map + inboundRemoteSubsetAcquisitions: Map + } + + try { + coordinator.dispose() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + unloadOffsets: unloadSubset.mock.calls.map( + ([options]) => options.offset, + ), + ownerErrors, + unhandled, + collections: internals.collections.size, + pendingRPCs: internals.pendingRPCs.size, + owners: internals.remoteSubsetOwners.size, + outbound: internals.outboundRemoteSubsetAcquisitions.size, + inbound: internals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ + unloadOffsets: [23, 24], + ownerErrors: [unloadError], + unhandled: [], + collections: 0, + pendingRPCs: 0, + owners: 0, + outbound: 0, + inbound: 0, + }) + } finally { + process.off(`unhandledRejection`, onUnhandled) + unregisterOwner() + coordinator.dispose() + } + }) + + it(`rejects duplicate Electron remote-subset owners`, () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-subset-owner-duplicate`, + }) + registerCleanup(() => coordinator.dispose()) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) satisfies RemoteSubsetOwner + const duplicate = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) satisfies RemoteSubsetOwner + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + + try { + expect(() => + coordinator.registerRemoteSubsetOwner(`todos`, duplicate), + ).toThrowError( + expect.objectContaining({ + name: `DuplicateRemoteSubsetOwnerError`, + collectionId: `todos`, + }), + ) + } finally { + unregisterOwner() + } + }) + + it(`does not answer a held inbound request after its Electron leader is disposed`, async () => { + registerCleanup(installImmediatelyGrantedWebLocks()) + let applyEntered = false + let releaseApply = (): void => {} + const applyGate = new Promise((resolve) => { + releaseApply = resolve + }) + const durableTransactions: Array = [] + const leader = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-inbound`, + adapter: { + loadSubset: () => Promise.resolve([]), + applyCommittedTx: async (_collectionId, tx) => { + applyEntered = true + await applyGate + durableTransactions.push(structuredClone(tx)) + }, + ensureIndex: () => Promise.resolve(), + }, + }) + const follower = new ElectronCollectionCoordinator({ + dbName: `electron-dispose-held-inbound`, + }) + // This fixture isolates the leader's post-disposal response boundary. Its + // immediate-grant lock shim cannot model a blocked follower election. + ;( + follower as unknown as { + acquireLeadership: () => Promise + } + ).acquireLeadership = async () => {} + registerCleanup(() => leader.dispose()) + registerCleanup(() => follower.dispose()) + leader.subscribe(`todos`, () => {}) + await waitForLeadership(leader, `todos`) + + const leaderInternals = leader as unknown as { + channel: BroadcastChannel + appliedCommittedTxEnvelopes: Map + inFlightCommittedTxEnvelopes: Map + } + const followerInternals = follower as unknown as { + sendRPCOnce: ( + collectionId: string, + request: ApplyCommittedTxRequest, + ) => Promise + } + const originalPostMessage = leaderInternals.channel.postMessage.bind( + leaderInternals.channel, + ) + let leaderDisposed = false + let postAfterDispose = 0 + leaderInternals.channel.postMessage = (message: unknown) => { + if (leaderDisposed) postAfterDispose++ + originalPostMessage(message) + } + + const outcomePromise = followerInternals + .sendRPCOnce(`todos`, { + type: `rpc:applyCommittedTx:req`, + rpcId: `held-inbound-rpc`, + envelopeId: `held-inbound-envelope`, + tx: { + txId: `held-inbound-tx`, + term: 0, + seq: 0, + rowVersion: 0, + mutations: [], + }, + }) + .then( + (response) => ({ response }), + (error: unknown) => ({ error }), + ) + + try { + await vi.waitFor(() => expect(applyEntered).toBe(true)) + leaderDisposed = true + leader.dispose() + releaseApply() + await vi.waitFor(() => expect(durableTransactions).toHaveLength(1)) + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect({ + postAfterDispose, + completed: leaderInternals.appliedCommittedTxEnvelopes.size, + inFlight: leaderInternals.inFlightCommittedTxEnvelopes.size, + }).toEqual({ postAfterDispose: 0, completed: 0, inFlight: 0 }) + + follower.dispose() + expect(await outcomePromise).toEqual({ + error: new Error(`coordinator disposed`), + }) + } finally { + releaseApply() + leader.dispose() + follower.dispose() + } + }) + it(`returns deterministic timeout errors`, async () => { const neverInvoke: ElectronPersistenceInvoke = async () => await new Promise(() => {}) From 56ac0f46a6703fe58ffa474d1efe286187207f68 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 06:42:20 -0600 Subject: [PATCH 2/4] chore: add persistence coordination changeset --- .changeset/enforce-crash-only-persistence-coordination.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/enforce-crash-only-persistence-coordination.md diff --git a/.changeset/enforce-crash-only-persistence-coordination.md b/.changeset/enforce-crash-only-persistence-coordination.md new file mode 100644 index 0000000000..bf368d4375 --- /dev/null +++ b/.changeset/enforce-crash-only-persistence-coordination.md @@ -0,0 +1,8 @@ +--- +'@tanstack/browser-db-sqlite-persistence': patch +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/electron-db-sqlite-persistence': patch +'@tanstack/db': patch +--- + +Require coordinators to route complete committed transactions through the per-collection persistence owner, with named fail-stop errors for indeterminate commits and durability failures. Add clone-safe remote-subset leases with exact release, recursive wire validation, and matching Browser and Electron coordination. From 5ff207b5aaf3f96e9751c7a4219fbdc2a739dc50 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 17 Sep 2026 11:39:14 -0600 Subject: [PATCH 3/4] fix(sqlite): address coordinator review findings --- .../browser-db-sqlite-persistence/README.md | 7 +- .../src/browser-coordinator.ts | 45 +++-- .../tests/browser-coordinator.test.ts | 42 ++++- packages/db-sqlite-persistence-core/README.md | 29 ++-- .../src/persisted.ts | 38 +++-- .../tests/persisted.test.ts | 157 ++++++++++++++++++ .../electron-db-sqlite-persistence/README.md | 9 +- .../src/electron-coordinator.ts | 45 +++-- .../tests/electron-ipc.test.ts | 44 ++++- 9 files changed, 358 insertions(+), 58 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/README.md b/packages/browser-db-sqlite-persistence/README.md index 08a5222935..e6fcf38ec8 100644 --- a/packages/browser-db-sqlite-persistence/README.md +++ b/packages/browser-db-sqlite-persistence/README.md @@ -112,7 +112,8 @@ The route preserves truncation, row changes, row metadata, collection metadata, and stream position as one `PersistedTx`. It does not feature-detect a partial route or fall back to row-only mutation RPC. A custom coordinator that omits `requestApplyCommittedTx` is rejected while the collection is configured, -before its sync source can publish rows. +before its sync source can publish rows. An effect-free source commit neither +publishes nor consumes a coordinator sequence. Single-tab mode uses the same complete transaction contract. Its `SingleProcessCoordinator` skips election and channel traffic but still routes @@ -144,6 +145,10 @@ exact acquired options once. Leadership loss unloads the retiring owner's live leases, and requesters replay still-live acquisitions against the next leader. Registering a second owner for one collection throws `DuplicateRemoteSubsetOwnerError`; no adapter fallback replaces the owner. +Remote follower transport/admission failures reject and retain demand for its +normal retry without entering the follower's local owner lifecycle. Terminal +release identities expire after the existing RPC dedupe horizon while delayed +duplicates inside that horizon remain idempotent. ## Notes diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index e5c8c419cb..3d3072666f 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -36,6 +36,7 @@ const HEARTBEAT_INTERVAL_MS = 3_000 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 +const RPC_DEDUPE_RETENTION_MS = 60_000 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 @@ -218,6 +219,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina string, RemoteSubsetAcquisition >() + private readonly releasedRemoteSubsetAcquisitionTimes = new Map< + string, + number + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() @@ -431,12 +436,6 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina try { await work acquired = true - } catch (error) { - if (!route.localOwner) { - const owner = this.remoteSubsetOwners.get(acquisition.collectionId) - if (owner) reportRemoteSubsetOwnerError(owner, error) - } - throw error } finally { if (acquisition.inFlight === work) acquisition.inFlight = null const current = this.collections.get(acquisition.collectionId) @@ -456,7 +455,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ) { acquisition.forceReplay = false void this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }) } } @@ -583,6 +582,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } this.remoteSubsetOwners.clear() this.inboundRemoteSubsetAcquisitions.clear() + this.releasedRemoteSubsetAcquisitionTimes.clear() this.appliedEnvelopes.clear() this.inFlightEnvelopes.clear() } @@ -754,7 +754,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquisition.forceReplay = false replays.push( this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }), ) } @@ -1057,6 +1057,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1117,6 +1118,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina terminalRelease: false, release: null, } + this.releasedRemoteSubsetAcquisitionTimes.delete(key) this.inboundRemoteSubsetAcquisitions.set(key, acquisition) let resolveLoad!: () => void let rejectLoad!: (error: unknown) => void @@ -1160,6 +1162,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1167,7 +1170,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ) const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) if (!acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1177,7 +1180,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina acquisition.terminalRelease = true await this.releaseRemoteSubsetAcquisition(acquisition) if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1185,7 +1188,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }) } } else if (`awaitingOwner` in acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1814,13 +1817,31 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private pruneAppliedEnvelopes(): void { // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - 60_000 + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS for (const [key, envelope] of this.appliedEnvelopes) { if (envelope.appliedAt < cutoff) { this.appliedEnvelopes.delete(key) } } } + + private setReleasedRemoteSubsetAcquisition( + key: string, + acquisition: RemoteSubsetAcquisition, + ): void { + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) + } + + private pruneReleasedRemoteSubsetAcquisitions(): void { + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS + for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { + if (releasedAt < cutoff) { + this.releasedRemoteSubsetAcquisitionTimes.delete(key) + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + } } // --------------------------------------------------------------------------- diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 98fbcd751b..5860c5ee23 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -3313,7 +3313,7 @@ describe(`BrowserCollectionCoordinator`, () => { } }) - it(`reports a failed remote Browser follower replay to its local owner once`, async () => { + it(`keeps a failed remote Browser follower replay out of its local owner lifecycle`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) const internals = coordinator as unknown as { @@ -3379,7 +3379,7 @@ describe(`BrowserCollectionCoordinator`, () => { acquiredLeaderId: acquisition?.acquiredLeaderId, retained: internals.outboundRemoteSubsetAcquisitions.size, }).toEqual({ - ownerErrors: [replayError], + ownerErrors: [], attempts: 1, acquiredLeaderId: `remote-browser-a`, retained: 1, @@ -3390,6 +3390,44 @@ describe(`BrowserCollectionCoordinator`, () => { } }) + it(`expires Browser release tombstones after the existing RPC dedupe horizon`, async () => { + const coordinator = createCoordinator() + coordinator.subscribe(`todos`, () => {}) + await flush(50) + expect(coordinator.isLeader(`todos`)).toBe(true) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const now = vi.spyOn(Date, `now`) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map + } + + try { + for (let index = 0; index < 8; index++) { + now.mockReturnValue(index * 60_001) + const options = { offset: index } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + } + + expect({ + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + retained: internals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ loads: 8, unloads: 8, retained: 1 }) + } finally { + now.mockRestore() + unregisterOwner() + coordinator.dispose() + } + }) + it(`ignores a lower-term Browser heartbeat without replaying to its stale leader`, async () => { const coordinator = createCoordinator() coordinator.subscribe(`todos`, () => {}) diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 87bdc89371..a1da823079 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -94,8 +94,9 @@ effects is routed in full to the persistence adapter owned by that collection. The `PersistedTx` includes any truncate, row mutations, row metadata mutations, collection metadata mutations, and stream position. Multiprocess coordinators must preserve the complete transaction when they forward it to the elected -writer. This contract does not decide whether an effect-free source commit -should produce a persistence call. +writer. An effect-free source commit does not allocate a stream position, +publish an invalidation, or call persistence; the next durable transaction +retains the next coordinator-owned sequence. This is a required invariant, not a capability to detect at commit time. The TypeScript interface requires the method, and `persistedCollectionOptions(...)` @@ -162,18 +163,24 @@ acquisition; `ReleaseRemoteSubsetRequest.acquisitionId` must carry that same identity so the elected owner releases the matching lease. The owner also provides a required `onError(error)` callback. A load or unload -throw, or a runtime thenable rejection from either operation, is reported -through that callback with the original value before the awaited request -rejects. Retirement and disposal observe and report the same failure while -continuing cleanup of sibling leases; they do not retry or silently recover -the failed lifecycle. +throw by the actual elected owner, or a runtime thenable rejection from either +operation, is reported through that callback with the original value before +the awaited request rejects. A follower transport or remote-owner admission +failure rejects the acquisition and remains eligible for the existing demand +retry; it is not a failure of that follower's local owner lifecycle. Retirement +and disposal observe and report actual owner failures while continuing cleanup +of sibling leases; they do not retry or silently recover the failed lifecycle. Only one remote subset owner may be registered per collection in a coordinator. A second live registration throws `DuplicateRemoteSubsetOwnerError` instead of -replacing the first. When a leader loses ownership or is disposed, it unloads -every transferred lease. A requester that still owns a lease replays that same -acquisition when the next leader is observed. There is no adapter fallback or -direct-writer escape hatch for missing ownership. +replacing the first, and startup reports the named failure before readiness +rather than leaving a rejected registration promise unobserved. Released +acquisition identities remain as dedupe tombstones for the coordinator's +existing RPC dedupe horizon, then expire during later subset traffic. When a +leader loses ownership or is disposed, it unloads every transferred lease. A +requester that still owns a lease replays that same acquisition when the next +leader is observed. There is no adapter fallback or direct-writer escape hatch +for missing ownership. The named-error and exact-path guarantee covers values the boundary can identify with standard JavaScript reflection. A fully transparent `Proxy` is diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index f47999dfa9..c8e3d9c35f 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1855,28 +1855,16 @@ class PersistedCollectionRuntime< return } - const streamPosition = this.nextLocalStreamPosition() - if ( !transaction.truncate && transaction.operations.length === 0 && transaction.rowMetadataWrites.size === 0 && transaction.collectionMetadataWrites.size === 0 ) { - this.publishTxCommittedEvent( - this.createTxCommittedPayload({ - term: streamPosition.term, - seq: streamPosition.seq, - txId: safeRandomUUID(), - latestRowVersion: streamPosition.rowVersion, - changedRows: [], - deletedKeys: [], - requiresFullReload: true, - }), - ) return } + const streamPosition = this.nextLocalStreamPosition() const tx = this.createPersistedTxFromOperations(transaction, streamPosition) let response: ApplyCommittedTxResponse try { @@ -2700,7 +2688,13 @@ function createWrappedSyncConfig< const getOpenTransaction = () => transactionStack[transactionStack.length - 1] let fullStartPromise: Promise | null = null + let sourceResultPromise: Promise | null = null + let resolveSourceResultAssigned!: () => void + const sourceResultAssigned = new Promise((resolve) => { + resolveSourceResultAssigned = resolve + }) const startupState = { cleanedUp: false } + const isCleanedUp = () => startupState.cleanedUp const acquisitions = new Map() runtime.setSyncControls({ begin: params.begin, @@ -2719,8 +2713,16 @@ function createWrappedSyncConfig< markReady: () => { if (startupState.cleanedUp) return void (fullStartPromise ?? runtime.ensureStarted()) - .then(() => { - if (startupState.cleanedUp) return + .then(async () => { + if (isCleanedUp()) return + await sourceResultAssigned + try { + await sourceResultPromise + } catch (error) { + runtime.reportSyncError(error) + return + } + if (isCleanedUp()) return params.markReady() }) .catch((error) => { @@ -3002,7 +3004,7 @@ function createWrappedSyncConfig< let sourceResult: SyncConfigRes = {} fullStartPromise = runtime.ensureStarted() - const sourceResultPromise = (async () => { + sourceResultPromise = (async () => { await runtime.ensureStartupMetadataLoaded() if (startupState.cleanedUp) { @@ -3033,6 +3035,10 @@ function createWrappedSyncConfig< } return sourceResult })() + resolveSourceResultAssigned() + void sourceResultPromise.catch((error) => { + runtime.reportSyncError(error) + }) return { cleanup: () => { diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0862daa2ef..c893975092 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1063,6 +1063,69 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`surfaces duplicate remote-subset owner registration through the collection lifecycle`, async () => { + const coordinator = new SingleProcessCoordinator() + const unhandled: Array = [] + const sourceLoads: Array = [] + const onUnhandled = (error: unknown) => unhandled.push(error) + process.on(`unhandledRejection`, onUnhandled) + const create = (label: string) => + createCollection( + persistedCollectionOptions({ + id: `duplicate-owner-lifecycle`, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + sourceLoads.push(label) + return true + }, + unloadSubset: () => {}, + } + }, + }, + persistence: { adapter: createRecordingAdapter(), coordinator }, + }), + ) + const first = create(`first`) + let duplicate: ReturnType | undefined + + try { + first.startSyncImmediate() + await first.stateWhenReady() + await vi.waitFor(() => + expect( + ( + coordinator as unknown as { + remoteSubsetOwners: Map + } + ).remoteSubsetOwners.size, + ).toBe(1), + ) + duplicate = create(`duplicate`) + duplicate.startSyncImmediate() + const readinessError = await duplicate.stateWhenReady().then( + () => undefined, + (error: unknown) => error, + ) + await flushAsyncWork() + + expect(readinessError).toMatchObject({ + name: `DuplicateRemoteSubsetOwnerError`, + collectionId: `duplicate-owner-lifecycle`, + }) + expect(duplicate.status).toBe(`error`) + expect(sourceLoads).toEqual([]) + expect(unhandled).toEqual([]) + } finally { + process.off(`unhandledRejection`, onUnhandled) + await duplicate?.cleanup() + await first.cleanup() + } + }) + it(`resolves persistence per collection and forwards schemaVersion`, () => { const baseAdapter = createNoopAdapter() const syncAdapter = createNoopAdapter() @@ -1337,6 +1400,100 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`does not publish or allocate a sequence for an empty external source transaction`, async () => { + const adapter = Object.assign(createRecordingAdapter(), { + getStreamPosition: () => + Promise.resolve({ + latestTerm: 1, + latestSeq: 10, + latestRowVersion: 10, + }), + }) + let subscriber: ((message: ProtocolEnvelope) => void) | undefined + const published: Array> = [] + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `empty-source-follower`, + subscribe: (_collectionId, onMessage) => { + subscriber = onMessage + return () => { + subscriber = undefined + } + }, + publish: (_collectionId, message) => published.push(message), + isLeader: () => false, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestEnsureRemoteSubset: async () => {}, + requestReleaseRemoteSubset: async () => {}, + registerRemoteSubsetOwner: () => () => {}, + requestApplyCommittedTx: (_collectionId, tx) => + Promise.resolve({ + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }), + } + let sourceBegin: (() => void) | undefined + let sourceCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `empty-source-sequence`, + getKey: (row) => row.id, + sync: { + sync: ({ begin, commit, markReady }) => { + sourceBegin = begin + sourceCommit = commit + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + try { + await collection.stateWhenReady() + sourceBegin?.() + await sourceCommit?.() + await flushAsyncWork() + + expect(published).toEqual([]) + + subscriber?.({ + v: 1, + dbName: `empty-source-sequence`, + collectionId: `empty-source-sequence`, + senderId: `elected-owner`, + ts: Date.now(), + payload: { + type: `tx:committed`, + term: 1, + seq: 11, + txId: `first-real-coordinator-sequence`, + latestRowVersion: 11, + requiresFullReload: false, + changedRows: [ + { + key: `kept`, + value: { id: `kept`, title: `First real write` }, + }, + ], + deletedKeys: [], + }, + }) + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`kept`))).toEqual({ + id: `kept`, + title: `First real write`, + }) + } finally { + await collection.cleanup() + } + }) + it(`persists a wrapped sync transaction when abort follows application`, async () => { const adapter = createRecordingAdapter() let remoteBegin: (() => void) | undefined diff --git a/packages/electron-db-sqlite-persistence/README.md b/packages/electron-db-sqlite-persistence/README.md index c53dd7493e..cb3e6c56a8 100644 --- a/packages/electron-db-sqlite-persistence/README.md +++ b/packages/electron-db-sqlite-persistence/README.md @@ -101,7 +101,8 @@ Truncation, row changes, row metadata, collection metadata, and stream position stay in the same transaction. Electron does not feature-detect a partial route or fall back to row-only mutation RPC. A custom coordinator that omits `requestApplyCommittedTx` is rejected while the collection is configured, -before its sync source can publish rows. +before its sync source can publish rows. An effect-free source commit neither +publishes nor consumes a coordinator sequence. If a mutating RPC loses its response, Electron coordination replays it only while the requester still knows the same non-null leader id and term. An @@ -124,7 +125,11 @@ remain independent, and release unloads the exact transported options once. Leadership loss unloads the retiring renderer's leases; still-live requester leases replay against the next leader. A second owner registration for one collection throws `DuplicateRemoteSubsetOwnerError`, and there is no adapter -fallback for a missing owner. +fallback for a missing owner. Remote follower transport/admission failures +reject and retain demand for its normal retry without entering the follower's +local owner lifecycle. Terminal release identities expire after the existing +RPC dedupe horizon while delayed duplicates inside that horizon remain +idempotent. ## Notes diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 1c3ea73f0a..5614bc410e 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -37,6 +37,7 @@ const HEARTBEAT_INTERVAL_MS = 3_000 const RPC_TIMEOUT_MS = 10_000 const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 +const RPC_DEDUPE_RETENTION_MS = 60_000 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 @@ -192,6 +193,10 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin string, RemoteSubsetAcquisition >() + private readonly releasedRemoteSubsetAcquisitionTimes = new Map< + string, + number + >() private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() @@ -417,12 +422,6 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin try { await work acquired = true - } catch (error) { - if (!routedToLocalOwner) { - const owner = this.remoteSubsetOwners.get(acquisition.collectionId) - if (owner) reportRemoteSubsetOwnerError(owner, error) - } - throw error } finally { if (acquisition.inFlight === work) acquisition.inFlight = null const current = this.collections.get(acquisition.collectionId) @@ -442,7 +441,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ) { acquisition.forceReplay = false void this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }) } } @@ -569,6 +568,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } this.remoteSubsetOwners.clear() this.inboundRemoteSubsetAcquisitions.clear() + this.releasedRemoteSubsetAcquisitionTimes.clear() this.appliedEnvelopeIds.clear() this.inFlightLocalMutationEnvelopes.clear() this.appliedCommittedTxEnvelopes.clear() @@ -742,7 +742,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisition.forceReplay = false replays.push( this.acquireRemoteSubset(acquisition).catch(() => { - // Failure is already reported; only new demand or ownership change retries. + // Demand stays retained; only new demand or ownership change retries. }), ) } @@ -1045,6 +1045,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1105,6 +1106,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin terminalRelease: false, release: null, } + this.releasedRemoteSubsetAcquisitionTimes.delete(key) this.inboundRemoteSubsetAcquisitions.set(key, acquisition) let resolveLoad!: () => void let rejectLoad!: (error: unknown) => void @@ -1148,6 +1150,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin request: Extract, requesterId: string, ): Promise { + this.pruneReleasedRemoteSubsetAcquisitions() const key = inboundRemoteSubsetAcquisitionKey( collectionId, requesterId, @@ -1155,7 +1158,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin ) const acquisition = this.inboundRemoteSubsetAcquisitions.get(key) if (!acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1165,7 +1168,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin acquisition.terminalRelease = true await this.releaseRemoteSubsetAcquisition(acquisition) if (this.inboundRemoteSubsetAcquisitions.get(key) === acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1173,7 +1176,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin }) } } else if (`awaitingOwner` in acquisition) { - this.inboundRemoteSubsetAcquisitions.set(key, { + this.setReleasedRemoteSubsetAcquisition(key, { collectionId, requesterId, acquisitionId: request.acquisitionId, @@ -1755,7 +1758,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup - const cutoff = Date.now() - 60_000 + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS for (const [id, envelope] of this.appliedEnvelopeIds) { if (envelope.appliedAt < cutoff) { this.appliedEnvelopeIds.delete(id) @@ -1767,6 +1770,24 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } } } + + private setReleasedRemoteSubsetAcquisition( + key: string, + acquisition: RemoteSubsetAcquisition, + ): void { + this.inboundRemoteSubsetAcquisitions.set(key, acquisition) + this.releasedRemoteSubsetAcquisitionTimes.set(key, Date.now()) + } + + private pruneReleasedRemoteSubsetAcquisitions(): void { + const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS + for (const [key, releasedAt] of this.releasedRemoteSubsetAcquisitionTimes) { + if (releasedAt < cutoff) { + this.releasedRemoteSubsetAcquisitionTimes.delete(key) + this.inboundRemoteSubsetAcquisitions.delete(key) + } + } + } } // --------------------------------------------------------------------------- diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index c2913a0833..25c5b3e854 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -2876,7 +2876,7 @@ describe(`electron sqlite persistence bridge`, () => { } }) - it(`reports a failed remote Electron follower replay to its local owner once`, async () => { + it(`keeps a failed remote Electron follower replay out of its local owner lifecycle`, async () => { const dbName = `electron-remote-replay-error` const coordinator = new ElectronCollectionCoordinator({ dbName, @@ -2947,7 +2947,7 @@ describe(`electron sqlite persistence bridge`, () => { acquiredLeaderId: acquisition?.acquiredLeaderId, retained: internals.outboundRemoteSubsetAcquisitions.size, }).toEqual({ - ownerErrors: [replayError], + ownerErrors: [], attempts: 1, acquiredLeaderId: `remote-electron-a`, retained: 1, @@ -2958,6 +2958,46 @@ describe(`electron sqlite persistence bridge`, () => { } }) + it(`expires Electron release tombstones after the existing RPC dedupe horizon`, async () => { + const coordinator = new ElectronCollectionCoordinator({ + dbName: `electron-release-tombstone-expiry`, + adapter: createElectronCoordinatorTestAdapter(), + }) + registerCleanup(() => coordinator.dispose()) + coordinator.isLeader = () => true + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const now = vi.spyOn(Date, `now`) + const internals = coordinator as unknown as { + inboundRemoteSubsetAcquisitions: Map + } + + try { + for (let index = 0; index < 8; index++) { + now.mockReturnValue(index * 60_001) + const options = { offset: index } + await coordinator.requestEnsureRemoteSubset(`todos`, options) + await coordinator.requestReleaseRemoteSubset(`todos`, options) + } + + expect({ + loads: owner.mock.calls.length, + unloads: owner.unloadSubset.mock.calls.length, + retained: internals.inboundRemoteSubsetAcquisitions.size, + }).toEqual({ loads: 8, unloads: 8, retained: 1 }) + } finally { + now.mockRestore() + unregisterOwner() + coordinator.dispose() + } + }) + it(`ignores a lower-term Electron heartbeat without replaying to its stale leader`, () => { const dbName = `electron-stale-heartbeat` const coordinator = new ElectronCollectionCoordinator({ From 2934f15e6ee7929316426f449cf146abec1efb7f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 18 Sep 2026 17:10:53 -0600 Subject: [PATCH 4/4] ci: trigger PR checks