diff --git a/.changeset/fix-persisted-lifecycle-durability.md b/.changeset/fix-persisted-lifecycle-durability.md new file mode 100644 index 0000000000..6b98d61640 --- /dev/null +++ b/.changeset/fix-persisted-lifecycle-durability.md @@ -0,0 +1,6 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/electric-db-collection': patch +--- + +Fail-stop persisted collections when hydration or durability fails, while preserving lifecycle ownership across buffered and recovered sync work. Electric collections now surface these failures through the collection error state instead of continuing from an incomplete durable baseline. diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 1ed225957f..239f863436 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -81,6 +81,12 @@ while still handling per-collection schema versions correctly. - `InvalidPersistedStorageKeyError` - `InvalidPersistedStorageKeyEncodingError` - `PersistenceUnavailableError` +- `PersistenceDurabilityError` + +Adapter commit failures reject the applied receipt and become the collection's +terminal sync error as `PersistenceDurabilityError`. The error preserves its +original `cause` and any available string `code` and `path`. Hydration and +source errors remain distinct and are not wrapped as durability errors. ## Typical usage (via runtime wrappers) diff --git a/packages/db-sqlite-persistence-core/src/errors.ts b/packages/db-sqlite-persistence-core/src/errors.ts index f0ffc330eb..8af8c525d8 100644 --- a/packages/db-sqlite-persistence-core/src/errors.ts +++ b/packages/db-sqlite-persistence-core/src/errors.ts @@ -71,3 +71,27 @@ export class PersistenceUnavailableError extends PersistedCollectionCoreError { this.name = `PersistenceUnavailableError` } } + +export class PersistenceDurabilityError extends PersistedCollectionCoreError { + readonly code: string | undefined + readonly path: string + + constructor(cause: unknown, fallbackPath: string) { + const causeMessage = + cause instanceof Error ? cause.message : `Unknown persistence failure` + const causeRecord = + typeof cause === `object` && cause !== null + ? (cause as Record) + : undefined + const code = + typeof causeRecord?.code === `string` ? causeRecord.code : undefined + const path = + typeof causeRecord?.path === `string` ? causeRecord.path : fallbackPath + + super(`Persistence durability failed at ${path}: ${causeMessage}`) + this.name = `PersistenceDurabilityError` + this.cause = cause + this.code = code + this.path = path + } +} diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..ab1ca90649 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -12,6 +12,7 @@ import { InvalidPersistedStorageKeyError, InvalidPersistenceAdapterError, InvalidSyncConfigError, + PersistenceDurabilityError, } from './errors' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { @@ -440,6 +441,7 @@ type SyncControlFns = { commit: ((signal?: AbortSignal) => SyncAppliedReceipt) | null truncate: (() => void) | null metadata: SyncMetadataApi | null + markError: ((error: unknown) => void) | null } /** @@ -590,16 +592,22 @@ type BufferedSyncTransaction = { > truncate: boolean internal: boolean + lifecycleGeneration: number + beginOptions?: { immediate?: boolean } signal?: AbortSignal + applyToCollection: () => SyncAppliedReceipt + shouldFailStopOnAbort?: () => boolean resolveApplied?: () => void rejectApplied?: (error: unknown) => void } -type OpenSyncTransaction< - T extends object, - TKey extends string | number, -> = BufferedSyncTransaction & { +type OpenSyncTransaction = Omit< + BufferedSyncTransaction, + `applyToCollection` | `shouldFailStopOnAbort` +> & { queuedBecauseHydrating: boolean + hasDependentSuccessor: boolean + terminalFailure?: { error: unknown } } type SyncWriteNormalization = { @@ -618,13 +626,40 @@ type SyncWriteNormalization = { class ApplyMutex { private queue: Promise = Promise.resolve() + private pending = 0 + + run(task: () => Promise): Promise { + const runImmediately = this.pending === 0 + this.pending++ + + if (runImmediately) { + let releaseReservation!: () => void + this.queue = new Promise((resolve) => { + releaseReservation = resolve + }) + let taskPromise: Promise + try { + taskPromise = Promise.resolve(task()) + } catch (error) { + taskPromise = Promise.reject(error) + } + const markComplete = () => { + this.pending-- + releaseReservation() + } + void taskPromise.then(markComplete, markComplete) + return taskPromise + } - async run(task: () => Promise): Promise { const taskPromise = this.queue.then(() => task()) this.queue = taskPromise.then( () => undefined, () => undefined, ) + const markComplete = () => { + this.pending-- + } + void taskPromise.then(markComplete, markComplete) return taskPromise } } @@ -790,7 +825,10 @@ class PersistedCollectionRuntime< private readonly queuedHydrationTransactions: Array< BufferedSyncTransaction > = [] - private readonly queuedTxCommitted: Array = [] + private readonly queuedTxCommitted: Array<{ + txCommitted: TxCommitted + lifecycleGeneration: number + }> = [] private readonly requestIds = new WeakMap() private collection: Collection | null = @@ -801,6 +839,7 @@ class PersistedCollectionRuntime< commit: null, truncate: null, metadata: null, + markError: null, } private started = false private startupMetadataPromise: Promise | null = null @@ -811,6 +850,9 @@ class PersistedCollectionRuntime< private appliedReceiptSequence = 0 private readonly pendingAppliedReceipts = new Map>() private hydratingGeneration: number | null = null + private terminalFailure: + | { lifecycleGeneration: number; error: unknown } + | undefined private coordinatorUnsubscribe: (() => void) | null = null private indexAddedUnsubscribe: (() => void) | null = null private indexRemovedUnsubscribe: (() => void) | null = null @@ -863,13 +905,14 @@ class PersistedCollectionRuntime< ) } - clearSyncControls(): void { + private clearSyncControls(): void { this.syncControls = { begin: null, write: null, commit: null, truncate: null, metadata: null, + markError: null, } } @@ -877,6 +920,53 @@ class PersistedCollectionRuntime< return this.hydratingGeneration === this.lifecycleGeneration } + getCurrentTerminalFailure(): { error: unknown } | undefined { + const failure = this.terminalFailure + return failure?.lifecycleGeneration === this.lifecycleGeneration + ? failure + : undefined + } + + getLifecycleGeneration(): number { + return this.lifecycleGeneration + } + + private throwIfLifecycleReplaced(lifecycleGeneration: number): void { + if (lifecycleGeneration !== this.lifecycleGeneration) { + throw new SyncTransactionAbortedError() + } + } + + private throwIfTerminal(): void { + const failure = this.getCurrentTerminalFailure() + if (failure) throw failure.error + } + + private markTerminalFailure( + error: unknown, + lifecycleGeneration = this.lifecycleGeneration, + ): unknown { + if (lifecycleGeneration !== this.lifecycleGeneration) { + return error + } + + const existing = this.getCurrentTerminalFailure() + if (existing) { + return existing.error + } + + this.terminalFailure = { lifecycleGeneration, error } + this.pendingRemoteSubsetEnsures.clear() + this.queuedTxCommitted.length = 0 + if (this.remoteEnsureRetryTimer !== null) { + clearTimeout(this.remoteEnsureRetryTimer) + this.remoteEnsureRetryTimer = null + } + this.rejectQueuedHydrationTransactions(error) + this.syncControls.markError?.(error) + return error + } + isApplyingInternally(): boolean { return this.internalApplyDepth > 0 } @@ -900,16 +990,22 @@ class PersistedCollectionRuntime< } async ensureStarted(): Promise { + this.throwIfTerminal() if (this.startPromise) { return this.startPromise } const lifecycleGeneration = this.lifecycleGeneration - this.startPromise = this.startInternal(lifecycleGeneration) + this.startPromise = this.startInternal(lifecycleGeneration).catch( + (error) => { + throw this.markTerminalFailure(error, lifecycleGeneration) + }, + ) return this.startPromise } ensureResumeBaselineHydrated(): Promise { + this.throwIfTerminal() if (this.resumeBaselinePromise) { return this.resumeBaselinePromise } @@ -921,7 +1017,9 @@ class PersistedCollectionRuntime< if (this.syncMode !== `on-demand`) return await this.hydrateBaseline(lifecycleGeneration) - })() + })().catch((error) => { + throw this.markTerminalFailure(error, lifecycleGeneration) + }) return this.resumeBaselinePromise } @@ -943,13 +1041,17 @@ class PersistedCollectionRuntime< } async ensureStartupMetadataLoaded(): Promise { + this.throwIfTerminal() if (this.startupMetadataPromise) { return this.startupMetadataPromise } const lifecycleGeneration = this.lifecycleGeneration - this.startupMetadataPromise = - this.loadStartupMetadataInternal(lifecycleGeneration) + this.startupMetadataPromise = this.loadStartupMetadataInternal( + lifecycleGeneration, + ).catch((error) => { + throw this.markTerminalFailure(error, lifecycleGeneration) + }) return this.startupMetadataPromise } @@ -992,7 +1094,8 @@ class PersistedCollectionRuntime< const collectionMetadata = await this.loadCollectionMetadataSnapshot() if (lifecycleGeneration !== this.lifecycleGeneration) return - this.replaceCollectionMetadataSnapshot(collectionMetadata) + const applied = this.replaceCollectionMetadataSnapshot(collectionMetadata) + if (applied !== true) await applied } private async loadCollectionMetadataSnapshot(): Promise< @@ -1007,13 +1110,13 @@ class PersistedCollectionRuntime< private replaceCollectionMetadataSnapshot( collectionMetadata: Array<{ key: string; value: unknown }>, - ): void { + ): SyncAppliedReceipt { if ( !this.syncControls.begin || !this.syncControls.commit || !this.syncControls.metadata ) { - return + return true } const nextMetadata = new Map( @@ -1023,7 +1126,7 @@ class PersistedCollectionRuntime< .list() .map(({ key }) => key) - this.withInternalApply(() => { + return this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) currentKeys.forEach((key) => { @@ -1036,7 +1139,7 @@ class PersistedCollectionRuntime< this.syncControls.metadata?.collection.set(key, value) }) - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) } @@ -1044,18 +1147,23 @@ class PersistedCollectionRuntime< options: LoadSubsetOptions, upstreamLoadSubset?: LoadSubsetFn, ): Promise { + this.throwIfTerminal() const lifecycleGeneration = this.lifecycleGeneration this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, - lifecycleGeneration, - }), - ) - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.waitForAppliedReceiptsAfter(appliedCursor) + try { + await this.applyMutex.run(() => + this.hydrateSubsetUnsafe(options, { + requestRemoteEnsure: this.mode === `sync-present`, + lifecycleGeneration, + }), + ) + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.waitForAppliedReceiptsAfter(appliedCursor) + } catch (error) { + throw this.markTerminalFailure(error, lifecycleGeneration) + } if (upstreamLoadSubset) { try { @@ -1083,12 +1191,14 @@ class PersistedCollectionRuntime< options: LoadSubsetOptions, upstreamUnloadSubset?: (options: LoadSubsetOptions) => void, ): void { - this.activeSubsets.delete(this.getSubsetKey(options)) - this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) + const subsetKey = this.getSubsetKey(options) + this.activeSubsets.delete(subsetKey) + this.pendingRemoteSubsetEnsures.delete(subsetKey) upstreamUnloadSubset?.(options) } async forceReloadSubset(options: LoadSubsetOptions): Promise { + this.throwIfTerminal() const lifecycleGeneration = this.lifecycleGeneration // A one-shot refresh does not acquire an enduring subscription lease. await this.applyMutex.run(() => @@ -1102,14 +1212,21 @@ class PersistedCollectionRuntime< queueHydrationBufferedTransaction( transaction: BufferedSyncTransaction, ): void { + const failure = this.getCurrentTerminalFailure() + if (failure) { + transaction.rejectApplied?.(failure.error) + return + } this.queuedHydrationTransactions.push(transaction) } - async persistAndBroadcastExternalSyncTransaction( + applyHydrationBufferedTransaction( transaction: BufferedSyncTransaction, ): Promise { - await this.applyMutex.run(() => - this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction), + const failure = this.getCurrentTerminalFailure() + if (failure) return Promise.reject(failure.error) + return this.applyMutex.run(() => + this.applyBufferedSyncTransactionUnsafe(transaction), ) } @@ -1176,14 +1293,21 @@ class PersistedCollectionRuntime< async persistAndConfirmCollectionMutations( mutations: Array>, + lifecycleGeneration = this.lifecycleGeneration, ): Promise { + this.throwIfTerminal() + this.throwIfLifecycleReplaced(lifecycleGeneration) if (mutations.length === 0) { return } await this.applyMutex.run(async () => { - const acceptedMutationIds = - await this.persistCollectionMutationsUnsafe(mutations) + this.throwIfLifecycleReplaced(lifecycleGeneration) + const acceptedMutationIds = await this.persistCollectionMutationsUnsafe( + mutations, + lifecycleGeneration, + ) + this.throwIfLifecycleReplaced(lifecycleGeneration) const acceptedMutationIdSet = new Set(acceptedMutationIds) const acceptedMutations = mutations.filter((mutation) => acceptedMutationIdSet.has(mutation.mutationId), @@ -1195,13 +1319,18 @@ class PersistedCollectionRuntime< ) } - this.confirmMutationsSyncUnsafe(acceptedMutations) + try { + await this.confirmMutationsSyncUnsafe(acceptedMutations) + } catch (error) { + throw this.markTerminalFailure(error, lifecycleGeneration) + } }) } async acceptTransactionMutations(transaction: { mutations: Array>> }): Promise { + this.throwIfTerminal() const collectionMutations = this.filterMutationsForCollection( transaction.mutations, ) @@ -1247,6 +1376,7 @@ class PersistedCollectionRuntime< this.startupMetadataPromise = null this.startPromise = null this.resumeBaselinePromise = null + this.terminalFailure = undefined } private withInternalApply(task: () => TResult): TResult { @@ -1292,6 +1422,7 @@ class PersistedCollectionRuntime< async scanPersistedRows( options?: PersistedRowScanOptions, ): Promise>> { + this.throwIfTerminal() return this.applyMutex.run(() => this.scanPersistedRowsUnsafe(options)) } @@ -1302,38 +1433,51 @@ class PersistedCollectionRuntime< lifecycleGeneration: number }, ): Promise { - this.hydratingGeneration = config.lifecycleGeneration try { - const rows = await this.loadSubsetRowsUnsafe(options) - if (config.lifecycleGeneration !== this.lifecycleGeneration) return + this.throwIfTerminal() + this.hydratingGeneration = config.lifecycleGeneration + try { + const rows = await this.loadSubsetRowsUnsafe(options) + if (config.lifecycleGeneration !== this.lifecycleGeneration) return + + const applied = this.applyRowsToCollection(rows) + if (applied !== true) await applied + } finally { + if (this.hydratingGeneration === config.lifecycleGeneration) { + this.hydratingGeneration = null + } + } - this.applyRowsToCollection(rows) - } finally { - if (this.hydratingGeneration === config.lifecycleGeneration) { - this.hydratingGeneration = null + await this.flushQueuedHydrationTransactionsUnsafe() + await this.flushQueuedTxCommittedUnsafe() + + if (config.requestRemoteEnsure) { + this.queueRemoteSubsetEnsure(options) } + } catch (error) { + throw this.markTerminalFailure(error, config.lifecycleGeneration) } + } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() - - if (config.requestRemoteEnsure) { - this.queueRemoteSubsetEnsure(options) + private rejectQueuedHydrationTransactions(error: unknown): void { + for (const transaction of this.queuedHydrationTransactions) { + transaction.rejectApplied?.(error) } + this.queuedHydrationTransactions.length = 0 } private applyRowsToCollection( rows: Array<{ key: TKey; value: T; metadata?: unknown }>, - ): void { + ): SyncAppliedReceipt { if ( !this.syncControls.begin || !this.syncControls.write || !this.syncControls.commit ) { - return + return true } - this.withInternalApply(() => { + return this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) for (const row of rows) { @@ -1347,21 +1491,21 @@ class PersistedCollectionRuntime< }) } - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) } private replaceCollectionSnapshot( rows: Array<{ key: TKey; value: T; metadata?: unknown }>, collectionMetadata: Array<{ key: string; value: unknown }>, - ): void { + ): SyncAppliedReceipt { if ( !this.syncControls.begin || !this.syncControls.write || !this.syncControls.commit || !this.syncControls.metadata ) { - return + return true } const nextMetadata = new Map( @@ -1371,7 +1515,7 @@ class PersistedCollectionRuntime< .list() .map(({ key }) => key) - this.withInternalApply(() => { + return this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) this.syncControls.truncate?.() @@ -1393,7 +1537,7 @@ class PersistedCollectionRuntime< this.syncControls.metadata?.collection.set(key, value) }) - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) } @@ -1419,73 +1563,44 @@ class PersistedCollectionRuntime< private async applyBufferedSyncTransactionUnsafe( transaction: BufferedSyncTransaction, ): Promise { + this.throwIfTerminal() + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) if (transaction.signal?.aborted) { - transaction.rejectApplied?.(new SyncTransactionAbortedError()) - return - } - - const { begin, write, commit, truncate, metadata } = this.syncControls - if (!begin || !write || !commit) { - transaction.rejectApplied?.(new SyncTransactionAbortedError()) - return - } - - const applyToCollection = (): SyncAppliedReceipt => { - begin() - - if (transaction.truncate) { - truncate?.() - } - - for (const operation of transaction.operations) { - if (operation.type === `delete`) { - write({ - type: `delete`, - key: operation.key, - }) - } else { - write({ - type: `update`, - value: operation.value, - metadata: operation.metadata, - }) - } - } - - for (const [key, metadataWrite] of transaction.rowMetadataWrites) { - if (metadataWrite.type === `delete`) { - metadata?.row.delete(key) - } else { - metadata?.row.set(key, metadataWrite.value) - } + const error = new SyncTransactionAbortedError() + transaction.rejectApplied?.(error) + if (transaction.shouldFailStopOnAbort?.()) { + throw this.markTerminalFailure(error, transaction.lifecycleGeneration) } - - for (const [key, metadataWrite] of transaction.collectionMetadataWrites) { - if (metadataWrite.type === `delete`) { - metadata?.collection.delete(key) - } else { - metadata?.collection.set(key, metadataWrite.value) - } + if (!transaction.rejectApplied) { + throw error } - - return commit(transaction.signal) + return } try { const applied = transaction.internal - ? this.withInternalApply(applyToCollection) - : applyToCollection() + ? this.withInternalApply(transaction.applyToCollection) + : transaction.applyToCollection() if (applied !== true) { await applied } + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) if (!transaction.internal) { await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) } transaction.resolveApplied?.() } catch (error) { - transaction.rejectApplied?.(error) - throw error + const aborted = + transaction.signal?.aborted || + error instanceof SyncTransactionAbortedError + const terminalError = aborted + ? transaction.shouldFailStopOnAbort?.() + ? this.markTerminalFailure(error, transaction.lifecycleGeneration) + : error + : this.markTerminalFailure(error, transaction.lifecycleGeneration) + transaction.rejectApplied?.(terminalError) + throw terminalError } } @@ -1496,6 +1611,8 @@ class PersistedCollectionRuntime< return } + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) + const streamPosition = this.nextLocalStreamPosition() if ( @@ -1504,6 +1621,7 @@ class PersistedCollectionRuntime< transaction.rowMetadataWrites.size === 0 && transaction.collectionMetadataWrites.size === 0 ) { + if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return this.publishTxCommittedEvent( this.createTxCommittedPayload({ term: streamPosition.term, @@ -1520,7 +1638,8 @@ class PersistedCollectionRuntime< const tx = this.createPersistedTxFromOperations(transaction, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + await this.applyCommittedTx(tx, transaction.lifecycleGeneration) + if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return this.publishTxCommittedEvent( this.createTxCommittedPayload({ term: tx.term, @@ -1543,6 +1662,25 @@ class PersistedCollectionRuntime< ) } + private async applyCommittedTx( + tx: PersistedTx, + lifecycleGeneration: number, + ): Promise { + try { + this.throwIfTerminal() + await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + } catch (error) { + const durabilityError = + error instanceof PersistenceDurabilityError + ? error + : new PersistenceDurabilityError( + error, + `persistence.adapter.applyCommittedTx`, + ) + throw this.markTerminalFailure(durabilityError, lifecycleGeneration) + } + } + private createPersistedTxFromOperations( transaction: BufferedSyncTransaction, streamPosition: { term: number; seq: number; rowVersion: number }, @@ -1622,9 +1760,9 @@ class PersistedCollectionRuntime< } } - private confirmMutationsSyncUnsafe( + private async confirmMutationsSyncUnsafe( mutations: Array>, - ): void { + ): Promise { if ( !this.syncControls.begin || !this.syncControls.write || @@ -1633,7 +1771,7 @@ class PersistedCollectionRuntime< return } - this.withInternalApply(() => { + const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) for (const mutation of mutations) { @@ -1650,8 +1788,12 @@ class PersistedCollectionRuntime< } } - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) + + if (applied !== true) { + await applied + } } private filterMutationsForCollection( @@ -1668,7 +1810,9 @@ class PersistedCollectionRuntime< private async persistCollectionMutationsUnsafe( mutations: Array>, + lifecycleGeneration: number, ): Promise> { + this.throwIfLifecycleReplaced(lifecycleGeneration) // When a coordinator with requestApplyLocalMutations is available, always // route through it — even on the leader tab. This ensures the coordinator's // seq/rowVersion counters stay in sync with actual writes. Without this, @@ -1688,6 +1832,8 @@ class PersistedCollectionRuntime< envelopeMutations, ) + this.throwIfLifecycleReplaced(lifecycleGeneration) + if (!response.ok) { throw new Error( `failed to apply local mutations through coordinator: ${response.error}`, @@ -1723,7 +1869,9 @@ class PersistedCollectionRuntime< // SingleProcessCoordinator). Apply directly and broadcast. const streamPosition = this.nextLocalStreamPosition() const tx = this.createPersistedTxFromMutations(mutations, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + await this.applyCommittedTx(tx, lifecycleGeneration) + + this.throwIfLifecycleReplaced(lifecycleGeneration) this.publishTxCommittedEvent( this.createTxCommittedPayload({ @@ -1870,19 +2018,23 @@ class PersistedCollectionRuntime< private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { if ( + this.getCurrentTerminalFailure() || this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset || - this.activeSubsets.get(this.getSubsetKey(options)) !== options + !this.persistence.coordinator.requestEnsureRemoteSubset ) { return } - this.pendingRemoteSubsetEnsures.set(this.getSubsetKey(options), options) + const subsetKey = this.getSubsetKey(options) + if (this.activeSubsets.get(subsetKey) !== options) return + + this.pendingRemoteSubsetEnsures.set(subsetKey, options) void this.flushPendingRemoteSubsetEnsures() } private scheduleRemoteEnsureRetry(): void { if ( + this.getCurrentTerminalFailure() || this.mode !== `sync-present` || !this.persistence.coordinator.requestEnsureRemoteSubset ) { @@ -1904,6 +2056,7 @@ class PersistedCollectionRuntime< private async flushPendingRemoteSubsetEnsures(): Promise { if ( + this.getCurrentTerminalFailure() || this.mode !== `sync-present` || !this.persistence.coordinator.requestEnsureRemoteSubset ) { @@ -1947,8 +2100,12 @@ class PersistedCollectionRuntime< if (message.collectionId !== this.collectionId) { return } + if (this.getCurrentTerminalFailure()) { + return + } const { payload } = message + const lifecycleGeneration = this.lifecycleGeneration const isSelf = message.senderId === this.persistence.coordinator.getNodeId() // Allow tx:committed from self — the coordinator produces these on behalf @@ -1956,14 +2113,17 @@ class PersistedCollectionRuntime< // processCommittedTxUnsafe prevents double-processing of our own writes. if (isTxCommittedPayload(payload)) { if (this.isHydratingNow()) { - this.queuedTxCommitted.push(payload) + this.queuedTxCommitted.push({ + txCommitted: payload, + lifecycleGeneration, + }) return } void this.applyMutex - .run(() => this.processCommittedTxUnsafe(payload)) + .run(() => this.processCommittedTxUnsafe(payload, lifecycleGeneration)) .catch((error) => { - console.warn(`Failed to process tx:committed message:`, error) + this.markTerminalFailure(error, lifecycleGeneration) }) return } @@ -1975,9 +2135,9 @@ class PersistedCollectionRuntime< if (isCollectionResetPayload(payload)) { void this.applyMutex - .run(() => this.truncateAndReloadUnsafe()) + .run(() => this.truncateAndReloadUnsafe(lifecycleGeneration)) .catch((error) => { - console.warn(`Failed to process collection reset message:`, error) + this.markTerminalFailure(error, lifecycleGeneration) }) } } @@ -1988,13 +2148,18 @@ class PersistedCollectionRuntime< if (!queued) { continue } - await this.processCommittedTxUnsafe(queued) + await this.processCommittedTxUnsafe( + queued.txCommitted, + queued.lifecycleGeneration, + ) } } private async processCommittedTxUnsafe( txCommitted: TxCommitted, + lifecycleGeneration = this.lifecycleGeneration, ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (txCommitted.term < this.latestTerm) { return } @@ -2014,7 +2179,8 @@ class PersistedCollectionRuntime< const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms if (hasGap) { - await this.recoverFromSeqGapUnsafe() + await this.recoverFromSeqGapUnsafe(lifecycleGeneration) + if (lifecycleGeneration !== this.lifecycleGeneration) return if ( txCommitted.term < this.latestTerm || (txCommitted.term === this.latestTerm && @@ -2033,13 +2199,23 @@ class PersistedCollectionRuntime< await this.invalidateFromCommittedTxUnsafe(txCommitted) } - private async recoverFromSeqGapUnsafe(): Promise { + private async recoverFromSeqGapUnsafe( + lifecycleGeneration: number, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.persistence.coordinator.pullSince && this.latestRowVersion >= 0) { + let pullResponse: PullSinceResponse | undefined try { - const pullResponse = await this.persistence.coordinator.pullSince( + pullResponse = await this.persistence.coordinator.pullSince( this.collectionId, this.latestRowVersion, ) + } catch (error) { + console.warn(`Failed pullSince recovery attempt:`, error) + } + + if (pullResponse) { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (pullResponse.ok) { this.observeStreamPosition( @@ -2053,6 +2229,7 @@ class PersistedCollectionRuntime< } for (const delta of pullResponse.deltas) { + if (lifecycleGeneration !== this.lifecycleGeneration) return await this.invalidateFromCommittedTxUnsafe({ type: `tx:committed`, term: pullResponse.latestTerm, @@ -2065,15 +2242,15 @@ class PersistedCollectionRuntime< rowMetadataMutations: delta.rowMetadataMutations, collectionMetadataMutations: delta.collectionMetadataMutations, }) + if (lifecycleGeneration !== this.lifecycleGeneration) return } return } - } catch (error) { - console.warn(`Failed pullSince recovery attempt:`, error) } } - await this.truncateAndReloadUnsafe() + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.truncateAndReloadUnsafe(lifecycleGeneration) if (this.mode === `sync-present`) { for (const options of this.activeSubsets.values()) { @@ -2082,15 +2259,20 @@ class PersistedCollectionRuntime< } } - private async truncateAndReloadUnsafe(): Promise { + private async truncateAndReloadUnsafe( + lifecycleGeneration = this.lifecycleGeneration, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.syncControls.begin && this.syncControls.commit) { - this.withInternalApply(() => { + const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) this.syncControls.truncate?.() - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) + if (applied !== true) await applied } + if (lifecycleGeneration !== this.lifecycleGeneration) return await this.reloadActiveSubsetsUnsafe() } @@ -2130,7 +2312,7 @@ class PersistedCollectionRuntime< (opt) => (opt.where ? compileSingleRowExpression(opt.where) : null), ) - this.withInternalApply(() => { + const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) for (const { @@ -2175,8 +2357,9 @@ class PersistedCollectionRuntime< } }) - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) + if (applied !== true) await applied } private async reloadActiveSubsetsUnsafe(): Promise { @@ -2202,7 +2385,7 @@ class PersistedCollectionRuntime< } } - this.replaceCollectionSnapshot( + const applied = this.replaceCollectionSnapshot( Array.from(mergedRows.entries()).map(([key, row]) => ({ key, value: row.value, @@ -2210,6 +2393,7 @@ class PersistedCollectionRuntime< })), collectionMetadata, ) + if (applied !== true) await applied } finally { if (this.hydratingGeneration === lifecycleGeneration) { this.hydratingGeneration = null @@ -2326,17 +2510,192 @@ function createWrappedSyncConfig< ...sourceSyncConfig, sync: (params) => { const transactionStack: Array> = [] + const pendingPublicationTransactions: Array< + OpenSyncTransaction + > = [] + const publicationMarkers = new Map< + OpenSyncTransaction, + (typeof params.collection._state.pendingSyncedTransactions)[number] + >() const getOpenTransaction = () => transactionStack[transactionStack.length - 1] + const markPendingMetadataDependency = ( + transaction: OpenSyncTransaction, + ) => { + const openTransaction = getOpenTransaction() + if (openTransaction && !openTransaction.internal) { + transaction.hasDependentSuccessor = true + } + } + const removePendingPublicationTransaction = ( + transaction: OpenSyncTransaction, + ) => { + const index = pendingPublicationTransactions.indexOf(transaction) + if (index !== -1) pendingPublicationTransactions.splice(index, 1) + } + const reservePublicationMarker = ( + transaction: OpenSyncTransaction, + ) => { + const pending = params.collection._state.pendingSyncedTransactions + const previousLength = pending.length + params.begin(transaction.beginOptions) + const marker = pending.at(-1) + if (!marker || pending.length !== previousLength + 1) { + throw new InvalidPersistedCollectionConfigError( + `wrapped sync begin did not reserve a publication transaction`, + ) + } + publicationMarkers.set(transaction, marker) + } + const activatePublicationMarker = ( + transaction: OpenSyncTransaction, + ) => { + const marker = publicationMarkers.get(transaction) + const pending = params.collection._state.pendingSyncedTransactions + const index = marker ? pending.indexOf(marker) : -1 + if (!marker || index === -1) { + throw new InvalidPersistedCollectionConfigError( + `wrapped sync transaction lost its publication reservation`, + ) + } + if (index !== pending.length - 1) { + pending.splice(index, 1) + pending.push(marker) + } + } + const removePublicationMarker = ( + transaction: OpenSyncTransaction, + ) => { + const marker = publicationMarkers.get(transaction) + if (!marker) return + const pending = params.collection._state.pendingSyncedTransactions + const index = pending.indexOf(marker) + if (index !== -1) pending.splice(index, 1) + publicationMarkers.delete(transaction) + } + const settlePendingTransaction = ( + transaction: OpenSyncTransaction, + ) => { + removePendingPublicationTransaction(transaction) + removePublicationMarker(transaction) + } + const getPendingRowMetadataWrite = (key: TKey) => { + for ( + let index = pendingPublicationTransactions.length - 1; + index >= 0; + index-- + ) { + const transaction = pendingPublicationTransactions[index]! + const write = transaction.rowMetadataWrites.get(key) + if (write) { + markPendingMetadataDependency(transaction) + return { found: true, write } + } + if (transaction.truncate) { + markPendingMetadataDependency(transaction) + return { + found: true, + write: { type: `delete` as const }, + } + } + } + return { found: false as const } + } + const getPendingCollectionMetadataWrite = (key: string) => { + for ( + let index = pendingPublicationTransactions.length - 1; + index >= 0; + index-- + ) { + const transaction = pendingPublicationTransactions[index]! + const write = transaction.collectionMetadataWrites.get(key) + if (write) { + markPendingMetadataDependency(transaction) + return { found: true, write } + } + } + return { found: false as const } + } let fullStartPromise: Promise | null = null const startupState = { cleanedUp: false } const acquisitions = new Map() + const getTerminalFailure = () => runtime.getCurrentTerminalFailure() + const createHandledRejection = (error: unknown): Promise => { + const rejected = Promise.reject(error) + void rejected.catch(() => undefined) + return rejected + } + const applyTransactionToCollection = ( + transaction: OpenSyncTransaction, + signal?: AbortSignal, + ): SyncAppliedReceipt => { + activatePublicationMarker(transaction) + try { + if (transaction.truncate) { + params.truncate() + } + + for (const operation of transaction.operations) { + if (operation.type === `delete`) { + params.write({ + type: `delete`, + key: operation.key, + }) + } else { + params.write({ + type: `update`, + value: operation.value, + metadata: operation.metadata, + }) + } + } + + for (const [key, metadataWrite] of transaction.rowMetadataWrites) { + if (metadataWrite.type === `delete`) { + params.metadata?.row.delete(key) + } else { + params.metadata?.row.set(key, metadataWrite.value) + } + } + + for (const [ + key, + metadataWrite, + ] of transaction.collectionMetadataWrites) { + if (metadataWrite.type === `delete`) { + params.metadata?.collection.delete(key) + } else { + params.metadata?.collection.set(key, metadataWrite.value) + } + } + + const applied = params.commit(signal) + if (applied === true) { + removePendingPublicationTransaction(transaction) + } else { + void applied.then( + () => removePendingPublicationTransaction(transaction), + () => removePendingPublicationTransaction(transaction), + ) + } + return applied + } catch (error) { + // Queued source transactions must remain publicly atomic, but later + // source work still needs their accepted metadata decisions while + // publication is pending. Once publication runs, the collection's + // metadata API becomes authoritative again. A prepublication failure + // removes the layer without exposing it. + removePendingPublicationTransaction(transaction) + throw error + } + } runtime.setSyncControls({ begin: params.begin, write: params.write as SyncControlFns[`write`], commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, @@ -2345,42 +2704,51 @@ function createWrappedSyncConfig< const wrappedParams = { ...params, markReady: () => { - if (startupState.cleanedUp) return + if (startupState.cleanedUp || getTerminalFailure()) return void (fullStartPromise ?? runtime.ensureStarted()) .then(() => { - if (startupState.cleanedUp) return - params.markReady() - }) - .catch((error) => { - if (startupState.cleanedUp) return - console.warn( - `Failed persisted sync startup before markReady:`, - error, - ) + if (startupState.cleanedUp || getTerminalFailure()) return params.markReady() }) + .catch(() => undefined) }, begin: (options?: { immediate?: boolean }) => { if (startupState.cleanedUp) return + const terminalFailure = getTerminalFailure() + const internal = runtime.isApplyingInternally() const transaction: OpenSyncTransaction = { operations: [], rowMetadataWrites: new Map(), collectionMetadataWrites: new Map(), truncate: false, - internal: runtime.isApplyingInternally(), + internal, + lifecycleGeneration: runtime.getLifecycleGeneration(), + beginOptions: options, + hasDependentSuccessor: false, queuedBecauseHydrating: - !runtime.isApplyingInternally() && runtime.isHydratingNow(), + terminalFailure === undefined && + !internal && + runtime.isHydratingNow(), + ...(terminalFailure === undefined ? {} : { terminalFailure }), } - transactionStack.push(transaction) - - if (!transaction.queuedBecauseHydrating) { + if (transaction.internal && !terminalFailure) { params.begin(options) + } else if (!transaction.internal && !terminalFailure) { + reservePublicationMarker(transaction) } + transactionStack.push(transaction) }, write: (message: ChangeMessageOrDeleteKeyMessage) => { if (startupState.cleanedUp) return - const normalization = runtime.normalizeSyncWriteMessage(message) const openTransaction = getOpenTransaction() + const terminalFailure = + openTransaction?.terminalFailure ?? getTerminalFailure() + if (terminalFailure) { + if (openTransaction) + openTransaction.terminalFailure = terminalFailure + return + } + const normalization = runtime.normalizeSyncWriteMessage(message) if (!openTransaction) { params.write(normalization.forwardMessage) @@ -2417,7 +2785,7 @@ function createWrappedSyncConfig< value: normalization.operation.metadata, }) } - if (!openTransaction.queuedBecauseHydrating) { + if (openTransaction.internal) { params.write(normalization.forwardMessage) } }, @@ -2441,6 +2809,12 @@ function createWrappedSyncConfig< if (openTransaction?.truncate) { return undefined } + const pending = getPendingRowMetadataWrite(key) + if (pending.found) { + return pending.write.type === `delete` + ? undefined + : pending.write.value + } return params.metadata!.row.get(key) }, scanPersisted: (options?: PersistedRowScanOptions) => @@ -2450,6 +2824,8 @@ function createWrappedSyncConfig< set: (key: TKey, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.row.set must be called within an open sync transaction`, @@ -2459,13 +2835,15 @@ function createWrappedSyncConfig< type: `set`, value, }) - if (!openTransaction.queuedBecauseHydrating) { + if (openTransaction.internal) { params.metadata!.row.set(key, value) } }, delete: (key: TKey) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.row.delete must be called within an open sync transaction`, @@ -2474,7 +2852,7 @@ function createWrappedSyncConfig< openTransaction.rowMetadataWrites.set(key, { type: `delete`, }) - if (!openTransaction.queuedBecauseHydrating) { + if (openTransaction.internal) { params.metadata!.row.delete(key) } }, @@ -2490,11 +2868,19 @@ function createWrappedSyncConfig< ? undefined : pendingWrite.value } + const pending = getPendingCollectionMetadataWrite(key) + if (pending.found) { + return pending.write.type === `delete` + ? undefined + : pending.write.value + } return params.metadata!.collection.get(key) }, set: (key: string, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.collection.set must be called within an open sync transaction`, @@ -2504,13 +2890,15 @@ function createWrappedSyncConfig< type: `set`, value, }) - if (!openTransaction.queuedBecauseHydrating) { + if (openTransaction.internal) { params.metadata!.collection.set(key, value) } }, delete: (key: string) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.collection.delete must be called within an open sync transaction`, @@ -2519,7 +2907,7 @@ function createWrappedSyncConfig< openTransaction.collectionMetadataWrites.set(key, { type: `delete`, }) - if (!openTransaction.queuedBecauseHydrating) { + if (openTransaction.internal) { params.metadata!.collection.delete(key) } }, @@ -2530,6 +2918,28 @@ function createWrappedSyncConfig< .metadata!.collection.list() .map(({ key, value }) => [key, value]), ) + const pendingOwners = new Map< + string, + OpenSyncTransaction + >() + for (const transaction of pendingPublicationTransactions) { + for (const [ + key, + metadataWrite, + ] of transaction.collectionMetadataWrites) { + if (!prefix || key.startsWith(prefix)) { + pendingOwners.set(key, transaction) + } + if (metadataWrite.type === `delete`) { + merged.delete(key) + } else { + merged.set(key, metadataWrite.value) + } + } + } + for (const owner of new Set(pendingOwners.values())) { + markPendingMetadataDependency(owner) + } const openTransaction = getOpenTransaction() if (openTransaction) { for (const [ @@ -2557,6 +2967,7 @@ function createWrappedSyncConfig< truncate: () => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) return if (!openTransaction) { params.truncate() return @@ -2569,23 +2980,49 @@ function createWrappedSyncConfig< // collection-scoped metadata before truncating row data, and those // writes must commit atomically with the truncate transaction. openTransaction.truncate = true - if (!openTransaction.queuedBecauseHydrating) { + if (openTransaction.internal) { params.truncate() } }, commit: (signal?: AbortSignal) => { if (startupState.cleanedUp) return true const openTransaction = transactionStack.pop() + const terminalFailure = + openTransaction?.terminalFailure ?? getTerminalFailure() + if (terminalFailure) { + if (openTransaction) settlePendingTransaction(openTransaction) + return createHandledRejection(terminalFailure.error) + } if (!openTransaction) { return params.commit(signal) } - if (openTransaction.queuedBecauseHydrating) { - if (signal?.aborted) { - const aborted = Promise.reject(new SyncTransactionAbortedError()) - void aborted.catch(() => undefined) - return aborted - } + if (openTransaction.internal) { + return params.commit(signal) + } + + if (signal?.aborted) { + settlePendingTransaction(openTransaction) + return createHandledRejection(new SyncTransactionAbortedError()) + } + const transaction = { + operations: openTransaction.operations, + rowMetadataWrites: openTransaction.rowMetadataWrites, + collectionMetadataWrites: openTransaction.collectionMetadataWrites, + truncate: openTransaction.truncate, + internal: false, + lifecycleGeneration: openTransaction.lifecycleGeneration, + beginOptions: openTransaction.beginOptions, + signal, + applyToCollection: () => + applyTransactionToCollection(openTransaction, signal), + shouldFailStopOnAbort: () => openTransaction.hasDependentSuccessor, + } + pendingPublicationTransactions.push(openTransaction) + if ( + openTransaction.queuedBecauseHydrating && + runtime.isHydratingNow() + ) { let resolveApplied!: () => void let rejectApplied!: (error: unknown) => void const applied = new Promise((resolve, reject) => { @@ -2594,36 +3031,22 @@ function createWrappedSyncConfig< }) void applied.catch(() => undefined) runtime.queueHydrationBufferedTransaction({ - operations: openTransaction.operations, - rowMetadataWrites: openTransaction.rowMetadataWrites, - collectionMetadataWrites: - openTransaction.collectionMetadataWrites, - truncate: openTransaction.truncate, - internal: openTransaction.internal, - signal, + ...transaction, resolveApplied, rejectApplied, }) + void applied.then( + () => settlePendingTransaction(openTransaction), + () => settlePendingTransaction(openTransaction), + ) return applied } - const applied = params.commit(signal) - if (!openTransaction.internal) { - const persistAfterApplication = async () => { - if (applied !== true) await applied - await runtime.persistAndBroadcastExternalSyncTransaction({ - operations: openTransaction.operations, - rowMetadataWrites: openTransaction.rowMetadataWrites, - collectionMetadataWrites: - openTransaction.collectionMetadataWrites, - truncate: openTransaction.truncate, - internal: false, - }) - } - const persisted = persistAfterApplication() - void persisted.catch(() => undefined) - return persisted - } + const applied = runtime.applyHydrationBufferedTransaction(transaction) + void applied.then( + () => settlePendingTransaction(openTransaction), + () => settlePendingTransaction(openTransaction), + ) return applied }, } @@ -2647,9 +3070,12 @@ function createWrappedSyncConfig< cleanup: () => { startupState.cleanedUp = true acquisitions.clear() + pendingPublicationTransactions.length = 0 + for (const transaction of publicationMarkers.keys()) { + removePublicationMarker(transaction) + } sourceResult.cleanup?.() runtime.cleanup() - runtime.clearSyncControls() }, loadSubset: async (options: LoadSubsetOptions) => { const acquisition = { forwarded: false } @@ -2708,6 +3134,7 @@ function createLoopbackSyncConfig< commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, @@ -2716,17 +3143,14 @@ function createLoopbackSyncConfig< void runtime .ensureStarted() .then(() => { + if (runtime.getCurrentTerminalFailure()) return params.markReady() }) - .catch((error) => { - console.warn(`Failed persisted loopback startup:`, error) - params.markReady() - }) + .catch(() => undefined) return { cleanup: () => { runtime.cleanup() - runtime.clearSyncControls() }, loadSubset: (options: LoadSubsetOptions) => runtime.loadSubset(options), unloadSubset: (options: LoadSubsetOptions) => @@ -2841,6 +3265,7 @@ export function persistedCollectionOptions< const wrappedOnInsert = async ( params: InsertMutationFnParams, ) => { + const lifecycleGeneration = runtime.getLifecycleGeneration() const handlerResult = localOnlyOptions.onInsert ? await localOnlyOptions.onInsert( params as unknown as InsertMutationFnParams, @@ -2849,6 +3274,7 @@ export function persistedCollectionOptions< await runtime.persistAndConfirmCollectionMutations( params.transaction.mutations as Array>, + lifecycleGeneration, ) return handlerResult ?? {} @@ -2857,6 +3283,7 @@ export function persistedCollectionOptions< const wrappedOnUpdate = async ( params: UpdateMutationFnParams, ) => { + const lifecycleGeneration = runtime.getLifecycleGeneration() const handlerResult = localOnlyOptions.onUpdate ? await localOnlyOptions.onUpdate( params as unknown as UpdateMutationFnParams, @@ -2865,6 +3292,7 @@ export function persistedCollectionOptions< await runtime.persistAndConfirmCollectionMutations( params.transaction.mutations as Array>, + lifecycleGeneration, ) return handlerResult ?? {} @@ -2873,6 +3301,7 @@ export function persistedCollectionOptions< const wrappedOnDelete = async ( params: DeleteMutationFnParams, ) => { + const lifecycleGeneration = runtime.getLifecycleGeneration() const handlerResult = localOnlyOptions.onDelete ? await localOnlyOptions.onDelete( params as unknown as DeleteMutationFnParams, @@ -2881,6 +3310,7 @@ export function persistedCollectionOptions< await runtime.persistAndConfirmCollectionMutations( params.transaction.mutations as Array>, + lifecycleGeneration, ) return handlerResult ?? {} diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 313cdad860..d05b84fd55 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -12,6 +12,7 @@ import { InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, InvalidSyncConfigError, + PersistenceDurabilityError, SingleProcessCoordinator, createPersistedTableName, decodePersistedStorageKey, @@ -27,7 +28,34 @@ import type { PullSinceResponse, TxCommitted, } from '../src' -import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { + LoadSubsetOptions, + PendingMutation, + SyncConfig, +} from '@tanstack/db' + +/** + * # Does persisted wrapping preserve one Collection history? + * + * RFC #1659 invariant 7 requires each accepted sync transaction to become + * durable, remain replayable, or fail through an observable channel. Startup + * hydrates rows and metadata, buffers concurrent source work, then publishes a + * coherent public snapshot. Publication precedes durability, but a rejected + * durability boundary must reject its applied receipt and fail-stop that sync + * run without admitting a suffix. + * + * `foldDurabilityLedger` is the independent model for append-only source + * obligations. The history grammar crosses hydration, sibling commits, abort, + * receipt rejection, coordinator replay, cleanup, and restart. Tests drive the + * real persisted wrapper, Collection, coordinator, adapter, and local mutation + * path. Named checkpoints compare public and durable rows, metadata, receipt + * settlement, exact errors, call order, and lifecycle ownership. + * + * Fixed witnesses and bounded schedule tables preserve the known failure + * paths. The model's omission and reordering controls challenge its judgment. + * Native SQLite hosts, live Electric service behavior, adapter cancellation, + * and the open B2-failure/E4 schedule remain separate evidence. + */ /** * # Does persisted wrapping preserve one Collection history? @@ -55,6 +83,8 @@ type Todo = { title: string } +type TodoSyncParams = Parameters[`sync`]>[0] + type RecordingAdapter = PersistenceAdapter & { applyCommittedTxCalls: Array<{ collectionId: string @@ -261,6 +291,52 @@ function createCoordinatorHarness(): CoordinatorHarness { return harness } +type FailStopCoordinatorHarness = PersistedCollectionCoordinator & { + emit: (payload: unknown, senderId?: string) => void + publishCalls: Array> + remoteEnsureCalls: Array + unsubscribeCalls: number +} + +function createFailStopCoordinatorHarness( + collectionId: string, +): FailStopCoordinatorHarness { + let subscriber: ((message: ProtocolEnvelope) => void) | undefined + const harness: FailStopCoordinatorHarness = { + publishCalls: [], + remoteEnsureCalls: [], + unsubscribeCalls: 0, + getNodeId: () => `fail-stop-coordinator`, + subscribe: (_collectionId, onMessage) => { + subscriber = onMessage + return () => { + harness.unsubscribeCalls++ + subscriber = undefined + } + }, + publish: (_collectionId, message) => { + harness.publishCalls.push(message) + }, + isLeader: () => true, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestEnsureRemoteSubset: async (_collectionId, options) => { + harness.remoteEnsureCalls.push(options) + }, + emit: (payload, senderId = `remote-fail-stop-peer`) => { + subscriber?.({ + v: 1, + dbName: `test-db`, + collectionId, + senderId, + ts: Date.now(), + payload, + }) + }, + } + return harness +} + const stripVirtualProps = | undefined>( value: T, ): T => { @@ -279,6 +355,417 @@ async function flushAsyncWork(delayMs: number = 0): Promise { await new Promise((resolve) => setTimeout(resolve, delayMs)) } +function createEventGate(): { + promise: Promise + resolve: () => void + reject: (error: unknown) => void +} { + let resolve!: () => void + let reject!: (error: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +async function atPersistedOracleCheckpoint( + promise: Promise, + label: string, + timeout = 1000, +): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject( + new Error(`Persisted oracle checkpoint timed out: ${label}`), + ), + timeout, + ) + }), + ]) + } finally { + clearTimeout(timer) + } +} + +async function cleanupPersistedOracle( + actions: Array<() => void | Promise>, + hasPrimaryFailure: boolean, +): Promise { + const failures: Array = [] + for (const [index, action] of actions.entries()) { + try { + await atPersistedOracleCheckpoint( + Promise.resolve().then(action), + `cleanup stage ${index}`, + 250, + ) + } catch (error) { + failures.push(error) + } + } + if (failures.length > 0) { + if (hasPrimaryFailure) { + console.warn( + `Persisted oracle cleanup failed after the primary failure:`, + failures, + ) + } else { + throw new AggregateError(failures, `Persisted oracle cleanup failed`) + } + } +} + +type DurabilityLedgerEvent = + | { type: `begin`; transactionId: string } + | { type: `write`; transactionId: string; row: Todo } + | { type: `commit`; transactionId: string } + | { type: `abort`; transactionId: string } + +// RFC #1659 durability law: a source commit remains one append-only obligation +// across a persistence hydration boundary. The independent fold deliberately +// has no hydration phase, queue, generation, or production transaction helper. +// These fixtures exercise the core adapter contract with in-memory storage; +// they do not claim native SQLite host or device execution. +function foldDurabilityLedger(events: ReadonlyArray): { + committedRows: Map + commitOrder: Array +} { + const open = new Map>() + const committedRows = new Map() + const commitOrder: Array = [] + + for (const event of events) { + if (event.type === `begin`) { + open.set(event.transactionId, []) + } else if (event.type === `write`) { + open.get(event.transactionId)?.push(structuredClone(event.row)) + } else if (event.type === `abort`) { + open.delete(event.transactionId) + } else { + const rows = open.get(event.transactionId) + if (!rows) continue + for (const row of rows) committedRows.set(row.id, row) + commitOrder.push(event.transactionId) + open.delete(event.transactionId) + } + } + + return { committedRows, commitOrder } +} + +function observeSettlement(promise: Promise): { + read: () => + | { status: `pending` } + | { status: `fulfilled` } + | { status: `rejected`; reason: unknown } +} { + let outcome: + | { status: `pending` } + | { status: `fulfilled` } + | { status: `rejected`; reason: unknown } = { status: `pending` } + void promise.then( + () => { + outcome = { status: `fulfilled` } + }, + (reason: unknown) => { + outcome = { status: `rejected`, reason } + }, + ) + return { read: () => outcome } +} + +async function runRejectedHydrationBufferWitness( + id: string, + rows: ReadonlyArray, + options: { commitAfterHydrationFailure?: boolean } = {}, +): Promise { + const hydrationEntered = createEventGate() + const releaseHydration = createEventGate() + const hydrationRejected = createEventGate() + const hydrationError = new Error(`persisted hydration rejected exactly`) + const adapter = createRecordingAdapter() + const successfulLoadSubset = adapter.loadSubset.bind(adapter) + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await releaseHydration.promise + hydrationRejected.resolve() + throw hydrationError + } + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + let initialCollectionCleaned = false + const cleanupActions: Array<() => void | Promise> = [ + () => (initialCollectionCleaned ? undefined : collection.cleanup()), + ] + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + + try { + void collection.stateWhenReady().catch(() => undefined) + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `${id} hydration entered`, + ) + expect(remoteBegin).toBeTypeOf(`function`) + expect(remoteWrite).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + + const committers = rows.map((row) => { + remoteBegin!() + remoteWrite!({ type: `insert`, value: row }) + return () => { + const receipt = remoteCommit!() + expect(receipt).toBeInstanceOf(Promise) + return observeSettlement(Promise.resolve(receipt).then(() => undefined)) + } + }) + let receipts = options.commitAfterHydrationFailure + ? [] + : committers.map((commit) => commit()) + + releaseHydration.resolve() + await atPersistedOracleCheckpoint( + hydrationRejected.promise, + `${id} hydration rejected`, + ) + await flushAsyncWork() + + if (options.commitAfterHydrationFailure) { + receipts = committers.map((commit) => commit()) + await flushAsyncWork() + } + + const receiptOutcomes = receipts.map((receipt) => { + const outcome = receipt.read() + return outcome.status === `rejected` + ? { + status: outcome.status, + exactReason: outcome.reason === hydrationError, + } + : outcome + }) + expect({ + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === hydrationError, + receiptOutcomes, + visibleRows: rows.filter((row) => collection.has(row.id)), + durableRows: rows.filter((row) => adapter.rows.has(row.id)), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + status: `error`, + exactPublicError: true, + receiptOutcomes: rows.map(() => ({ + status: `rejected`, + exactReason: true, + })), + visibleRows: [], + durableRows: [], + durabilityCalls: 0, + }) + + await atPersistedOracleCheckpoint( + collection.cleanup(), + `${id} failed collection cleanup`, + ) + initialCollectionCleaned = true + adapter.loadSubset = successfulLoadSubset + + const reopened = createCollection( + persistedCollectionOptions({ + id, + getKey: (item) => item.id, + persistence: { adapter }, + }), + ) + cleanupActions.push(() => reopened.cleanup()) + await atPersistedOracleCheckpoint( + reopened.stateWhenReady(), + `${id} reopened hydration`, + ) + await flushAsyncWork() + + expect({ + visibleRows: rows.filter((row) => reopened.has(row.id)), + durableRows: rows.filter((row) => adapter.rows.has(row.id)), + rowMetadata: Array.from(adapter.rowMetadata.entries()), + collectionMetadata: Array.from(adapter.collectionMetadata.entries()), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + visibleRows: [], + durableRows: [], + rowMetadata: [], + collectionMetadata: [], + durabilityCalls: 0, + }) + + const freshRow = { id: `${id}:fresh`, title: `post-restart control` } + const freshTransaction = reopened.insert(freshRow) + await atPersistedOracleCheckpoint( + (async () => { + await freshTransaction.isPersisted.promise + })(), + `${id} fresh transaction persisted`, + ) + await flushAsyncWork() + + expect({ + visibleFreshRow: stripVirtualProps(reopened.get(freshRow.id)), + durableFreshRow: adapter.rows.get(freshRow.id), + staleVisibleRows: rows.filter((row) => reopened.has(row.id)), + staleDurableRows: rows.filter((row) => adapter.rows.has(row.id)), + rowMetadata: Array.from(adapter.rowMetadata.entries()), + collectionMetadata: Array.from(adapter.collectionMetadata.entries()), + appliedKeys: adapter.applyCommittedTxCalls.flatMap(({ tx }) => + tx.mutations.map((mutation) => mutation.key), + ), + }).toEqual({ + visibleFreshRow: freshRow, + durableFreshRow: freshRow, + staleVisibleRows: [], + staleDurableRows: [], + rowMetadata: [], + collectionMetadata: [], + appliedKeys: [freshRow.id], + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHydration.resolve() + warning.mockRestore() + await cleanupPersistedOracle(cleanupActions, hasPrimaryFailure) + } +} + +async function createTerminalFailureHarness( + kind: `hydration` | `durability`, + id: string, +) { + const seed = { id: `seed`, title: `stable before terminal failure` } + const adapter = createRecordingAdapter([seed]) + adapter.rowMetadata.set(seed.id, `row-metadata-before-failure`) + adapter.collectionMetadata.set(`resume`, `collection-metadata-before-failure`) + const coordinator = createFailStopCoordinatorHarness(id) + const upstreamLoads: Array = [] + const upstreamUnloads: Array = [] + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + return { + loadSubset: (options) => { + upstreamLoads.push(options) + return true + }, + unloadSubset: (options) => { + upstreamUnloads.push(options) + }, + } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const initialLease: LoadSubsetOptions = { limit: 1 } + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `${id} initially ready`, + ) + await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset(initialLease)), + `${id} initial lease hydrated`, + ) + await flushAsyncWork() + + let terminalError: unknown + if (kind === `hydration`) { + const hydrationError = new Error(`${id} terminal hydration failure`) + adapter.loadSubset = () => Promise.reject(hydrationError) + const outcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 2 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${id} hydration failure settled`, + ) + expect(outcome).toEqual({ status: `rejected`, reason: hydrationError }) + terminalError = hydrationError + } else { + const adapterError = Object.assign(new Error(`${id} storage failure`), { + code: `SQLITE_FULL`, + path: `adapter.applyCommittedTx`, + }) + adapter.applyCommittedTx = () => Promise.reject(adapterError) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `published before terminal failure` }, + }) + const outcome = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${id} durability failure settled`, + ) + expect(outcome.status).toBe(`rejected`) + terminalError = outcome.status === `rejected` ? outcome.reason : undefined + expect(terminalError).toBeInstanceOf(PersistenceDurabilityError) + expect( + terminalError instanceof Error ? terminalError.cause : undefined, + ).toBe(adapterError) + } + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(terminalError) + + return { + adapter, + collection, + coordinator, + initialLease, + seed, + sourceParams, + terminalError, + upstreamLoads, + upstreamUnloads, + } +} + describe(`persistedCollectionOptions`, () => { it(`provides a sync-absent loopback configuration with persisted utils`, async () => { const adapter = createRecordingAdapter() @@ -866,7 +1353,6 @@ describe(`persistedCollectionOptions`, () => { persistence: { adapter }, }), ) - try { await collection.stateWhenReady() const abortController = new AbortController() @@ -883,6 +1369,24 @@ describe(`persistedCollectionOptions`, () => { expect(collection.get(`aborted`)).toBeUndefined() expect(adapter.applyCommittedTxCalls).toHaveLength(0) + expect(collection._state.pendingSyncedTransactions).toHaveLength(0) + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `recovered`, title: `Later transaction` }, + }) + await remoteCommit?.() + + expect(stripVirtualProps(collection.get(`recovered`))).toEqual({ + id: `recovered`, + title: `Later transaction`, + }) + expect(adapter.rows.get(`recovered`)).toEqual({ + id: `recovered`, + title: `Later transaction`, + }) + expect(collection._state.pendingSyncedTransactions).toHaveLength(0) } finally { await collection.cleanup() } @@ -964,10 +1468,148 @@ describe(`persistedCollectionOptions`, () => { } }) - it(`rejects a wrapped sync receipt when persistence fails`, async () => { + it(`fail-stops a queued suffix when its staged predecessor aborts before publication`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-abort-before-publication-suffix`, + getKey: (item) => item.id, + sync: { + sync: (params) => { + sourceParams = params + remoteBegin = params.begin + remoteWrite = params.write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = params.commit + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const localPersistence = createEventGate() + const localTransaction = createTransaction({ + mutationFn: () => localPersistence.promise, + }) + const aborted = new AbortController() + let abortedReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await collection.stateWhenReady() + localTransaction.mutate(() => { + collection.insert({ id: `local`, title: `publication gate` }) + }) + + remoteBegin?.() + sourceParams.metadata!.row.set(`shared`, { owner: `aborted` }) + abortedReceipt = Promise.resolve(remoteCommit?.(aborted.signal)).then( + () => undefined, + ) + void abortedReceipt.catch(() => undefined) + + remoteBegin?.() + const stagedOwner = sourceParams.metadata!.row.get(`shared`) + remoteWrite?.({ + type: `insert`, + value: { + id: `dependent`, + title: + (stagedOwner as { owner?: string } | undefined)?.owner ?? `missing`, + }, + }) + dependentReceipt = Promise.resolve(remoteCommit?.()).then(() => undefined) + void dependentReceipt.catch(() => undefined) + + expect(stagedOwner).toEqual({ owner: `aborted` }) + aborted.abort() + const abortedOutcome = await atPersistedOracleCheckpoint( + abortedReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `staged predecessor aborted before publication`, + ) + localPersistence.resolve() + await localTransaction.isPersisted.promise + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-abort staged suffix settled`, + ) + const terminalError = + abortedOutcome.status === `rejected` ? abortedOutcome.reason : undefined + + expect({ + abortedStatus: abortedOutcome.status, + abortedName: + terminalError instanceof Error ? terminalError.name : undefined, + dependentStatus: dependentOutcome.status, + dependentExact: + dependentOutcome.status === `rejected` && + dependentOutcome.reason === terminalError, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + visibleDependent: collection.get(`dependent`), + durableDependent: adapter.rows.get(`dependent`), + durableMetadata: adapter.rowMetadata.get(`shared`), + }).toEqual({ + abortedStatus: `rejected`, + abortedName: `AbortError`, + dependentStatus: `rejected`, + dependentExact: true, + status: `error`, + exactPublicError: true, + visibleDependent: undefined, + durableDependent: undefined, + durableMetadata: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + localPersistence.resolve() + await cleanupPersistedOracle( + [ + () => localTransaction.isPersisted.promise.catch(() => undefined), + () => abortedReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects a wrapped sync receipt with a named terminal persistence error`, async () => { const adapter = createRecordingAdapter() - const persistenceError = new Error(`persistence failed`) - adapter.applyCommittedTx = () => Promise.reject(persistenceError) + const adapterError = Object.assign( + new Error(`adapter write failed exactly`), + { + code: `SQLITE_FULL`, + path: `adapter.applyCommittedTx`, + }, + ) + const persistenceEntered = createEventGate() + const persistence = createEventGate() + adapter.applyCommittedTx = async () => { + persistenceEntered.resolve() + await persistence.promise + } let remoteBegin: (() => void) | undefined let remoteWrite: | ((message: { type: `insert`; value: Todo }) => void) @@ -991,6 +1633,7 @@ describe(`persistedCollectionOptions`, () => { persistence: { adapter }, }), ) + let hasPrimaryFailure = false try { await collection.stateWhenReady() @@ -1000,65 +1643,3094 @@ describe(`persistedCollectionOptions`, () => { value: { id: `failed`, title: `Not durable` }, }) - await expect(Promise.resolve(remoteCommit?.())).rejects.toBe( - persistenceError, + const receipt = Promise.resolve(remoteCommit?.()).then(() => undefined) + const settlement = observeSettlement(receipt) + await atPersistedOracleCheckpoint( + persistenceEntered.promise, + `terminal persistence entered`, + ) + await flushAsyncWork() + + expect({ + receipt: settlement.read(), + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + visible: stripVirtualProps(collection.get(`failed`)), + durable: adapter.rows.get(`failed`), + }).toEqual({ + receipt: { status: `pending` }, + status: `ready`, + publicError: undefined, + visible: { id: `failed`, title: `Not durable` }, + durable: undefined, + }) + + persistence.reject(adapterError) + const outcome = await receipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), ) + + const publicError = collection._lifecycle.getSyncError() + const receiptError = + outcome.status === `rejected` && outcome.reason instanceof Error + ? (outcome.reason as Error & { + code?: unknown + path?: unknown + }) + : undefined + expect({ + receiptStatus: outcome.status, + receiptErrorName: receiptError?.name, + receiptHasNamedPersistenceSemantics: + outcome.status === `rejected` && + outcome.reason instanceof PersistenceDurabilityError, + receiptCauseIsAdapterError: receiptError?.cause === adapterError, + receiptErrorCode: receiptError?.code, + receiptErrorPath: receiptError?.path, + publicErrorIsReceipt: + outcome.status === `rejected` && publicError === outcome.reason, + status: collection.status, + visible: stripVirtualProps(collection.get(`failed`)), + durable: adapter.rows.get(`failed`), + }).toEqual({ + receiptStatus: `rejected`, + receiptErrorName: `PersistenceDurabilityError`, + receiptHasNamedPersistenceSemantics: true, + receiptCauseIsAdapterError: true, + receiptErrorCode: `SQLITE_FULL`, + receiptErrorPath: `adapter.applyCommittedTx`, + publicErrorIsReceipt: true, + status: `error`, + visible: { id: `failed`, title: `Not durable` }, + durable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error } finally { - await collection.cleanup() + persistence.reject(adapterError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) } }) - it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { + it(`keeps durability failure terminal when publication aborts the signal`, async () => { const adapter = createRecordingAdapter() - const ownership = { queryCollection: { owners: [`gc:q1`] } } - const sync: SyncConfig = { - sync: ({ begin, write, commit, markReady, metadata }) => { - begin() - metadata?.row.set(`remote-1`, ownership) - write({ - type: `insert`, - value: { - id: `remote-1`, - title: `From remote`, - }, - }) - commit() - markReady() + const storageError = Object.assign( + new Error(`post-publication durability failed exactly`), + { + code: `SQLITE_IOERR`, + path: `adapter.applyCommittedTx`, }, + ) + const persistenceEntered = createEventGate() + const releasePersistence = createEventGate() + adapter.applyCommittedTx = async () => { + persistenceEntered.resolve() + await releasePersistence.promise + throw storageError } - + let sourceParams!: TodoSyncParams + const abortController = new AbortController() const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, - getKey: (item: Todo) => item.id, - sync, - persistence: { - adapter, + id: `post-publication-abort-durability`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, }, + persistence: { adapter }, }), ) + const publication = collection.subscribeChanges((changes) => { + if (changes.some((entry) => entry.key === `failed`)) { + abortController.abort() + } + }) + let receipt: Promise | undefined + let hasPrimaryFailure = false - await collection.stateWhenReady() - await flushAsyncWork() + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `post-publication abort collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `failed`, title: `visible before durability` }, + }) + receipt = Promise.resolve( + sourceParams.commit(abortController.signal), + ).then(() => undefined) + void receipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + persistenceEntered.promise, + `post-publication abort persistence entered`, + ) - expect(adapter.rowMetadata.get(`remote-1`)).toEqual(ownership) - expect(collection._state.syncedMetadata.get(`remote-1`)).toEqual(ownership) + expect({ + signalAborted: abortController.signal.aborted, + visible: stripVirtualProps(collection.get(`failed`)), + status: collection.status, + }).toEqual({ + signalAborted: true, + visible: { id: `failed`, title: `visible before durability` }, + status: `ready`, + }) + + releasePersistence.resolve() + const outcome = await atPersistedOracleCheckpoint( + receipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-publication abort durability receipt`, + ) + const terminalError = + outcome.status === `rejected` ? outcome.reason : undefined + + expect({ + receiptStatus: outcome.status, + namedDurability: terminalError instanceof PersistenceDurabilityError, + cause: terminalError instanceof Error ? terminalError.cause : undefined, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + status: collection.status, + durable: adapter.rows.get(`failed`), + }).toEqual({ + receiptStatus: `rejected`, + namedDurability: true, + cause: storageError, + exactPublicError: true, + status: `error`, + durable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releasePersistence.resolve() + publication.unsubscribe() + await cleanupPersistedOracle( + [() => receipt?.catch(() => undefined), () => collection.cleanup()], + hasPrimaryFailure, + ) + } }) - it(`resets stale row metadata for a metadata-less insert with no queued metadata`, async () => { + it(`reserves FIFO admission before a publication callback commits a sibling`, async () => { const adapter = createRecordingAdapter() - adapter.rowMetadata.set(`remote-1`, { stale: true }) - const sync: SyncConfig = { - sync: ({ begin, write, commit, markReady }) => { - begin() - write({ - type: `insert`, - value: { - id: `remote-1`, - title: `From remote`, - }, - }) - commit() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const firstStorageError = new Error(`reentrant first durability failed`) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + throw firstStorageError + } + await successfulApply(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `reentrant-fifo-admission`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + let siblingReceipt: Promise | undefined + let publicationTriggers = 0 + const subscription = collection.subscribeChanges((changes) => { + if ( + publicationTriggers === 0 && + changes.some((entry) => entry.key === `first`) + ) { + publicationTriggers++ + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `sibling`, title: `must remain queued` }, + }) + siblingReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void siblingReceipt.catch(() => undefined) + } + }) + let firstReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reentrant FIFO collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `publishes before durability` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `reentrant first persistence entered`, + ) + await Promise.resolve() + + expect(siblingReceipt).toBeInstanceOf(Promise) + const firstSettlement = observeSettlement(firstReceipt) + const siblingSettlement = observeSettlement(siblingReceipt!) + await Promise.resolve() + expect({ + publicationTriggers, + applyCalls, + firstReceipt: firstSettlement.read(), + siblingReceipt: siblingSettlement.read(), + visibleFirst: stripVirtualProps(collection.get(`first`)), + visibleSibling: collection.get(`sibling`), + durableFirst: adapter.rows.get(`first`), + durableSibling: adapter.rows.get(`sibling`), + }).toEqual({ + publicationTriggers: 1, + applyCalls: 1, + firstReceipt: { status: `pending` }, + siblingReceipt: { status: `pending` }, + visibleFirst: { id: `first`, title: `publishes before durability` }, + visibleSibling: undefined, + durableFirst: undefined, + durableSibling: undefined, + }) + + releaseFirstPersistence.resolve() + const firstOutcome = await atPersistedOracleCheckpoint( + firstReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `reentrant first receipt rejected`, + ) + const siblingOutcome = await atPersistedOracleCheckpoint( + siblingReceipt!.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `reentrant sibling receipt rejected`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + expect({ + firstStatus: firstOutcome.status, + siblingStatus: siblingOutcome.status, + sameTerminal: + siblingOutcome.status === `rejected` && + siblingOutcome.reason === terminalError, + status: collection.status, + publicErrorIsFirst: + collection._lifecycle.getSyncError() === terminalError, + visibleSibling: collection.get(`sibling`), + durableSibling: adapter.rows.get(`sibling`), + applyCalls, + }).toEqual({ + firstStatus: `rejected`, + siblingStatus: `rejected`, + sameTerminal: true, + status: `error`, + publicErrorIsFirst: true, + visibleSibling: undefined, + durableSibling: undefined, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + subscription.unsubscribe() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => siblingReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects a queued suffix derived from an aborted staged metadata layer`, async () => { + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await successfulApply(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `aborted-staged-metadata-suffix`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const aborted = new AbortController() + let firstReceipt: Promise | undefined + let abortedReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `aborted staged metadata collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `durability gate` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first persistence entered before staged suffix`, + ) + + sourceParams.begin() + sourceParams.metadata!.row.set(`shared`, { owner: `aborted` }) + abortedReceipt = Promise.resolve( + sourceParams.commit(aborted.signal), + ).then(() => undefined) + void abortedReceipt.catch(() => undefined) + + sourceParams.begin() + const stagedOwner = sourceParams.metadata!.row.get(`shared`) + sourceParams.write({ + type: `insert`, + value: { + id: `dependent`, + title: + (stagedOwner as { owner?: string } | undefined)?.owner ?? `missing`, + }, + }) + dependentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void dependentReceipt.catch(() => undefined) + + expect(stagedOwner).toEqual({ owner: `aborted` }) + aborted.abort() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint( + firstReceipt, + `first durability gate settled`, + ) + const abortedOutcome = await atPersistedOracleCheckpoint( + abortedReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `aborted staged receipt settled`, + ) + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `dependent staged receipt settled`, + ) + const terminalError = + abortedOutcome.status === `rejected` ? abortedOutcome.reason : undefined + + expect({ + abortedStatus: abortedOutcome.status, + abortedName: + terminalError instanceof Error ? terminalError.name : undefined, + dependentStatus: dependentOutcome.status, + dependentExact: + dependentOutcome.status === `rejected` && + dependentOutcome.reason === terminalError, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + visibleDependent: collection.get(`dependent`), + durableDependent: adapter.rows.get(`dependent`), + durableMetadata: adapter.rowMetadata.get(`shared`), + applyCalls, + }).toEqual({ + abortedStatus: `rejected`, + abortedName: `AbortError`, + dependentStatus: `rejected`, + dependentExact: true, + status: `error`, + exactPublicError: true, + visibleDependent: undefined, + durableDependent: undefined, + durableMetadata: undefined, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => abortedReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not fail-stop when an aborted staged collection metadata write was superseded`, async () => { + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await successfulApply(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `superseded-staged-collection-metadata`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const olderAbort = new AbortController() + let firstReceipt: Promise | undefined + let olderReceipt: Promise | undefined + let newerReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `superseded collection metadata owner ready`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `durability gate` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first persistence entered before superseded metadata writes`, + ) + + sourceParams.begin() + sourceParams.metadata!.collection.set(`shared`, { owner: `older` }) + olderReceipt = Promise.resolve( + sourceParams.commit(olderAbort.signal), + ).then(() => undefined) + void olderReceipt.catch(() => undefined) + + sourceParams.begin() + sourceParams.metadata!.collection.set(`shared`, { owner: `newer` }) + newerReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void newerReceipt.catch(() => undefined) + + sourceParams.begin() + const stagedOwner = sourceParams + .metadata!.collection.list() + .find(({ key }) => key === `shared`)?.value + sourceParams.write({ + type: `insert`, + value: { + id: `dependent`, + title: + (stagedOwner as { owner?: string } | undefined)?.owner ?? `missing`, + }, + }) + dependentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void dependentReceipt.catch(() => undefined) + + expect(stagedOwner).toEqual({ owner: `newer` }) + olderAbort.abort() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint( + firstReceipt, + `superseded metadata durability gate settled`, + ) + const olderOutcome = await atPersistedOracleCheckpoint( + olderReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `superseded older metadata receipt settled`, + ) + const newerOutcome = await atPersistedOracleCheckpoint( + newerReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `newer metadata receipt settled`, + ) + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `newest-owner dependent receipt settled`, + ) + + expect({ + olderStatus: olderOutcome.status, + newerStatus: newerOutcome.status, + dependentStatus: dependentOutcome.status, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + visibleDependent: stripVirtualProps(collection.get(`dependent`)), + durableDependent: adapter.rows.get(`dependent`), + durableMetadata: adapter.collectionMetadata.get(`shared`), + applyCalls, + }).toEqual({ + olderStatus: `rejected`, + newerStatus: `fulfilled`, + dependentStatus: `fulfilled`, + status: `ready`, + publicError: undefined, + visibleDependent: { id: `dependent`, title: `newer` }, + durableDependent: { id: `dependent`, title: `newer` }, + durableMetadata: { owner: `newer` }, + applyCalls: 3, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => olderReceipt?.catch(() => undefined), + () => newerReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let an old external adapter success affect a restarted lifecycle`, async () => { + const id = `external-lifecycle-success` + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const oldPersistenceEntered = createEventGate() + const releaseOldPersistence = createEventGate() + adapter.applyCommittedTx = async (...args) => { + oldPersistenceEntered.resolve() + await releaseOldPersistence.promise + await successfulApply(...args) + } + const coordinator = createFailStopCoordinatorHarness(id) + let sourceParams!: TodoSyncParams + const replacement = { + id: `replacement`, + title: `owned by restarted lifecycle`, + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let oldSettlement: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `${id} initial lifecycle ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `old`, title: `must remain owned by old lifecycle` }, + }) + oldSettlement = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + oldPersistenceEntered.promise, + `${id} old persistence entered`, + ) + + await collection.cleanup() + cleanedUp = true + adapter.loadSubset = () => + Promise.resolve([{ key: replacement.id, value: replacement }]) + collection.startSyncImmediate() + cleanedUp = false + const replacementSettlement = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseOldPersistence.resolve() + + const oldOutcome = await atPersistedOracleCheckpoint( + oldSettlement, + `${id} old receipt settled`, + ) + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementSettlement, + `${id} replacement lifecycle settled`, + ) + await flushAsyncWork() + + expect({ + oldReceiptStatus: oldOutcome.status, + replacementStatus: replacementOutcome.status, + replacementVisible: stripVirtualProps(collection.get(replacement.id)), + replacementPublicError: collection._lifecycle.getSyncError(), + staleBroadcasts: coordinator.publishCalls.length, + }).toEqual({ + oldReceiptStatus: `fulfilled`, + replacementStatus: `fulfilled`, + replacementVisible: replacement, + replacementPublicError: undefined, + staleBroadcasts: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldPersistence.resolve() + await cleanupPersistedOracle( + [ + () => oldSettlement, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let an old external adapter failure poison a restarted lifecycle`, async () => { + const id = `external-lifecycle-failure` + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const oldPersistenceEntered = createEventGate() + const releaseOldPersistence = createEventGate() + const oldAdapterError = new Error(`old lifecycle adapter failure`) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + oldPersistenceEntered.resolve() + await releaseOldPersistence.promise + throw oldAdapterError + } + await successfulApply(...args) + } + const coordinator = createFailStopCoordinatorHarness(id) + let sourceParams!: TodoSyncParams + const replacement = { + id: `replacement`, + title: `owned by restarted lifecycle`, + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const startAndWaitForReady = async (label: string) => { + const ready = createEventGate() + const unsubscribe = collection.on(`status:ready`, () => ready.resolve()) + collection.startSyncImmediate() + try { + await atPersistedOracleCheckpoint(ready.promise, label) + } finally { + unsubscribe() + } + } + let oldSettlement: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let replacementReceipt: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let hasPrimaryFailure = false + + try { + await startAndWaitForReady(`${id} initial lifecycle ready`) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `old`, title: `must remain owned by old lifecycle` }, + }) + oldSettlement = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + oldPersistenceEntered.promise, + `${id} old persistence entered`, + ) + + await collection.cleanup() + await startAndWaitForReady( + `${id} replacement lifecycle ready before old failure`, + ) + sourceParams.begin() + sourceParams.write({ type: `insert`, value: replacement }) + replacementReceipt = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseOldPersistence.resolve() + + const oldOutcome = await atPersistedOracleCheckpoint( + oldSettlement, + `${id} old receipt settled`, + ) + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReceipt, + `${id} replacement transaction settled`, + ) + await flushAsyncWork() + + expect({ + oldReceiptStatus: oldOutcome.status, + replacementStatus: replacementOutcome.status, + replacementVisible: stripVirtualProps(collection.get(replacement.id)), + replacementPublicError: collection._lifecycle.getSyncError(), + broadcasts: coordinator.publishCalls.length, + applyCalls, + }).toEqual({ + oldReceiptStatus: `rejected`, + replacementStatus: `fulfilled`, + replacementVisible: replacement, + replacementPublicError: undefined, + broadcasts: 1, + applyCalls: 2, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldPersistence.resolve() + await cleanupPersistedOracle( + [ + () => oldSettlement, + () => replacementReceipt, + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not deliver coordinator work accepted before lifecycle replacement into the new owner`, async () => { + const id = `queued-coordinator-lifecycle` + const adapter = createRecordingAdapter() + const oldReloadEntered = createEventGate() + const releaseOldReload = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + oldReloadEntered.resolve() + await releaseOldReload.promise + return [] + } + if (subsetCalls === 3) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const replacementControlsReady = createEventGate() + let sourceRuns = 0 + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceRuns++ + if (sourceRuns === 2) replacementControlsReady.resolve() + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `queued coordinator initial owner ready`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `old-reload-gate`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await atPersistedOracleCheckpoint( + oldReloadEntered.promise, + `old coordinator reload entered`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `queued-before-restart`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [ + { + key: `stale`, + value: { id: `stale`, title: `must stay with old owner` }, + }, + ], + deletedKeys: [], + }) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + replacementControlsReady.promise, + `replacement sync controls installed`, + ) + releaseOldReload.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after queued coordinator work`, + ) + + expect({ + sourceRuns, + subsetCalls, + staleVisible: collection.get(`stale`), + staleDurable: adapter.rows.get(`stale`), + replacementPublicError: collection._lifecycle.getSyncError(), + }).toEqual({ + sourceRuns: 2, + subsetCalls: 3, + staleVisible: undefined, + staleDurable: undefined, + replacementPublicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldReload.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`discards a queued collection reset when its owner is cleaned up`, async () => { + const id = `queued-reset-lifecycle` + const adapter = createRecordingAdapter() + const oldReloadEntered = createEventGate() + const releaseOldReload = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + oldReloadEntered.resolve() + await releaseOldReload.promise + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `queued reset initial owner ready`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `old-reset-gate`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await atPersistedOracleCheckpoint( + oldReloadEntered.promise, + `old reload entered before queued reset`, + ) + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + + await collection.cleanup() + cleanedUp = true + releaseOldReload.resolve() + await flushAsyncWork() + await flushAsyncWork() + + expect({ + subsetCalls, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + subsetCalls: 2, + status: `cleaned-up`, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldReload.resolve() + await cleanupPersistedOracle( + [() => (cleanedUp ? undefined : collection.cleanup())], + hasPrimaryFailure, + ) + } + }) + + it(`does not apply a delayed seq-gap replay response into a restarted lifecycle`, async () => { + const id = `delayed-pull-since-lifecycle` + const adapter = createRecordingAdapter() + const pullEntered = createEventGate() + const releasePull = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => { + pullEntered.resolve() + await releasePull.promise + return { + type: `rpc:pullSince:res`, + rpcId: `old-delayed-pull`, + ok: true, + latestTerm: 1, + latestSeq: 2, + latestRowVersion: 2, + requiresFullReload: false, + changedKeys: [`stale`], + deletedKeys: [], + deltas: [ + { + txId: `old-delayed-delta`, + latestRowVersion: 2, + changedRows: [ + { + key: `stale`, + value: { id: `stale`, title: `must stay with old owner` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [], + collectionMetadataMutations: [], + }, + ], + } + } + const replacementControlsReady = createEventGate() + let sourceRuns = 0 + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceRuns++ + if (sourceRuns === 2) replacementControlsReady.resolve() + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `delayed pull initial owner ready`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `gap-trigger`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + pullEntered.promise, + `old pullSince entered`, + ) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + replacementControlsReady.promise, + `delayed pull replacement controls installed`, + ) + releasePull.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after delayed pull replay`, + ) + + expect({ + sourceRuns, + subsetCalls, + staleVisible: collection.get(`stale`), + staleDurable: adapter.rows.get(`stale`), + replacementPublicError: collection._lifecycle.getSyncError(), + }).toEqual({ + sourceRuns: 2, + subsetCalls: 2, + staleVisible: undefined, + staleDurable: undefined, + replacementPublicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releasePull.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not apply a later seq-gap replay delta through replacement controls`, async () => { + const id = `two-delta-replay-lifecycle` + const firstReceiptEntered = createEventGate() + const releaseFirstReceipt = createEventGate() + const replacementControlsInstalled = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `B baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const pullArguments: Array<{ + collectionId: string + fromRowVersion: number + }> = [] + coordinator.pullSince = async (collectionId, fromRowVersion) => { + pullArguments.push({ collectionId, fromRowVersion }) + return { + type: `rpc:pullSince:res`, + rpcId: `two-delta-replay`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: false, + changedKeys: [`first`, `second`], + deletedKeys: [], + deltas: [ + { + txId: `A-delta-1`, + latestRowVersion: 1, + changedRows: [ + { + key: `first`, + value: { id: `first`, title: `A first delta` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [ + { type: `set`, key: `first`, value: { owner: `A-first` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `replay:first`, value: `A-first` }, + ], + }, + { + txId: `A-delta-2`, + latestRowVersion: 2, + changedRows: [ + { + key: `second`, + value: { id: `second`, title: `A second delta` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [ + { type: `set`, key: `second`, value: { owner: `A-second` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `replay:second`, value: `A-second` }, + ], + }, + ], + } + } + + let gateNextApplicationReceipt = false + let controlsRun = 0 + const applicationReceipts: Array<{ + controlsOwner: `A` | `B` + returnedReceipt: `actual` | `held` + rowMetadataKeys: Array + collectionMetadataKeys: Array + }> = [] + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => { + controlsRun++ + const controlsOwner = + controlsRun === 1 ? (`A` as const) : (`B` as const) + if (controlsOwner === `B`) replacementControlsInstalled.resolve() + let rowMetadataKeys: Array = [] + let collectionMetadataKeys: Array = [] + return baseSync({ + ...params, + begin: (options) => { + rowMetadataKeys = [] + collectionMetadataKeys = [] + params.begin(options) + }, + metadata: { + row: { + ...params.metadata!.row, + set: (key, value) => { + rowMetadataKeys.push(String(key)) + params.metadata!.row.set(key, value) + }, + delete: (key) => { + rowMetadataKeys.push(String(key)) + params.metadata!.row.delete(key) + }, + }, + collection: { + ...params.metadata!.collection, + set: (key, value) => { + collectionMetadataKeys.push(key) + params.metadata!.collection.set(key, value) + }, + delete: (key) => { + collectionMetadataKeys.push(key) + params.metadata!.collection.delete(key) + }, + }, + }, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if ( + !gateNextApplicationReceipt && + applicationReceipts.length === 0 + ) { + return actualReceipt + } + const trace = { + controlsOwner, + returnedReceipt: `actual` as `actual` | `held`, + rowMetadataKeys: [...rowMetadataKeys], + collectionMetadataKeys: [...collectionMetadataKeys], + } + applicationReceipts.push(trace) + if (gateNextApplicationReceipt) { + gateNextApplicationReceipt = false + trace.returnedReceipt = `held` + firstReceiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return releaseFirstReceipt.promise + } + return actualReceipt + }, + }) + }, + }, + }) + let replacementReady: Promise | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `two-delta A lifecycle ready`, + ) + applicationReceipts.length = 0 + gateNextApplicationReceipt = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap-trigger`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstReceiptEntered.promise, + `two-delta first A receipt held`, + ) + expect(collection.get(`first`)?.title).toBe(`A first delta`) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + replacementControlsInstalled.promise, + `two-delta B controls installed`, + ) + + releaseFirstReceipt.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `two-delta B hydration held`, + ) + await Promise.resolve() + + const secondReceipts = applicationReceipts.filter( + (receipt) => + receipt.rowMetadataKeys.includes(`second`) || + receipt.collectionMetadataKeys.includes(`replay:second`), + ) + expect({ + controlsRun, + pullArguments, + secondReceipts, + secondRowMetadata: collection._state.syncedMetadata.get(`second`), + secondCollectionMetadata: + collection._state.syncedCollectionMetadata.get(`replay:second`), + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + controlsRun: 2, + pullArguments: [{ collectionId: id, fromRowVersion: 0 }], + secondReceipts: [], + secondRowMetadata: undefined, + secondCollectionMetadata: undefined, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstReceipt.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not confirm a delayed local coordinator response through a restarted owner`, async () => { + const id = `delayed-local-confirmation-lifecycle` + const adapter = createRecordingAdapter() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const requestEntered = createEventGate() + const releaseRequest = createEventGate() + coordinator.requestApplyLocalMutations = async ( + _collectionId, + mutations, + ) => { + requestEntered.resolve() + await releaseRequest.promise + return { + type: `rpc:applyLocalMutations:res`, + rpcId: `old-delayed-local`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: mutations.map((mutation) => mutation.mutationId), + } + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + persistence: { adapter, coordinator }, + }), + ) + const oldRow = { id: `old-local`, title: `must stay with old owner` } + let localTransaction: ReturnType | undefined + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `delayed local initial owner ready`, + ) + localTransaction = collection.insert(oldRow) + await atPersistedOracleCheckpoint( + requestEntered.promise, + `old local coordinator request entered`, + ) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseRequest.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after delayed local confirmation`, + ) + + expect({ + subsetCalls, + staleVisible: collection.get(oldRow.id), + staleDurable: adapter.rows.get(oldRow.id), + replacementPublicError: collection._lifecycle.getSyncError(), + }).toEqual({ + subsetCalls: 2, + staleVisible: undefined, + staleDurable: undefined, + replacementPublicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseRequest.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => localTransaction?.isPersisted.promise.catch(() => undefined), + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not admit an old pending user hook through a restarted coordinator`, async () => { + const id = `pending-user-hook-lifecycle` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const hookEntered = createEventGate() + const releaseHook = createEventGate() + let coordinatorRequests = 0 + coordinator.requestApplyLocalMutations = async ( + _collectionId, + mutations, + ) => { + coordinatorRequests++ + return { + type: `rpc:applyLocalMutations:res`, + rpcId: `stale-user-hook-request`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: mutations.map((mutation) => mutation.mutationId), + } + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + onInsert: async () => { + hookEntered.resolve() + await releaseHook.promise + return {} + }, + persistence: { adapter, coordinator }, + }), + ) + let transaction: ReturnType | undefined + let transactionOutcome: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `pending user hook initial owner ready`, + ) + transaction = collection.insert({ + id: `old-hook`, + title: `must not enter replacement coordinator`, + }) + transactionOutcome = transaction.isPersisted.promise.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + hookEntered.promise, + `old user hook entered before coordinator admission`, + ) + expect(coordinatorRequests).toBe(0) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReady, + `pending user hook replacement owner ready`, + ) + releaseHook.resolve() + const outcome = await atPersistedOracleCheckpoint( + transactionOutcome, + `pending user hook transaction settled`, + ) + + expect({ + replacementStatus: replacementOutcome.status, + transactionStatus: outcome.status, + coordinatorRequests, + durableOldRow: adapter.rows.get(`old-hook`), + }).toEqual({ + replacementStatus: `fulfilled`, + transactionStatus: `rejected`, + coordinatorRequests: 0, + durableOldRow: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHook.resolve() + await cleanupPersistedOracle( + [ + () => transactionOutcome, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let old loopback readiness complete a restarted hydration`, async () => { + const adapter = createRecordingAdapter() + const oldHydrationEntered = createEventGate() + const releaseOldHydration = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 1) { + oldHydrationEntered.resolve() + await releaseOldHydration.promise + return [] + } + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + const collection = createCollection( + persistedCollectionOptions({ + id: `loopback-late-readiness-lifecycle`, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + const oldReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + oldHydrationEntered.promise, + `old loopback hydration entered`, + ) + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + releaseOldHydration.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement loopback hydration entered`, + ) + await flushAsyncWork() + + expect({ + subsetCalls, + replacementIsReady: collection.isReady(), + replacementVisible: collection.get(`replacement`), + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + subsetCalls: 2, + replacementIsReady: false, + replacementVisible: undefined, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldHydration.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => oldReady, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`marks a targeted invalidation commit-receipt rejection terminal`, async () => { + const id = `targeted-invalidation-receipt` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const rejectedReceipt = createEventGate() + const receiptError = new Error( + `targeted invalidation receipt failed exactly`, + ) + let rejectNextCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!rejectNextCommit) return actualReceipt + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return rejectedReceipt.promise + }, + }), + }, + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `targeted receipt collection ready`, + ) + rejectNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `targeted-receipt-failure`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `targeted`, + value: { id: `targeted`, title: `publication attempted` }, + }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `targeted invalidation receipt entered`, + ) + rejectedReceipt.reject(receiptError) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === receiptError, + }).toEqual({ + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + rejectedReceipt.reject(receiptError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`waits for a reset truncate receipt before reloading`, async () => { + const id = `reset-truncate-receipt` + const seed = { id: `seed`, title: `must survive a failed reset` } + const adapter = createRecordingAdapter([seed]) + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const rejectedReceipt = createEventGate() + const receiptError = new Error(`reset truncate receipt failed exactly`) + let rejectNextCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!rejectNextCommit) return actualReceipt + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return rejectedReceipt.promise + }, + }), + }, + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reset receipt collection ready`, + ) + const baselineLoadCount = adapter.loadSubsetCalls.length + rejectNextCommit = true + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `reset truncate receipt entered`, + ) + await flushAsyncWork() + await flushAsyncWork() + + expect(adapter.loadSubsetCalls.length).toBe(baselineLoadCount) + + rejectedReceipt.reject(receiptError) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === receiptError, + }).toEqual({ + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + rejectedReceipt.reject(receiptError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`keeps local persistence pending through confirmation receipt settlement`, async () => { + const id = `local-confirmation-receipt` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const rejectedReceipt = createEventGate() + const receiptError = new Error(`local confirmation receipt failed exactly`) + let rejectNextCommit = false + coordinator.requestApplyLocalMutations = async ( + _collectionId, + mutations, + ) => { + rejectNextCommit = true + return { + type: `rpc:applyLocalMutations:res`, + rpcId: `local-confirmation`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: mutations.map((mutation) => mutation.mutationId), + } + } + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!rejectNextCommit) return actualReceipt + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return rejectedReceipt.promise + }, + }), + }, + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `local confirmation receipt collection ready`, + ) + const transaction = collection.insert({ + id: `local`, + title: `durable before confirmation settles`, + }) + const transactionOutcome = transaction.isPersisted.promise.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `local confirmation receipt entered`, + ) + const pending = observeSettlement( + transactionOutcome.then(() => undefined), + ) + await Promise.resolve() + expect(pending.read()).toEqual({ status: `pending` }) + + rejectedReceipt.reject(receiptError) + const outcome = await atPersistedOracleCheckpoint( + transactionOutcome, + `local confirmation transaction settled`, + ) + await flushAsyncWork() + + expect({ + transactionStatus: outcome.status, + transactionExact: + outcome.status === `rejected` && outcome.reason === receiptError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === receiptError, + }).toEqual({ + transactionStatus: `rejected`, + transactionExact: true, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + rejectedReceipt.reject(receiptError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`does not continue an old hydration into a replacement after its applied receipt settles`, async () => { + const id = `hydration-applied-receipt-lifecycle` + const adapter = createRecordingAdapter() + const receiptEntered = createEventGate() + const releaseReceipt = createEventGate() + let gateNextCommit = false + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 1) { + gateNextCommit = true + return [ + { + key: `old`, + value: { id: `old`, title: `old owner hydration` }, + }, + ] + } + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner hydration` }, + }, + ] + } + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!gateNextCommit) return actualReceipt + gateNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return releaseReceipt.promise + }, + }), + }, + }) + const oldReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `old hydration applied receipt entered`, + ) + expect(stripVirtualProps(collection.get(`old`))).toEqual({ + id: `old`, + title: `old owner hydration`, + }) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseReceipt.resolve() + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReady, + `replacement hydration after old applied receipt`, + ) + + expect({ + replacementStatus: replacementOutcome.status, + subsetCalls, + oldVisible: collection.get(`old`), + replacementVisible: stripVirtualProps(collection.get(`replacement`)), + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + replacementStatus: `fulfilled`, + subsetCalls: 2, + oldVisible: undefined, + replacementVisible: { + id: `replacement`, + title: `new owner hydration`, + }, + status: `ready`, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseReceipt.resolve() + await cleanupPersistedOracle( + [ + () => oldReady, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let an old post-receipt continuation flush replacement coordinator work`, async () => { + const id = `post-receipt-queue-lifecycle` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const releaseReceipt = createEventGate() + const replacementSourceStarted = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let gateNextCommit = false + let subsetCalls = 0 + let sourceRuns = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 1) { + gateNextCommit = true + return [ + { + key: `old`, + value: { id: `old`, title: `old owner hydration` }, + }, + ] + } + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner hydration` }, + }, + ] + } + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceRuns++ + if (sourceRuns === 2) replacementSourceStarted.resolve() + markReady() + }, + }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!gateNextCommit) return actualReceipt + gateNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return releaseReceipt.promise + }, + }), + }, + }) + const oldReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `old post-receipt hydration entered`, + ) + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + replacementSourceStarted.promise, + `replacement source subscribed behind old receipt`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `replacement-queued-work`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `queued`, + value: { id: `queued`, title: `belongs after replacement hydrate` }, + }, + ], + deletedKeys: [], + }) + + releaseReceipt.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after old receipt`, + ) + await flushAsyncWork() + + expect({ + sourceRuns, + subsetCalls, + queuedVisible: collection.get(`queued`), + }).toEqual({ + sourceRuns: 2, + subsetCalls: 2, + queuedVisible: undefined, + }) + + releaseReplacementHydration.resolve() + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReady, + `replacement hydration settled before coordinator work`, + ) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + replacementStatus: replacementOutcome.status, + oldVisible: collection.get(`old`), + replacementVisible: stripVirtualProps(collection.get(`replacement`)), + queuedVisible: stripVirtualProps(collection.get(`queued`)), + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + replacementStatus: `fulfilled`, + oldVisible: undefined, + replacementVisible: { + id: `replacement`, + title: `new owner hydration`, + }, + queuedVisible: { + id: `queued`, + title: `belongs after replacement hydrate`, + }, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseReceipt.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => oldReady, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects new source transactions after a terminal persistence failure`, async () => { + const adapter = createRecordingAdapter() + const adapterError = Object.assign( + new Error(`terminal adapter write failed exactly`), + { + code: `SQLITE_FULL`, + path: `adapter.applyCommittedTx`, + }, + ) + const successfulApply = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (collectionId, tx) => { + applyCalls++ + if (applyCalls === 1) throw adapterError + await successfulApply(collectionId, tx) + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-terminal-persistence-admission`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `terminal persistence admission collection ready`, + ) + remoteBegin!() + remoteWrite!({ + type: `insert`, + value: { id: `first`, title: `published before persistence` }, + }) + const firstOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit!()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `first terminal persistence receipt`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + + expect({ + receiptStatus: firstOutcome.status, + errorName: + terminalError instanceof Error ? terminalError.name : undefined, + cause: terminalError instanceof Error ? terminalError.cause : undefined, + publicErrorIsReceipt: + collection._lifecycle.getSyncError() === terminalError, + status: collection.status, + firstVisible: stripVirtualProps(collection.get(`first`)), + firstDurable: adapter.rows.get(`first`), + applyCalls, + }).toEqual({ + receiptStatus: `rejected`, + errorName: `PersistenceDurabilityError`, + cause: adapterError, + publicErrorIsReceipt: true, + status: `error`, + firstVisible: { + id: `first`, + title: `published before persistence`, + }, + firstDurable: undefined, + applyCalls: 1, + }) + + remoteBegin!() + remoteWrite!({ + type: `insert`, + value: { id: `late`, title: `must not be admitted` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit!()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `late terminal persistence receipt`, + ) + + expect({ + receiptStatus: lateOutcome.status, + exactReceipt: + lateOutcome.status === `rejected` && + lateOutcome.reason === terminalError, + publicErrorIsTerminal: + collection._lifecycle.getSyncError() === terminalError, + status: collection.status, + lateVisible: collection.get(`late`), + lateDurable: adapter.rows.get(`late`), + applyCalls, + }).toEqual({ + receiptStatus: `rejected`, + exactReceipt: true, + publicErrorIsTerminal: true, + status: `error`, + lateVisible: undefined, + lateDurable: undefined, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it.each([`hydration`, `durability`] as const)( + `fences metadata and truncate transactions after a terminal %s failure`, + async (failureKind) => { + const harness = await createTerminalFailureHarness( + failureKind, + `terminal-transaction-surface-${failureKind}`, + ) + const { + adapter, + collection, + coordinator, + seed, + sourceParams, + terminalError, + } = harness + const applyCallsBefore = adapter.applyCommittedTxCalls.length + const publishCallsBefore = coordinator.publishCalls.length + const operations: Array<{ name: string; apply: () => void }> = [ + { + name: `row metadata set`, + apply: () => + sourceParams.metadata!.row.set(seed.id, `must-not-replace-row`), + }, + { + name: `row metadata delete`, + apply: () => sourceParams.metadata!.row.delete(seed.id), + }, + { + name: `collection metadata set`, + apply: () => + sourceParams.metadata!.collection.set( + `resume`, + `must-not-replace-collection`, + ), + }, + { + name: `collection metadata delete`, + apply: () => sourceParams.metadata!.collection.delete(`resume`), + }, + { + name: `truncate`, + apply: () => sourceParams.truncate(), + }, + ] + let hasPrimaryFailure = false + + try { + for (const operation of operations) { + sourceParams.begin() + const operationOutcome = await Promise.resolve() + .then(operation.apply) + .then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const receiptOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${failureKind} ${operation.name} terminal receipt`, + ) + + expect({ + operation: operationOutcome.status, + receipt: receiptOutcome.status, + exactReceipt: + receiptOutcome.status === `rejected` && + receiptOutcome.reason === terminalError, + visibleSeed: stripVirtualProps(collection.get(seed.id)), + durableSeed: adapter.rows.get(seed.id), + rowMetadata: sourceParams.metadata!.row.get(seed.id), + collectionMetadata: sourceParams.metadata!.collection.get(`resume`), + applyCalls: adapter.applyCommittedTxCalls.length, + publishCalls: coordinator.publishCalls.length, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + }).toEqual({ + operation: `fulfilled`, + receipt: `rejected`, + exactReceipt: true, + visibleSeed: seed, + durableSeed: seed, + rowMetadata: `row-metadata-before-failure`, + collectionMetadata: `collection-metadata-before-failure`, + applyCalls: applyCallsBefore, + publishCalls: publishCallsBefore, + status: `error`, + exactPublicError: true, + }) + } + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }, + ) + + it(`fences reload, upstream hydration, and late readiness after terminal persistence failure`, async () => { + const harness = await createTerminalFailureHarness( + `durability`, + `terminal-persistence-reload-surfaces`, + ) + const { + adapter, + collection, + coordinator, + sourceParams, + terminalError, + upstreamLoads, + } = harness + const late = { id: `late-reload`, title: `must not hydrate` } + const loadCallsBefore = adapter.loadSubsetCalls.length + const upstreamCallsBefore = upstreamLoads.length + const publishCallsBefore = coordinator.publishCalls.length + adapter.loadSubset = (collectionId, options, context) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: context?.requiredIndexSignatures ?? [], + }) + return Promise.resolve([{ key: late.id, value: late }]) + } + let hasPrimaryFailure = false + + try { + const loadOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 3 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-persistence terminal loadSubset`, + ) + sourceParams.markReady() + await flushAsyncWork() + + expect({ + loadStatus: loadOutcome.status, + loadExact: + loadOutcome.status === `rejected` && + loadOutcome.reason === terminalError, + adapterLoads: adapter.loadSubsetCalls.length, + upstreamLoads: upstreamLoads.length, + publishCalls: coordinator.publishCalls.length, + visibleLate: collection.get(late.id), + durableLate: adapter.rows.get(late.id), + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + }).toEqual({ + loadStatus: `rejected`, + loadExact: true, + adapterLoads: loadCallsBefore, + upstreamLoads: upstreamCallsBefore, + publishCalls: publishCallsBefore, + visibleLate: undefined, + durableLate: undefined, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`rejects an already-admitted sibling when the preceding persistence fails`, async () => { + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const firstPersistence = createEventGate() + const adapterError = new Error(`first pending persistence failed exactly`) + const successfulApply = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await firstPersistence.promise + return + } + await successfulApply(...args) + } + const coordinator = createFailStopCoordinatorHarness( + `terminal-pending-sibling`, + ) + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-pending-sibling`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `pending sibling collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `a`, title: `first publishes before persistence` }, + }) + const firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first persistence entered before sibling admission`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `b`, title: `must not publish or persist` }, + }) + const secondReceipt = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const secondSettlement = observeSettlement( + secondReceipt.then( + () => undefined, + () => undefined, + ), + ) + expect(secondSettlement.read()).toEqual({ status: `pending` }) + + firstPersistence.reject(adapterError) + const firstOutcome = await atPersistedOracleCheckpoint( + firstReceipt, + `first failed persistence receipt`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + const secondOutcome = await atPersistedOracleCheckpoint( + secondReceipt, + `already-admitted sibling terminal receipt`, + ) + + expect({ + firstStatus: firstOutcome.status, + terminalName: + terminalError instanceof Error ? terminalError.name : undefined, + terminalCause: + terminalError instanceof Error ? terminalError.cause : undefined, + secondStatus: secondOutcome.status, + secondExact: + secondOutcome.status === `rejected` && + secondOutcome.reason === terminalError, + applyCalls, + firstVisible: stripVirtualProps(collection.get(`a`)), + firstDurable: adapter.rows.get(`a`), + secondVisible: collection.get(`b`), + secondDurable: adapter.rows.get(`b`), + publishCalls: coordinator.publishCalls.length, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + }).toEqual({ + firstStatus: `rejected`, + terminalName: `PersistenceDurabilityError`, + terminalCause: adapterError, + secondStatus: `rejected`, + secondExact: true, + applyCalls: 1, + firstVisible: { id: `a`, title: `first publishes before persistence` }, + firstDurable: undefined, + secondVisible: undefined, + secondDurable: undefined, + publishCalls: 0, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + firstPersistence.reject(adapterError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`marks direct adapter failure terminal and fences later local mutations`, async () => { + const adapter = createRecordingAdapter() + const adapterError = Object.assign( + new Error(`local adapter failed exactly`), + { + code: `SQLITE_IOERR`, + path: `adapter.applyCommittedTx`, + }, + ) + const successfulApply = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) throw adapterError + await successfulApply(...args) + } + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-loopback-local-durability`, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `local durability collection ready`, + ) + const firstTransaction = collection.insert({ + id: `first-local`, + title: `must roll back`, + }) + const firstOutcome = await atPersistedOracleCheckpoint( + firstTransaction.isPersisted.promise.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `first local durability failure`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + + const loadCallsBefore = adapter.loadSubsetCalls.length + adapter.loadSubset = (collectionId, options, context) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: context?.requiredIndexSignatures ?? [], + }) + return Promise.resolve([ + { + key: `forced-late`, + value: { id: `forced-late`, title: `must not hydrate` }, + }, + ]) + } + const forceOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection.utils.forceReloadSubset!({ limit: 4 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `force reload after local durability failure`, + ) + + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve() + .then(async () => { + const lateTransaction = collection.insert({ + id: `late-local`, + title: `must not be admitted`, + }) + await lateTransaction.isPersisted.promise + }) + .then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `late local terminal receipt`, + ) + + expect({ + firstStatus: firstOutcome.status, + errorName: + terminalError instanceof Error ? terminalError.name : undefined, + errorCause: + terminalError instanceof Error ? terminalError.cause : undefined, + errorCode: + terminalError instanceof Error + ? (terminalError as Error & { code?: unknown }).code + : undefined, + forceStatus: forceOutcome.status, + forceExact: + forceOutcome.status === `rejected` && + forceOutcome.reason === terminalError, + loadCalls: adapter.loadSubsetCalls.length, + forceVisible: collection.get(`forced-late`), + lateStatus: lateOutcome.status, + lateErrorName: + lateOutcome.status === `rejected` && + lateOutcome.reason instanceof Error + ? lateOutcome.reason.name + : undefined, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + applyCalls, + firstVisible: collection.get(`first-local`), + firstDurable: adapter.rows.get(`first-local`), + lateVisible: collection.get(`late-local`), + lateDurable: adapter.rows.get(`late-local`), + }).toEqual({ + firstStatus: `rejected`, + errorName: `PersistenceDurabilityError`, + errorCause: adapterError, + errorCode: `SQLITE_IOERR`, + forceStatus: `rejected`, + forceExact: true, + loadCalls: loadCallsBefore, + forceVisible: undefined, + lateStatus: `rejected`, + lateErrorName: `CollectionStateError`, + status: `error`, + exactPublicError: true, + applyCalls: 1, + firstVisible: undefined, + firstDurable: undefined, + lateVisible: undefined, + lateDurable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`fences direct acceptMutations and local mutations after hydration failure`, async () => { + const adapter = createRecordingAdapter() + const hydrationError = new Error(`loopback hydration failed exactly`) + adapter.loadSubset = () => Promise.reject(hydrationError) + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-loopback-hydration`, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await expect( + atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `loopback hydration rejected`, + ), + ).rejects.toBe(hydrationError) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationError) + + adapter.loadSubset = () => Promise.resolve([]) + const now = new Date() + const directRow = { id: `direct`, title: `must not be accepted` } + const directMutation: PendingMutation = { + mutationId: `direct-after-hydration`, + original: {}, + modified: directRow, + changes: directRow, + globalKey: `terminal-loopback-hydration:direct`, + key: directRow.id, + type: `insert`, + metadata: undefined, + syncMetadata: {}, + optimistic: true, + createdAt: now, + updatedAt: now, + collection, + } + const directOutcome = await atPersistedOracleCheckpoint( + Promise.resolve( + collection.utils.acceptMutations({ + mutations: [ + directMutation as unknown as PendingMutation< + Record + >, + ], + }), + ).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `direct acceptMutations after hydration failure`, + ) + const localOutcome = await Promise.resolve() + .then(() => + collection.insert({ + id: `late-local-hydration`, + title: `must not be admitted`, + }), + ) + .then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + expect({ + directStatus: directOutcome.status, + directExact: + directOutcome.status === `rejected` && + directOutcome.reason === hydrationError, + localStatus: localOutcome.status, + localErrorName: + localOutcome.status === `rejected` && + localOutcome.reason instanceof Error + ? localOutcome.reason.name + : undefined, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === hydrationError, + applyCalls: adapter.applyCommittedTxCalls.length, + directVisible: collection.get(directRow.id), + directDurable: adapter.rows.get(directRow.id), + localVisible: collection.get(`late-local-hydration`), + localDurable: adapter.rows.get(`late-local-hydration`), + }).toEqual({ + directStatus: `rejected`, + directExact: true, + localStatus: `rejected`, + localErrorName: `CollectionStateError`, + status: `error`, + exactPublicError: true, + applyCalls: 0, + directVisible: undefined, + directDurable: undefined, + localVisible: undefined, + localDurable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { + const adapter = createRecordingAdapter() + const ownership = { queryCollection: { owners: [`gc:q1`] } } + const sync: SyncConfig = { + sync: ({ begin, write, commit, markReady, metadata }) => { + begin() + metadata?.row.set(`remote-1`, ownership) + write({ + type: `insert`, + value: { + id: `remote-1`, + title: `From remote`, + }, + }) + commit() + markReady() + }, + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item: Todo) => item.id, + sync, + persistence: { + adapter, + }, + }), + ) + + await collection.stateWhenReady() + await flushAsyncWork() + + expect(adapter.rowMetadata.get(`remote-1`)).toEqual(ownership) + expect(collection._state.syncedMetadata.get(`remote-1`)).toEqual(ownership) + }) + + it(`resets stale row metadata for a metadata-less insert with no queued metadata`, async () => { + const adapter = createRecordingAdapter() + adapter.rowMetadata.set(`remote-1`, { stale: true }) + const sync: SyncConfig = { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { + id: `remote-1`, + title: `From remote`, + }, + }) + commit() markReady() }, } @@ -1167,77 +4839,486 @@ describe(`persistedCollectionOptions`, () => { }), ) - const preSyncIndex = collection.createIndex((row) => row.title, { - name: `pre-sync-title`, - }) - const expectedPreSyncSignature = collection.getIndexMetadata()[0]?.signature - - await collection.preload() - await flushAsyncWork() - - expect(expectedPreSyncSignature).toBeDefined() - expect( - adapter.ensureIndexCalls.some( - (call) => call.signature === expectedPreSyncSignature, - ), - ).toBe(true) - - const runtimeIndex = collection.createIndex((row) => row.id, { - name: `runtime-id`, - }) - await flushAsyncWork() + const preSyncIndex = collection.createIndex((row) => row.title, { + name: `pre-sync-title`, + }) + const expectedPreSyncSignature = collection.getIndexMetadata()[0]?.signature + + await collection.preload() + await flushAsyncWork() + + expect(expectedPreSyncSignature).toBeDefined() + expect( + adapter.ensureIndexCalls.some( + (call) => call.signature === expectedPreSyncSignature, + ), + ).toBe(true) + + const runtimeIndex = collection.createIndex((row) => row.id, { + name: `runtime-id`, + }) + await flushAsyncWork() + + const runtimeSignature = collection + .getIndexMetadata() + .find((index) => index.indexId === runtimeIndex.id)?.signature + expect(runtimeSignature).toBeDefined() + expect( + adapter.ensureIndexCalls.some( + (call) => call.signature === runtimeSignature, + ), + ).toBe(true) + + collection.removeIndex(preSyncIndex) + await flushAsyncWork() + expect( + adapter.markIndexRemovedCalls.some( + (call) => call.signature === expectedPreSyncSignature, + ), + ).toBe(true) + }) + + it(`queues remote sync writes that arrive during hydration`, async () => { + const adapter = createRecordingAdapter([ + { + id: `cached-1`, + title: `Cached row`, + }, + ]) + let resolveLoadSubset: (() => void) | undefined + adapter.loadSubset = async () => { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + return [ + { + key: `cached-1`, + value: { + id: `cached-1`, + title: `Cached row`, + }, + }, + ] + } + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => void) | undefined + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return {} + }, + }, + persistence: { + adapter, + }, + }), + ) + + const readyPromise = collection.stateWhenReady() + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } + + expect(resolveLoadSubset).toBeDefined() + expect(remoteBegin).toBeDefined() + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { + id: `during-hydrate`, + title: `During hydrate`, + }, + }) + remoteCommit?.() + + resolveLoadSubset?.() + await readyPromise + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`cached-1`))).toEqual({ + id: `cached-1`, + title: `Cached row`, + }) + expect(stripVirtualProps(collection.get(`during-hydrate`))).toEqual({ + id: `during-hydrate`, + title: `During hydrate`, + }) + }) + + it(`preserves a source transaction committed before the next persisted hydration`, async () => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `before-hydration-boundary`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + collection.startSyncImmediate() + await vi.waitFor(() => { + expect(remoteBegin).toBeTypeOf(`function`) + expect(remoteWrite).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + }) + const row = { id: `before`, title: `before` } + remoteBegin?.() + remoteWrite?.({ type: `insert`, value: row }) + const receipt = remoteCommit?.() + expect(receipt).toBeInstanceOf(Promise) + await receipt + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return Array.from(adapter.rows, ([key, value]) => ({ key, value })) + } + const rehydrating = Promise.resolve(collection._sync.loadSubset({})) + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `on-demand hydration entered`, + ) + + expect(stripVirtualProps(collection.get(`before`))).toEqual(row) + expect(adapter.rows.get(`before`)).toEqual(row) + hydration.resolve() + await atPersistedOracleCheckpoint( + rehydrating, + `on-demand hydration completed`, + ) + expect(stripVirtualProps(collection.get(`before`))).toEqual(row) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it.each([`during`, `straddles`, `after`] as const)( + `durably settles a source transaction that commits %s persisted hydration`, + async (position) => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + + const persistedTransactionOrder: Array = [] + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + const title = args[1].mutations.find( + (mutation) => mutation.type !== `delete`, + )?.value as Todo | undefined + if (title) persistedTransactionOrder.push(title.title) + await applyCommittedTx(...args) + } + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `hydration-boundary-${position}`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const ledger: Array = [] + const transactionId = position + const row = { id: `shared`, title: position } + let receipt: Promise | undefined + let hasPrimaryFailure = false + + const beginAndWrite = () => { + ledger.push({ type: `begin`, transactionId }) + remoteBegin?.() + ledger.push({ type: `write`, transactionId, row }) + remoteWrite?.({ type: `insert`, value: row }) + } + const commit = () => { + ledger.push({ type: `commit`, transactionId }) + const applied = remoteCommit?.() + expect(applied).toBeInstanceOf(Promise) + receipt = Promise.resolve(applied).then(() => undefined) + } + + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `${position} hydration entered`, + ) + expect(remoteBegin).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + + if (position === `during`) { + beginAndWrite() + commit() + hydration.resolve() + await atPersistedOracleCheckpoint( + ready, + `${position} hydration completed`, + ) + } else { + if (position === `straddles`) beginAndWrite() + hydration.resolve() + await atPersistedOracleCheckpoint( + ready, + `${position} hydration completed`, + ) + if (position === `after`) beginAndWrite() + commit() + } + + const settlement = observeSettlement(receipt!) + const checkpointId = `checkpoint-${position}` + const checkpointRow = { id: checkpointId, title: checkpointId } + ledger.push({ type: `begin`, transactionId: checkpointId }) + remoteBegin?.() + ledger.push({ + type: `write`, + transactionId: checkpointId, + row: checkpointRow, + }) + remoteWrite?.({ type: `insert`, value: checkpointRow }) + ledger.push({ type: `commit`, transactionId: checkpointId }) + const checkpointReceipt = remoteCommit?.() + expect(checkpointReceipt).toBeInstanceOf(Promise) + await atPersistedOracleCheckpoint( + Promise.resolve(checkpointReceipt), + `${position} later durable receipt`, + ) + + // A settled later durable receipt is the exact cut: FIFO forbids it + // from overtaking an earlier commit opened in the old phase. + expect(settlement.read()).toEqual({ status: `fulfilled` }) + const expected = foldDurabilityLedger(ledger) + expect(persistedTransactionOrder).toEqual(expected.commitOrder) + expect(stripVirtualProps(collection.get(`shared`))).toEqual(row) + expect(adapter.rows.get(`shared`)).toEqual(row) + expect(adapter.rows.get(checkpointId)).toEqual(checkpointRow) + + await collection.cleanup() + adapter.loadSubset = () => + Promise.resolve( + Array.from(adapter.rows, ([key, value]) => ({ key, value })), + ) + const reopened = createCollection( + persistedCollectionOptions({ + id: `hydration-boundary-${position}`, + getKey: (item) => item.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter }, + }), + ) + let hasReopenFailure = false + try { + await atPersistedOracleCheckpoint( + reopened.stateWhenReady(), + `${position} reopened hydration completed`, + ) + expect(stripVirtualProps(reopened.get(`shared`))).toEqual( + expected.committedRows.get(`shared`), + ) + } catch (error) { + hasReopenFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => reopened.cleanup()], + hasReopenFailure, + ) + } + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }, + ) + + it(`does not let a later same-key transaction overtake a hydration-straddling commit`, async () => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + const persistedTransactionOrder: Array = [] + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + const row = args[1].mutations.find( + (mutation) => mutation.type !== `delete`, + )?.value as Todo | undefined + if (row) persistedTransactionOrder.push(row.title) + await applyCommittedTx(...args) + } + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `hydration-straddle-sibling-fifo`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const ledger: Array = [] + let firstReceipt: Promise | undefined + let secondReceipt: Promise | undefined + let hasPrimaryFailure = false + + const sourceTransaction = (transactionId: string, title: string) => { + const row = { id: `shared`, title } + ledger.push({ type: `begin`, transactionId }) + remoteBegin?.() + ledger.push({ type: `write`, transactionId, row }) + remoteWrite?.({ type: `insert`, value: row }) + ledger.push({ type: `commit`, transactionId }) + return Promise.resolve(remoteCommit?.()).then(() => undefined) + } + + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `FIFO hydration entered`, + ) - const runtimeSignature = collection - .getIndexMetadata() - .find((index) => index.indexId === runtimeIndex.id)?.signature - expect(runtimeSignature).toBeDefined() - expect( - adapter.ensureIndexCalls.some( - (call) => call.signature === runtimeSignature, - ), - ).toBe(true) + const firstRow = { id: `shared`, title: `first` } + ledger.push({ type: `begin`, transactionId: `first` }) + remoteBegin?.() + ledger.push({ type: `write`, transactionId: `first`, row: firstRow }) + remoteWrite?.({ type: `insert`, value: firstRow }) + hydration.resolve() + await atPersistedOracleCheckpoint(ready, `FIFO hydration completed`) + ledger.push({ type: `commit`, transactionId: `first` }) + firstReceipt = Promise.resolve(remoteCommit?.()).then(() => undefined) + const firstSettlement = observeSettlement(firstReceipt) + + secondReceipt = sourceTransaction(`second`, `second`) + const secondSettlement = observeSettlement(secondReceipt) + await atPersistedOracleCheckpoint( + secondReceipt, + `later same-key receipt settled`, + ) - collection.removeIndex(preSyncIndex) - await flushAsyncWork() - expect( - adapter.markIndexRemovedCalls.some( - (call) => call.signature === expectedPreSyncSignature, - ), - ).toBe(true) + const expected = foldDurabilityLedger(ledger) + expect(persistedTransactionOrder).toEqual(expected.commitOrder) + expect(firstSettlement.read()).toEqual({ status: `fulfilled` }) + expect(secondSettlement.read()).toEqual({ status: `fulfilled` }) + expect(adapter.rows.get(`shared`)).toEqual( + expected.committedRows.get(`shared`), + ) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } }) - it(`queues remote sync writes that arrive during hydration`, async () => { - const adapter = createRecordingAdapter([ - { - id: `cached-1`, - title: `Cached row`, - }, - ]) - let resolveLoadSubset: (() => void) | undefined + it(`settles an aborted hydration-straddling commit without applying it`, async () => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [ - { - key: `cached-1`, - value: { - id: `cached-1`, - title: `Cached row`, - }, - }, - ] + hydrationEntered.resolve() + await hydration.promise + return [] } - let remoteBegin: (() => void) | undefined let remoteWrite: | ((message: { type: `insert`; value: Todo }) => void) | undefined - let remoteCommit: (() => void) | undefined - + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, + id: `aborted-hydration-straddle`, getKey: (item) => item.id, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -1248,44 +5329,152 @@ describe(`persistedCollectionOptions`, () => { }) => void remoteCommit = commit markReady() - return {} }, }, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) + const ledger: Array = [] + const controller = new AbortController() + let receipt: Promise | undefined + let hasPrimaryFailure = false - const readyPromise = collection.stateWhenReady() - for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { - await flushAsyncWork() + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `abort hydration entered`, + ) + ledger.push({ type: `begin`, transactionId: `aborted` }) + remoteBegin?.() + const row = { id: `aborted`, title: `must not apply` } + ledger.push({ type: `write`, transactionId: `aborted`, row }) + remoteWrite?.({ type: `insert`, value: row }) + controller.abort() + ledger.push({ type: `abort`, transactionId: `aborted` }) + hydration.resolve() + await atPersistedOracleCheckpoint(ready, `abort hydration completed`) + receipt = Promise.resolve(remoteCommit?.(controller.signal)).then( + () => undefined, + ) + await expect( + atPersistedOracleCheckpoint(receipt, `aborted receipt settled`), + ).rejects.toMatchObject({ name: `AbortError` }) + expect(foldDurabilityLedger(ledger).committedRows.size).toBe(0) + expect(collection.get(`aborted`)).toBeUndefined() + expect(adapter.rows.get(`aborted`)).toBeUndefined() + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) } + }) - expect(resolveLoadSubset).toBeDefined() - expect(remoteBegin).toBeDefined() + it(`rejects every buffered receipt with the exact persisted hydration failure`, async () => { + await runRejectedHydrationBufferWitness(`rejected-hydration-buffer-fixed`, [ + { id: `first`, title: `first buffered commit` }, + { id: `second`, title: `second buffered commit` }, + ]) + }) - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { - id: `during-hydrate`, - title: `During hydrate`, - }, - }) - remoteCommit?.() + it(`rejects a hydration-straddling transaction committed after hydration fails`, async () => { + await runRejectedHydrationBufferWitness( + `rejected-hydration-late-commit-fixed`, + [{ id: `late`, title: `must not cross a failed baseline` }], + { commitAfterHydrationFailure: true }, + ) + }) - resolveLoadSubset?.() - await readyPromise - await flushAsyncWork() + it.each([ + { + schedule: `one late commit`, + rows: [{ id: `one-late`, title: `` }], + }, + { + schedule: `two sibling late commits`, + rows: [ + { id: `late-a`, title: `a` }, + { id: `late-b`, title: `b` }, + ], + }, + { + schedule: `same-key late replacements`, + rows: [ + { id: `late-shared`, title: `old` }, + { id: `late-shared`, title: `new` }, + ], + }, + ])( + `rejects generated late commits after hydration failure: $schedule`, + async ({ schedule, rows }) => { + await runRejectedHydrationBufferWitness( + `rejected-hydration-late-${schedule.replaceAll(` `, `-`)}`, + rows, + { commitAfterHydrationFailure: true }, + ) + }, + ) - expect(stripVirtualProps(collection.get(`cached-1`))).toEqual({ - id: `cached-1`, - title: `Cached row`, - }) - expect(stripVirtualProps(collection.get(`during-hydrate`))).toEqual({ - id: `during-hydrate`, - title: `During hydrate`, + it.each([ + { + schedule: `one buffered commit`, + rows: [{ id: `one`, title: `one` }], + }, + { + schedule: `three sibling commits`, + rows: [ + { id: `a`, title: `a` }, + { id: `b`, title: `b` }, + { id: `c`, title: `c` }, + ], + }, + { + schedule: `same-key replacement commits`, + rows: [ + { id: `shared`, title: `old` }, + { id: `shared`, title: `new` }, + ], + }, + ])( + `rejects all receipts and adopts no rows after hydration failure: $schedule`, + async ({ schedule, rows }) => { + await runRejectedHydrationBufferWitness( + `rejected-hydration-${schedule.replaceAll(` `, `-`)}`, + rows, + ) + }, + ) + + it(`append-only transaction judgment rejects omission and reordering`, () => { + const events: Array = [ + { type: `begin`, transactionId: `first` }, + { + type: `write`, + transactionId: `first`, + row: { id: `shared`, title: `first` }, + }, + { type: `commit`, transactionId: `first` }, + { type: `begin`, transactionId: `second` }, + { + type: `write`, + transactionId: `second`, + row: { id: `shared`, title: `second` }, + }, + { type: `commit`, transactionId: `second` }, + ] + const expected = foldDurabilityLedger(events) + + expect(expected.commitOrder).toEqual([`first`, `second`]) + expect(expected.commitOrder).not.toEqual([`second`]) + expect(expected.commitOrder).not.toEqual([`second`, `first`]) + expect(expected.committedRows.get(`shared`)).toEqual({ + id: `shared`, + title: `second`, }) }) @@ -1400,44 +5589,483 @@ describe(`persistedCollectionOptions`, () => { }), ) - const readyPromise = collection.stateWhenReady() - for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + const readyOutcome = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `during-hydrate`, title: `During hydrate` }, + }) + const failingReceipt = remoteCommit?.() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `sibling`, title: `Sibling` }, + }) + const siblingReceipt = remoteCommit?.() + expect(failingReceipt).toBeInstanceOf(Promise) + expect(siblingReceipt).toBeInstanceOf(Promise) + const failingExpectation = expect( + Promise.resolve(failingReceipt), + ).rejects.toBe(replayError) + const siblingExpectation = expect( + Promise.resolve(siblingReceipt), + ).rejects.toBe(replayError) + + resolveLoadSubset?.() + const ready = await atPersistedOracleCheckpoint( + readyOutcome, + `replay-failed hydration readiness settled`, + ) + await failingExpectation + await siblingExpectation + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `after-replay-failure`, title: `must not be admitted` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit?.()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-replay-failure source receipt`, + ) + + expect({ + readyStatus: ready.status, + readyExact: ready.status === `rejected` && ready.reason === replayError, + lateStatus: lateOutcome.status, + lateExact: + lateOutcome.status === `rejected` && lateOutcome.reason === replayError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === replayError, + visibleLate: collection.get(`after-replay-failure`), + durableLate: adapter.rows.get(`after-replay-failure`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + readyStatus: `rejected`, + readyExact: true, + lateStatus: `rejected`, + lateExact: true, + status: `error`, + exactPublicError: true, + visibleLate: undefined, + durableLate: undefined, + applyCalls: 0, + }) + + await collection.cleanup() + }) + + it.each([`stream-position`, `collection-metadata`, `row-apply`] as const)( + `fail-stops startup when the %s phase fails`, + async (phase) => { + const phaseError = new Error(`${phase} failed exactly`) + const adapter = createRecordingAdapter( + phase === `row-apply` + ? [{ id: `row-apply-failure`, title: `must not apply` }] + : [], + ) + if (phase === `stream-position`) { + adapter.getStreamPosition = () => Promise.reject(phaseError) + } else if (phase === `collection-metadata`) { + adapter.loadCollectionMetadata = () => Promise.reject(phaseError) + } + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-startup-${phase}`, + getKey: (row) => { + if (phase === `row-apply` && row.id === `row-apply-failure`) { + throw phaseError + } + return row.id + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + const startupOutcome = await atPersistedOracleCheckpoint( + collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${phase} startup boundary settled`, + ) + + expect({ + startupStatus: startupOutcome.status, + startupExact: + startupOutcome.status === `rejected` && + startupOutcome.reason === phaseError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === phaseError, + visibleRows: Array.from(collection.values()).map(stripVirtualProps), + durableRows: Array.from(adapter.rows.values()), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + startupStatus: `rejected`, + startupExact: true, + status: `error`, + exactPublicError: true, + visibleRows: [], + durableRows: + phase === `row-apply` + ? [{ id: `row-apply-failure`, title: `must not apply` }] + : [], + applyCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }, + ) + + it(`fail-stops a coordinator transaction flushed after hydration`, async () => { + const adapter = createRecordingAdapter() + const hydrationEntered = createEventGate() + const hydration = createEventGate() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + const flushError = new Error(`coordinator hydration flush failed exactly`) + const coordinator = createFailStopCoordinatorHarness( + `terminal-coordinator-hydration-flush`, + ) + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-coordinator-hydration-flush`, + getKey: (row) => { + if (row.id === `coordinator-bad`) throw flushError + return row.id + }, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const readyOutcome = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `coordinator flush hydration entered`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `coordinator-flush-failure`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `coordinator-bad`, + value: { id: `coordinator-bad`, title: `must not publish` }, + }, + ], + deletedKeys: [], + }) + hydration.resolve() + const ready = await atPersistedOracleCheckpoint( + readyOutcome, + `coordinator flush readiness settled`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `late`, title: `must not be admitted` }, + }) + const late = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `coordinator flush late receipt`, + ) + + expect({ + readyStatus: ready.status, + readyExact: ready.status === `rejected` && ready.reason === flushError, + lateStatus: late.status, + lateExact: late.status === `rejected` && late.reason === flushError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === flushError, + visibleBad: collection.get(`coordinator-bad`), + visibleLate: collection.get(`late`), + durableLate: adapter.rows.get(`late`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + readyStatus: `rejected`, + readyExact: true, + lateStatus: `rejected`, + lateExact: true, + status: `error`, + exactPublicError: true, + visibleBad: undefined, + visibleLate: undefined, + durableLate: undefined, + applyCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`applies hydration-queued sync work after an unrelated local persistence rollback`, async () => { + const adapter = createRecordingAdapter() + const hydrationEntered = createEventGate() + const hydration = createEventGate() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + const appliedError = new Error(`hydration applied receipt failed exactly`) + const localPersistence = createEventGate() + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-hydration-applied-receipt`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const localTransaction = createTransaction({ + mutationFn: () => localPersistence.promise, + }) + const readyOutcome = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let remoteReceipt: true | Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `applied-receipt hydration entered`, + ) + localTransaction.mutate(() => { + collection.insert({ id: `local-gate`, title: `local pending` }) + }) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `remote-applied`, title: `applies after local rollback` }, + }) + remoteReceipt = sourceParams.commit() + expect(remoteReceipt).toBeInstanceOf(Promise) + hydration.resolve() await flushAsyncWork() + localPersistence.reject(appliedError) + + const ready = await atPersistedOracleCheckpoint( + readyOutcome, + `applied-receipt readiness settled`, + ) + const remote = await atPersistedOracleCheckpoint( + Promise.resolve(remoteReceipt).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `hydration-applied remote receipt settled`, + ) + + expect({ + readyStatus: ready.status, + remoteStatus: remote.status, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + durableRemote: adapter.rows.get(`remote-applied`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + readyStatus: `fulfilled`, + remoteStatus: `fulfilled`, + status: `ready`, + publicError: undefined, + durableRemote: { + id: `remote-applied`, + title: `applies after local rollback`, + }, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + localPersistence.reject(appliedError) + await localTransaction.isPersisted.promise.catch(() => undefined) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) } + }) - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { id: `during-hydrate`, title: `During hydrate` }, - }) - const failingReceipt = remoteCommit?.() - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { id: `sibling`, title: `Sibling` }, - }) - const siblingReceipt = remoteCommit?.() - expect(failingReceipt).toBeInstanceOf(Promise) - expect(siblingReceipt).toBeInstanceOf(Promise) - const failingExpectation = expect( - Promise.resolve(failingReceipt), - ).rejects.toBe(replayError) - const siblingExpectation = expect( - Promise.resolve(siblingReceipt), - ).rejects.toBe(replayError) + it.each([ + { + label: `immediate source transaction`, + options: { immediate: true }, + expectedVisibleBeforeLocalSettlement: true, + expectedApplyCallsBeforeLocalSettlement: 2, + }, + { + label: `non-immediate control`, + options: undefined, + expectedVisibleBeforeLocalSettlement: false, + expectedApplyCallsBeforeLocalSettlement: 1, + }, + ] as const)( + `preserves $label admission through the persisted wrapper`, + async ({ + options, + expectedVisibleBeforeLocalSettlement, + expectedApplyCallsBeforeLocalSettlement, + }) => { + const adapter = createRecordingAdapter() + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `begin-options-${options?.immediate ? `immediate` : `normal`}`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const localPersistence = createEventGate() + const localTransaction = createTransaction({ + autoCommit: false, + mutationFn: () => localPersistence.promise, + }) + let localCommit: Promise | undefined + let remoteReceipt: true | Promise | undefined + let hasPrimaryFailure = false - resolveLoadSubset?.() - await readyPromise - await failingExpectation - await siblingExpectation + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `begin-options collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `seed`, title: `baseline` }, + }) + await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()), + `begin-options seed persisted`, + ) - await collection.cleanup() - }) + localTransaction.mutate(() => { + collection.update(`seed`, (draft) => { + draft.title = `local pending` + }) + }) + localCommit = localTransaction.commit() + void localCommit.catch(() => undefined) + expect(localTransaction.state).toBe(`persisting`) + + sourceParams.begin(options) + sourceParams.write({ + type: `insert`, + value: { id: `remote`, title: `source commit` }, + }) + remoteReceipt = sourceParams.commit() + expect(remoteReceipt).toBeInstanceOf(Promise) + + expect({ + localState: localTransaction.state, + remoteVisible: collection.has(`remote`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + localState: `persisting`, + remoteVisible: expectedVisibleBeforeLocalSettlement, + applyCalls: expectedApplyCallsBeforeLocalSettlement, + }) + + if (options?.immediate) { + await atPersistedOracleCheckpoint( + Promise.resolve(remoteReceipt), + `immediate source receipt before local settlement`, + ) + expect(localTransaction.state).toBe(`persisting`) + } else { + const settlement = observeSettlement( + Promise.resolve(remoteReceipt).then(() => undefined), + ) + await Promise.resolve() + expect(settlement.read()).toEqual({ status: `pending` }) + } + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + localPersistence.resolve() + await cleanupPersistedOracle( + [ + () => localCommit, + () => + remoteReceipt === true + ? undefined + : Promise.resolve(remoteReceipt), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }, + ) - it(`marks ready even when persisted startup fails before markReady`, async () => { + it(`marks the collection errored with the exact persisted startup hydration failure`, async () => { const adapter = createRecordingAdapter() + const startupError = new Error(`startup hydration failed exactly`) adapter.loadSubset = async () => { - throw new Error(`startup failure`) + throw startupError } const collection = createCollection( @@ -1456,9 +6084,22 @@ describe(`persistedCollectionOptions`, () => { }), ) - await collection.stateWhenReady() - await flushAsyncWork() - expect(collection.status).toBe(`ready`) + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + let hasPrimaryFailure = false + try { + await expect(collection.stateWhenReady()).rejects.toBe(startupError) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(startupError) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + warning.mockRestore() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } }) it(`reads staged metadata writes during hydration-queued transactions`, async () => { @@ -1609,40 +6250,502 @@ describe(`persistedCollectionOptions`, () => { title: `Post truncate`, }, }) - remoteCommit?.() - await flushAsyncWork() - - expect(adapter.applyCommittedTxCalls.at(-1)?.tx.truncate).toBe(true) + remoteCommit?.() + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls.at(-1)?.tx.truncate).toBe(true) + + const reloadedCollection = createCollection( + persistedCollectionOptions({ + id: `sync-present-truncate`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + }, + }), + ) + + await reloadedCollection.preload() + await flushAsyncWork() + + expect(reloadedCollection.get(`pre-truncate`)).toBeUndefined() + expect(stripVirtualProps(reloadedCollection.get(`post-truncate`))).toEqual({ + id: `post-truncate`, + title: `Post truncate`, + }) + expect( + reloadedCollection._state.syncedCollectionMetadata.get(`electric:resume`), + ).toEqual({ + kind: `reset`, + updatedAt: 1, + }) + }) + + it(`fail-stops an OK seq-gap delta when its application receipt rejects`, async () => { + const id = `gap-targeted-receipt` + const receiptError = new Error(`gap targeted receipt failed exactly`) + const firstApplied = createEventGate() + const receiptEntered = createEventGate() + const terminal = createEventGate() + const fallbackReload = createEventGate() + const barrierApplied = createEventGate() + let loadCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + loadCalls++ + if (loadCalls > 1) fallbackReload.resolve() + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => ({ + type: `rpc:pullSince:res`, + rpcId: `targeted-replay`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: false, + changedKeys: [`replayed`], + deletedKeys: [], + deltas: [ + { + txId: `replayed-delta`, + latestRowVersion: 2, + changedRows: [ + { + key: `replayed`, + value: { id: `replayed`, title: `targeted replay` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [], + collectionMetadataMutations: [], + }, + ], + }) + let rejectNextCommit = false + let observeNextCommit = false + let observeBarrierCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (rejectNextCommit) { + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return Promise.reject(receiptError) + } + if (observeNextCommit) { + observeNextCommit = false + void Promise.resolve(actualReceipt).then(() => + firstApplied.resolve(), + ) + } + if (observeBarrierCommit) { + observeBarrierCommit = false + void Promise.resolve(actualReceipt).then(() => + barrierApplied.resolve(), + ) + } + return actualReceipt + }, + }), + }, + }) + const unsubscribeError = collection.on(`status:error`, () => { + terminal.resolve() + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `targeted gap initial ready`, + ) + observeNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `first`, value: { id: `first`, title: `first committed` } }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstApplied.promise, + `targeted gap first transaction applied`, + ) + + rejectNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `targeted replay receipt entered`, + ) + const outcome = await atPersistedOracleCheckpoint( + Promise.race([ + terminal.promise.then(() => ({ kind: `terminal` as const })), + fallbackReload.promise.then(() => ({ kind: `fallback` as const })), + ]), + `targeted terminal error or fallback reload`, + ) + if (outcome.kind === `fallback`) { + observeBarrierCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 4, + txId: `barrier`, + latestRowVersion: 4, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + barrierApplied.promise, + `targeted fallback barrier`, + ) + } + + expect({ + outcome: outcome.kind, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + loadCalls, + }).toEqual({ + outcome: `terminal`, + status: `error`, + publicError: receiptError, + loadCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + unsubscribeError() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`fail-stops an OK seq-gap full reload when adapter loading rejects`, async () => { + const id = `gap-full-reload-load` + const loadError = new Error(`gap full reload load failed exactly`) + const firstApplied = createEventGate() + const failingLoadEntered = createEventGate() + const terminal = createEventGate() + const fallbackReload = createEventGate() + const barrierApplied = createEventGate() + let loadCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + loadCalls++ + if (loadCalls === 2) { + failingLoadEntered.resolve() + throw loadError + } + if (loadCalls === 3) fallbackReload.resolve() + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => ({ + type: `rpc:pullSince:res`, + rpcId: `full-reload`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: true, + }) + let observeNextCommit = false + let observeBarrierCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (observeNextCommit) { + observeNextCommit = false + void Promise.resolve(actualReceipt).then(() => + firstApplied.resolve(), + ) + } + if (observeBarrierCommit) { + observeBarrierCommit = false + void Promise.resolve(actualReceipt).then(() => + barrierApplied.resolve(), + ) + } + return actualReceipt + }, + }), + }, + }) + const unsubscribeError = collection.on(`status:error`, () => { + terminal.resolve() + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `full reload gap initial ready`, + ) + observeNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `first`, value: { id: `first`, title: `first committed` } }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstApplied.promise, + `full reload gap first transaction applied`, + ) + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + failingLoadEntered.promise, + `full reload gap failing load entered`, + ) + const outcome = await atPersistedOracleCheckpoint( + Promise.race([ + terminal.promise.then(() => ({ kind: `terminal` as const })), + fallbackReload.promise.then(() => ({ kind: `fallback` as const })), + ]), + `full reload terminal error or second fallback reload`, + ) + if (outcome.kind === `fallback`) { + observeBarrierCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 4, + txId: `barrier`, + latestRowVersion: 4, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + barrierApplied.promise, + `full reload fallback barrier`, + ) + } + + expect({ + outcome: outcome.kind, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + loadCalls, + }).toEqual({ + outcome: `terminal`, + status: `error`, + publicError: loadError, + loadCalls: 2, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + unsubscribeError() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`keeps seq-gap pull transport rejection on the fallback path`, async () => { + const id = `gap-transport-control` + const transportError = new Error(`gap pull transport failed exactly`) + const firstApplied = createEventGate() + const fallbackReload = createEventGate() + const barrierApplied = createEventGate() + let loadCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + loadCalls++ + if (loadCalls === 2) fallbackReload.resolve() + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => { + throw transportError + } + let observeNextCommit = false + let observeBarrierCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (observeNextCommit) { + observeNextCommit = false + void Promise.resolve(actualReceipt).then(() => + firstApplied.resolve(), + ) + } + if (observeBarrierCommit) { + observeBarrierCommit = false + void Promise.resolve(actualReceipt).then(() => + barrierApplied.resolve(), + ) + } + return actualReceipt + }, + }), + }, + }) + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + let hasPrimaryFailure = false - const reloadedCollection = createCollection( - persistedCollectionOptions({ - id: `sync-present-truncate`, - getKey: (item) => item.id, - sync: { - sync: ({ markReady }) => { - markReady() - }, - }, - persistence: { - adapter, - }, - }), - ) + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `transport gap initial ready`, + ) + observeNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `first`, value: { id: `first`, title: `first committed` } }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstApplied.promise, + `transport gap first transaction applied`, + ) - await reloadedCollection.preload() - await flushAsyncWork() + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + fallbackReload.promise, + `transport gap fallback reload`, + ) + observeBarrierCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 4, + txId: `barrier`, + latestRowVersion: 4, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + barrierApplied.promise, + `transport fallback barrier`, + ) - expect(reloadedCollection.get(`pre-truncate`)).toBeUndefined() - expect(stripVirtualProps(reloadedCollection.get(`post-truncate`))).toEqual({ - id: `post-truncate`, - title: `Post truncate`, - }) - expect( - reloadedCollection._state.syncedCollectionMetadata.get(`electric:resume`), - ).toEqual({ - kind: `reset`, - updatedAt: 1, - }) + expect(warning).toHaveBeenCalledWith( + `Failed pullSince recovery attempt:`, + transportError, + ) + expect({ + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + loadCalls, + }).toEqual({ + status: `ready`, + publicError: undefined, + loadCalls: 2, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + warning.mockRestore() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } }) it(`uses pullSince recovery when tx sequence gaps are detected`, async () => { @@ -2069,6 +7172,295 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`marks the collection errored with the exact later on-demand hydration failure`, async () => { + const hydrationFailure = new Error(`later on-demand hydration failed`) + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `later-on-demand-hydration-failure`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `on-demand collection initially ready`, + ) + expect(collection.status).toBe(`ready`) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + `later on-demand hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationFailure) + expect([...collection.values()]).toEqual([]) + expect(adapter.rows.size).toBe(0) + expect(adapter.applyCommittedTxCalls).toEqual([]) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`marks the collection errored and rejects later forced work after a hydration failure`, async () => { + const hydrationFailure = new Error(`later forced hydration failed`) + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `later-forced-hydration-failure`, + getKey: (row) => row.id, + syncMode: `on-demand`, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `forced-hydration collection initially ready`, + ) + expect(collection.status).toBe(`ready`) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection.utils.forceReloadSubset!({ limit: 1 })), + `later forced hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationFailure) + expect([...collection.values()]).toEqual([]) + expect(adapter.rows.size).toBe(0) + expect(adapter.applyCommittedTxCalls).toEqual([]) + + let laterAdapterCalls = 0 + adapter.loadSubset = () => { + laterAdapterCalls++ + return Promise.resolve([ + { + key: `late-forced`, + value: { id: `late-forced`, title: `must not be admitted` }, + }, + ]) + } + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection.utils.forceReloadSubset!({ limit: 2 })), + `later forced subset work settled`, + ), + ).rejects.toBe(hydrationFailure) + expect({ + laterAdapterCalls, + visible: collection.get(`late-forced`), + durable: adapter.rows.get(`late-forced`), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + laterAdapterCalls: 0, + visible: undefined, + durable: undefined, + durabilityCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`rejects subset work admitted after a terminal hydration failure`, async () => { + const hydrationFailure = new Error(`terminal hydration failed`) + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-hydration-rejects-later-work`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `terminal-hydration collection initially ready`, + ) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + `terminal hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + expect(collection.status).toBe(`error`) + + let laterAdapterCalls = 0 + adapter.loadSubset = () => { + laterAdapterCalls++ + return Promise.resolve([ + { + key: `late`, + value: { id: `late`, title: `must not be admitted` }, + }, + ]) + } + const laterOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 2 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `later subset work settled`, + ) + + expect({ + status: laterOutcome.status, + exactReason: + laterOutcome.status === `rejected` && + laterOutcome.reason === hydrationFailure, + laterAdapterCalls, + visible: collection.get(`late`), + durable: adapter.rows.get(`late`), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + status: `rejected`, + exactReason: true, + laterAdapterCalls: 0, + visible: undefined, + durable: undefined, + durabilityCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`rejects new source transactions after a terminal hydration failure`, async () => { + const hydrationFailure = new Error(`terminal source hydration failed`) + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-hydration-rejects-new-source-work`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `terminal source-hydration collection initially ready`, + ) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + `terminal source hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationFailure) + + remoteBegin!() + remoteWrite!({ + type: `insert`, + value: { id: `late`, title: `must not be admitted` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit!()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `late terminal hydration source receipt`, + ) + + expect({ + receiptStatus: lateOutcome.status, + exactReceipt: + lateOutcome.status === `rejected` && + lateOutcome.reason === hydrationFailure, + publicErrorIsHydration: + collection._lifecycle.getSyncError() === hydrationFailure, + status: collection.status, + lateVisible: collection.get(`late`), + lateDurable: adapter.rows.get(`late`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + receiptStatus: `rejected`, + exactReceipt: true, + publicErrorIsHydration: true, + status: `error`, + lateVisible: undefined, + lateDurable: undefined, + applyCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + it.each([`throw`, `reject`] as const)( `releases only transferred upstream ownership after a load %s`, async (mode) => { @@ -2521,6 +7913,270 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`fail-stops queued and later work when reset reload fails after truncation`, async () => { + const seed = { id: `seed`, title: `durable baseline` } + const adapter = createRecordingAdapter([seed]) + const coordinator = createFailStopCoordinatorHarness( + `terminal-reset-reload-failure`, + ) + const reloadEntered = createEventGate() + const reload = createEventGate() + const reloadError = new Error(`reset reload failed exactly`) + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-reset-reload-failure`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + let queuedReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reset reload collection ready`, + ) + expect(stripVirtualProps(collection.get(seed.id))).toEqual(seed) + adapter.loadSubset = async () => { + reloadEntered.resolve() + await reload.promise + throw reloadError + } + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + await atPersistedOracleCheckpoint( + reloadEntered.promise, + `reset reload entered after memory truncate`, + ) + expect(collection.get(seed.id)).toBeUndefined() + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `queued`, title: `must reject with reset failure` }, + }) + queuedReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void queuedReceipt.catch(() => undefined) + const queuedSettlement = observeSettlement(queuedReceipt) + expect(queuedSettlement.read()).toEqual({ status: `pending` }) + + reload.reject(reloadError) + await flushAsyncWork() + await flushAsyncWork() + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `late`, title: `must not continue after reset failure` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-reset-reload-failure receipt`, + ) + const queuedOutcome = queuedSettlement.read() + + expect({ + queuedStatus: queuedOutcome.status, + queuedExact: + queuedOutcome.status === `rejected` && + queuedOutcome.reason === reloadError, + lateStatus: lateOutcome.status, + lateExact: + lateOutcome.status === `rejected` && + lateOutcome.reason === reloadError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === reloadError, + visibleSeed: collection.get(seed.id), + durableSeed: adapter.rows.get(seed.id), + visibleQueued: collection.get(`queued`), + durableQueued: adapter.rows.get(`queued`), + visibleLate: collection.get(`late`), + durableLate: adapter.rows.get(`late`), + applyCalls: adapter.applyCommittedTxCalls.length, + publishCalls: coordinator.publishCalls.length, + }).toEqual({ + queuedStatus: `rejected`, + queuedExact: true, + lateStatus: `rejected`, + lateExact: true, + status: `error`, + exactPublicError: true, + visibleSeed: undefined, + durableSeed: seed, + visibleQueued: undefined, + durableQueued: undefined, + visibleLate: undefined, + durableLate: undefined, + applyCalls: 0, + publishCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + reload.reject(reloadError) + warning.mockRestore() + await cleanupPersistedOracle( + [ + () => collection.cleanup(), + () => queuedReceipt?.catch(() => undefined), + ], + hasPrimaryFailure, + ) + } + }) + + it(`fences coordinator delivery after terminal failure while allowing release and cleanup`, async () => { + const harness = await createTerminalFailureHarness( + `durability`, + `terminal-coordinator-delivery-fence`, + ) + const { + adapter, + collection, + coordinator, + initialLease, + terminalError, + upstreamLoads, + upstreamUnloads, + } = harness + const adapterLoadsBefore = adapter.loadSubsetCalls.length + const ensureCallsBefore = coordinator.remoteEnsureCalls.length + const upstreamLoadsBefore = upstreamLoads.length + const applyCallsBefore = adapter.applyCommittedTxCalls.length + const publishCallsBefore = coordinator.publishCalls.length + adapter.loadSubset = (collectionId, options, context) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: context?.requiredIndexSignatures ?? [], + }) + return Promise.resolve([ + { + key: `coordinator-late`, + value: { id: `coordinator-late`, title: `must not arrive` }, + }, + ]) + } + let cleanedUp = false + let hasPrimaryFailure = false + + try { + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `post-terminal-targeted`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `targeted-late`, + value: { id: `targeted-late`, title: `must not arrive` }, + }, + ], + deletedKeys: [], + }) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `post-terminal-full-reload`, + latestRowVersion: 2, + requiresFullReload: true, + }) + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 2, + }) + const remoteEnsureOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 9 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-terminal remote ensure delivery`, + ) + await flushAsyncWork() + await flushAsyncWork() + await flushAsyncWork() + + const beforeCleanup = { + targetedVisible: collection.get(`targeted-late`), + reloadVisible: collection.get(`coordinator-late`), + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + } + + collection._sync.unloadSubset(initialLease) + const unloadCount = upstreamUnloads.length + await collection.cleanup() + cleanedUp = true + + expect({ + remoteEnsureStatus: remoteEnsureOutcome.status, + remoteEnsureExact: + remoteEnsureOutcome.status === `rejected` && + remoteEnsureOutcome.reason === terminalError, + adapterLoads: adapter.loadSubsetCalls.length, + ensureCalls: coordinator.remoteEnsureCalls.length, + upstreamLoads: upstreamLoads.length, + unloadCount, + unsubscribeCalls: coordinator.unsubscribeCalls, + applyCalls: adapter.applyCommittedTxCalls.length, + publishCalls: coordinator.publishCalls.length, + targetedVisible: beforeCleanup.targetedVisible, + reloadVisible: beforeCleanup.reloadVisible, + targetedDurable: adapter.rows.get(`targeted-late`), + reloadDurable: adapter.rows.get(`coordinator-late`), + status: beforeCleanup.status, + exactPublicError: beforeCleanup.exactPublicError, + }).toEqual({ + remoteEnsureStatus: `rejected`, + remoteEnsureExact: true, + adapterLoads: adapterLoadsBefore, + ensureCalls: ensureCallsBefore, + upstreamLoads: upstreamLoadsBefore, + unloadCount: 1, + unsubscribeCalls: 1, + applyCalls: applyCallsBefore, + publishCalls: publishCallsBefore, + targetedVisible: undefined, + reloadVisible: undefined, + targetedDurable: undefined, + reloadDurable: undefined, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => (cleanedUp ? undefined : collection.cleanup())], + hasPrimaryFailure, + ) + } + }) + it(`keeps a hydrated resume baseline across narrow full reloads`, async () => { const adapter = createRecordingAdapter([ { id: `1`, title: `Narrow` }, diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index dd1d2d5848..b4544fdd8d 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1935,6 +1935,12 @@ function createElectricSync>( !isResettingSnapshot const bufferedMessages: Array> = [] // Buffer change messages during initial sync + // Presence belongs to the source transaction, which can span multiple + // ShapeStream callbacks. Rebuild it only when no transaction is open; + // otherwise later callbacks must see writes staged by earlier ones. + const pendingPresence = new Map() + let usesBaseline = true + // Track keys that have been synced to handle overlapping subset queries. // When multiple subset queries return the same row, the server sends `insert` // for each response. We convert subsequent inserts to updates to avoid @@ -2102,6 +2108,8 @@ function createElectricSync>( begin() transactionStarted = true truncate() + pendingPresence.clear() + usesBaseline = false syncedKeys.clear() clearTagTrackingState() isResettingSnapshot = true @@ -2111,23 +2119,25 @@ function createElectricSync>( // Applied rows can also arrive through persistence invalidations. // Overlay only unapplied writes, once per callback rather than once // per message. A queued truncate fences off the previous snapshot. - const pendingPresence = new Map() - let usesBaseline = true - for (const pending of collection._state.pendingSyncedTransactions) { - if (pending.truncate) { - pendingPresence.clear() - usesBaseline = false - } - for (const operation of pending.operations) { - pendingPresence.set(operation.key, operation.type !== `delete`) + if (!transactionStarted) { + pendingPresence.clear() + usesBaseline = true + for (const pending of collection._state.pendingSyncedTransactions) { + if (pending.truncate) { + pendingPresence.clear() + usesBaseline = false + } + for (const operation of pending.operations) { + pendingPresence.set(operation.key, operation.type !== `delete`) + } } - } - for (const message of bufferedMessages) { - if (isChangeMessage(message)) { - pendingPresence.set( - collection.getKeyFromItem(message.value), - message.headers.operation !== `delete`, - ) + for (const message of bufferedMessages) { + if (isChangeMessage(message)) { + pendingPresence.set( + collection.getKeyFromItem(message.value), + message.headers.operation !== `delete`, + ) + } } } @@ -2378,7 +2388,15 @@ function createElectricSync>( void applied.then( () => wrappedMarkReady(wasBufferingInitialSync, readyErrorVersion), - () => undefined, + (error: unknown) => { + if (!isActiveLifecycle() || abortController.signal.aborted) { + return + } + streamErrorVersion++ + unsubscribeStream() + abortController.abort() + if (collection.status !== `error`) markError(error) + }, ) } diff --git a/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts b/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts index e2e63cef03..e8dd6274ac 100644 --- a/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts +++ b/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts @@ -10,6 +10,26 @@ import type { Message } from '@electric-sql/client' import type { PersistenceAdapter } from '../../db-sqlite-persistence-core/src' import type { ElectricCollectionUtils } from '../src/electric' +/** + * # Can reused Electric descriptors keep independent owners and tag state? + * + * One descriptor may create several Collections, but each Collection must own + * its sync run, acknowledgement waiters, persisted tag membership, and cleanup. + * A compatible resume restores selected tags; a fresh snapshot replaces them. + * A `move-out` removes a row only after its modeled tag membership is empty. + * + * Plain Maps and tag sets form the independent model. Generated histories vary + * descriptor form, sync mode, warm and cold restart, interrupted recovery, + * edits, and tag removals. The driver uses the real Collection, Electric + * adapter, persisted wrapper, and a controlled ShapeStream SDK boundary. + * Checkpoints compare coherent public snapshots, durable rows, recovery traces, + * acknowledgement ownership, and unsubscribe calls. + * + * Fixed and random campaigns retain replay inputs through the shared oracle + * configuration. The controlled stream does not establish live Electric HTTP + * framing or service behavior; those have separate owners. + */ + type TestRow = { id: number; name: string; stable: string } type TagExposure = { cut: string; rows: Array } @@ -214,8 +234,12 @@ async function runTagHistory(history: { })) const durableRows = () => [...rows.values()].map((entry) => entry.value) const check = async () => { - expect(publicRows(), `cold=${cold}, fresh=${fresh}`).toEqual( - expectedRows(), + await vi.waitFor( + () => + expect(publicRows(), `cold=${cold}, fresh=${fresh}`).toEqual( + expectedRows(), + ), + { interval: 1 }, ) await vi.waitFor(() => expect(durableRows()).toEqual(expectedRows()), { interval: 1, @@ -646,8 +670,7 @@ it.each([`resume`, `fresh`] as const)( if (restart === `fresh`) streams[1]!.send([insert(1, currentTag), upToDate]) streams[1]!.send([moveOut(currentTag), upToDate]) - await vi.waitFor(() => expect(collection.status).toBe(`ready`)) - expect(collection.has(1)).toBe(false) + await vi.waitFor(() => expect(collection.has(1)).toBe(false)) await vi.waitFor(() => expect(rows.has(1)).toBe(false)) } finally { await collection.cleanup() diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index eb208a5d56..a5322b111e 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -4,11 +4,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { IR, createCollection, createTransaction } from '@tanstack/db' import { ShapeStream } from '@electric-sql/client' import { QueryClient } from '@tanstack/query-core' -import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' +import { + PersistenceDurabilityError, + persistedCollectionOptions, +} from '../../db-sqlite-persistence-core/src' import { queryCollectionOptions } from '../../query-db-collection/src/query' import { electricCollectionOptions } from '../src/electric' import { oraclePropertyOptions, oracleRuns } from '../../db/tests/oracle-config' -import { withElectricCleanup } from './electric-oracle-lifecycle' +import { atCheckpoint, withElectricCleanup } from './electric-oracle-lifecycle' import type { Collection, SyncMetadataApi } from '@tanstack/db' import type { ChangeMessage, Message, Offset, Row } from '@electric-sql/client' import type { @@ -23,22 +26,23 @@ import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' * Electric delivers change messages, reset controls, offsets, handles, and * snapshot availability through an SDK stream. The adapter must publish atomic * callback cuts, wait for applied receipts before readiness, persist valid - * resume evidence, reject unseen partial updates, and keep stale callbacks and - * metadata scoped to the lifecycle that created them. + * resume evidence for a valid source prefix, reject unseen partial updates, + * and keep stale callbacks and metadata scoped to the sync run that created + * them. * - * Independent Maps model rows, selected-tag membership, resume evidence, and - * persisted state. A controlled ShapeStream boundary supplies authored message - * partitions and settlement order. The production Collection, Query-backed - * persistence wrapper, metadata APIs, and cleanup all run unchanged. Checks - * cover rows, exact batches, readiness, errors, metadata, waiters, stream - * ownership, and restart. + * Independent Maps model rows, selected-tag membership, source commits, resume + * evidence, and persisted state. A controlled ShapeStream boundary supplies + * authored message partitions and settlement order. The production Collection, + * Query-backed persistence wrapper, metadata APIs, and cleanup run unchanged. + * Checks cover public and durable rows, exact batches, readiness, errors, + * metadata, waiters, stream ownership, and restart at named checkpoints. * - * The suite uses fixed protocol histories, exhaustive contiguous partitions, - * and fixed plus random generated campaigns. The installed-SDK delivery oracle - * separately owns real HTTP framing and pause behavior; PostgreSQL expression - * semantics and service-backed execution remain separate evidence. + * The suite retains fixed protocol histories, exhaustive contiguous + * partitions, fixed campaigns, random campaigns, and explicit seed/path replay + * for durability policies. Sensitivity checks reject missing prefix evidence. + * Installed-SDK HTTP delivery, PostgreSQL semantics, and live-service execution + * remain separate evidence. */ - type OracleRow = Row & { id: number name: string @@ -207,6 +211,194 @@ function createPersistedAdapter( } } +type SourceLedgerEvent = { + type: `source-commit` + offset: string + mutations: Array< + { type: `set`; row: OracleRow } | { type: `delete`; key: string | number } + > +} + +// A persisted resume offset is a claim about an append-only source prefix, not +// about whichever adapter calls happened to succeed. This fold shares neither +// Electric's callback state nor the persistence runtime's queue. The fixture +// runs the real Electric collection adapter with the installed SDK boundary +// mocked at ShapeStream; it does not earn a live Electric service receipt. +function foldSourceLedgerThrough( + events: ReadonlyArray, + resumeOffset: string, +): Map { + const rows = new Map() + for (const event of events) { + for (const mutation of event.mutations) { + if (mutation.type === `delete`) rows.delete(mutation.key) + else rows.set(mutation.row.id, structuredClone(mutation.row)) + } + if (event.offset === resumeOffset) return rows + } + throw new Error(`resume offset ${resumeOffset} is absent from source ledger`) +} + +function durablePrefixPropertyOptions(): { + numRuns: number + seed: number + path?: string +} { + const seedValue = process.env.TANSTACK_DB_ELECTRIC_DURABILITY_SEED + const path = process.env.TANSTACK_DB_ELECTRIC_DURABILITY_PATH + if (path !== undefined && seedValue === undefined) { + throw new Error( + `TANSTACK_DB_ELECTRIC_DURABILITY_PATH requires TANSTACK_DB_ELECTRIC_DURABILITY_SEED`, + ) + } + const seed = seedValue === undefined ? 20260921 : Number(seedValue) + if (!Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_ELECTRIC_DURABILITY_SEED must be an integer`) + } + return { + numRuns: path === undefined ? 10 : 1, + seed, + ...(path === undefined ? {} : { path }), + } +} + +function persistencePolicyPropertyOptions(): { + numRuns: number + seed: number + path?: string +} { + const seedValue = process.env.TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED + const path = process.env.TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_PATH + if (path !== undefined && seedValue === undefined) { + throw new Error( + `TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_PATH requires TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED`, + ) + } + const seed = seedValue === undefined ? 20260923 : Number(seedValue) + if (!Number.isSafeInteger(seed)) { + throw new Error( + `TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED must be an integer`, + ) + } + return { + numRuns: path === undefined ? 8 : 1, + seed, + ...(path === undefined ? {} : { path }), + } +} + +async function runRejectedDurablePrefixWitness( + id: string, + names: readonly [string, string], +): Promise { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const firstAttempt = createDeferred() + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + const persistenceError = new Error(`first persistence attempt rejected`) + const durableAttempts: Array<{ + txId: string + keys: Array + sourceTxids: Array + }> = [] + let attempt = 0 + adapter.applyCommittedTx = async (...args) => { + const tx = args[1] + durableAttempts.push({ + txId: tx.txId, + keys: tx.mutations.map((mutation) => mutation.key), + sourceTxids: (tx.rowMetadataMutations ?? []).flatMap((mutation) => { + if (mutation.type === `delete`) return [] + const metadata = mutation.value as { txids?: Array } + return metadata.txids ?? [] + }), + }) + attempt++ + if (attempt === 1) { + firstAttempt.resolve() + throw persistenceError + } + await applyCommittedTx(...args) + } + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + await withElectricCleanup(async () => { + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + + const firstRow: OracleRow = { + id: 1, + name: names[0], + stable: `stable-1`, + } + mockStream.lastOffset = `21_0` + const firstMessage = change(`insert`, firstRow.id, firstRow.name) + firstMessage.headers.txids = [701] + subscriber!([firstMessage, upToDate]) + await atCheckpoint(firstAttempt.promise, `first durability rejection`) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const secondRow: OracleRow = { + id: 2, + name: names[1], + stable: `stable-2`, + } + mockStream.lastOffset = `22_0` + const secondMessage = change(`insert`, secondRow.id, secondRow.name) + secondMessage.headers.txids = [702] + subscriber!([secondMessage, upToDate]) + await new Promise((resolve) => setTimeout(resolve, 0)) + const statusAtTerminalCut = collection.status + await atCheckpoint(collection.cleanup(), `durable-prefix terminal cleanup`) + + const durableResume = observableResume( + persistedMetadata.get(`electric:resume`), + ) + const durableOffset = + durableResume && + typeof durableResume === `object` && + `offset` in durableResume + ? durableResume.offset + : undefined + expect(durableAttempts).toHaveLength(1) + expect(durableAttempts[0]?.keys).toContain(1) + expect(durableAttempts[0]?.sourceTxids).toEqual([701]) + // A rejected durability boundary is terminal: it cannot retry, admit the + // later source commit, or publish a resume claim for an undurable prefix. + expect(statusAtTerminalCut).toBe(`error`) + expect(durableOffset).toBeUndefined() + expect(rowsFromMap(persistedRows)).toEqual([]) + }, [() => collection.cleanup()]) +} + function createOracleCollection( id: string, syncMode: ElectricSyncMode, @@ -875,6 +1067,261 @@ function expectPreloadOutcome( } } +async function runPublicationBeforePersistenceWitness( + id: string, + row: OracleRow, + txid: number, +): Promise { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistenceEntered = createDeferred() + const persistenceCompleted = createDeferred() + const releasePersistence = createDeferred() + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + persistenceEntered.resolve() + await releasePersistence.promise + await applyCommittedTx(...args) + persistenceCompleted.resolve() + } + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (value) => value.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + const publications = observePublications(collection) + + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + const txidEvidence = collection.utils.awaitTxId(txid, 250) + mockStream.lastOffset = `${txid}_0` + const message = change(`insert`, row.id, row.name) + message.value = structuredClone(row) + message.headers.txids = [txid] + subscriber!([message, upToDate]) + + await atCheckpoint(persistenceEntered.promise, `${id} persistence entered`) + publications.record(`persistence-pending`) + + expect(rowsFromCollection(collection)).toEqual( + rowsFromMap(new Map([[row.id, row]])), + ) + expect(publications.entries).toContainEqual({ + cut: `event`, + rows: rowsFromMap(new Map([[row.id, row]])), + }) + expect(publications.entries).toContainEqual({ + cut: `persistence-pending`, + rows: rowsFromMap(new Map([[row.id, row]])), + }) + expect(rowsFromMap(persistedRows)).toEqual([]) + await expect(txidEvidence).resolves.toBe(true) + + releasePersistence.resolve() + await atCheckpoint( + persistenceCompleted.promise, + `${id} persistence completed`, + ) + expect(rowsFromMap(persistedRows)).toEqual( + rowsFromMap(new Map([[row.id, row]])), + ) + }, [ + () => { + releasePersistence.resolve() + publications.stop() + }, + () => collection.cleanup(), + ]) +} + +async function runPersistenceFailureCrashOnlyWitness( + id: string, + firstRow: OracleRow, + secondRow: OracleRow, +): Promise { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistenceEntered = createDeferred() + let rejectPersistence!: (reason: unknown) => void + const persistence = new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + const persistenceRejected = createDeferred() + const rawPersistenceError = Object.assign( + new Error(`adapter durability rejected exactly`), + { + code: `SQLITE_IOERR`, + path: `electric.adapter.applyCommittedTx`, + }, + ) + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + let persistenceAttempts = 0 + adapter.applyCommittedTx = () => { + persistenceAttempts++ + persistenceEntered.resolve() + return persistence.finally(() => persistenceRejected.resolve()) + } + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (value) => value.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + const publications = observePublications(collection) + + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + mockStream.lastOffset = `31_0` + const firstMessage = change(`insert`, firstRow.id, firstRow.name) + firstMessage.value = structuredClone(firstRow) + firstMessage.headers.txids = [731] + subscriber!([firstMessage, upToDate]) + await atCheckpoint( + persistenceEntered.promise, + `${id} adapter persistence entered`, + ) + + // Visibility is the earlier boundary. This exact pending cut kills an + // implementation that rejects before publishing merely because it knows + // persistence will fail later. + expect({ + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + visibleRows: rowsFromCollection(collection), + durableRows: rowsFromMap(persistedRows), + published: publications.entries.some(({ rows }) => + isDeepStrictEqual( + rows, + rowsFromMap(new Map([[firstRow.id, firstRow]])), + ), + ), + }).toEqual({ + status: `ready`, + publicError: undefined, + visibleRows: rowsFromMap(new Map([[firstRow.id, firstRow]])), + durableRows: [], + published: true, + }) + + rejectPersistence(rawPersistenceError) + await atCheckpoint( + persistenceRejected.promise, + `${id} adapter persistence rejection`, + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const statusAtAdmissionCut = collection.status + const publicErrorAtAdmissionCut = collection._lifecycle.getSyncError() + const publicationsBeforeRetiredWork = structuredClone(publications.entries) + mockStream.lastOffset = `32_0` + const secondMessage = change(`insert`, secondRow.id, secondRow.name) + secondMessage.value = structuredClone(secondRow) + secondMessage.headers.txids = [732] + subscriber!([secondMessage, upToDate]) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const publicPersistenceError = collection._lifecycle.getSyncError() + const durabilityError = + publicPersistenceError instanceof Error + ? (publicPersistenceError as Error & { + code?: unknown + path?: unknown + }) + : undefined + expect({ + statusAtAdmissionCut, + publicErrorNameAtAdmissionCut: + publicErrorAtAdmissionCut instanceof Error + ? publicErrorAtAdmissionCut.name + : undefined, + status: collection.status, + publicErrorName: + publicPersistenceError instanceof Error + ? publicPersistenceError.name + : undefined, + hasNamedPersistenceSemantics: + publicPersistenceError instanceof PersistenceDurabilityError, + preservesCause: durabilityError?.cause === rawPersistenceError, + persistenceErrorCode: durabilityError?.code, + persistenceErrorPath: durabilityError?.path, + persistenceAttempts, + visibleRows: rowsFromCollection(collection), + durableRows: rowsFromMap(persistedRows), + publicationsUnchanged: isDeepStrictEqual( + publications.entries, + publicationsBeforeRetiredWork, + ), + }).toEqual({ + statusAtAdmissionCut: `error`, + publicErrorNameAtAdmissionCut: `PersistenceDurabilityError`, + status: `error`, + publicErrorName: `PersistenceDurabilityError`, + hasNamedPersistenceSemantics: true, + preservesCause: true, + persistenceErrorCode: `SQLITE_IOERR`, + persistenceErrorPath: `electric.adapter.applyCommittedTx`, + persistenceAttempts: 1, + visibleRows: rowsFromMap(new Map([[firstRow.id, firstRow]])), + durableRows: [], + publicationsUnchanged: true, + }) + }, [ + () => { + rejectPersistence(rawPersistenceError) + publications.stop() + }, + () => collection.cleanup(), + ]) +} + const processCommandArb: fc.Arbitrary = fc.oneof( fc.record({ kind: fc.constant(`create` as const), @@ -1574,6 +2021,7 @@ async function runSchedulerPermutation( describe(`Electric adapter laws`, () => { let processGrammarRun = 0 + let persistencePolicyRun = 0 beforeEach(() => { vi.clearAllMocks() @@ -1809,6 +2257,58 @@ describe(`Electric adapter laws`, () => { }, ) + it(`publishes an Electric source commit before its adapter persistence settles`, async () => { + await runPublicationBeforePersistenceWitness( + `publication-before-persistence-fixed`, + { id: 41, name: `visible first`, stable: `stable-41` }, + 741, + ) + }) + + fcTest.prop( + [fc.integer({ min: 1, max: 20 }), fc.string({ maxLength: 12 })], + // Replay one generated publication schedule directly with: + // TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED=20260923 + // TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_PATH= + persistencePolicyPropertyOptions(), + )( + `generated Electric commits publish before adapter persistence settles`, + async (id, name) => { + persistencePolicyRun++ + await runPublicationBeforePersistenceWitness( + `publication-before-persistence-generated-${persistencePolicyRun}`, + { id, name, stable: `stable-${id}` }, + 800 + id, + ) + }, + ) + + it(`turns adapter rejection into a named terminal Electric persistence error`, async () => { + await runPersistenceFailureCrashOnlyWitness( + `terminal-persistence-error-fixed`, + { id: 51, name: `published before failure`, stable: `stable-51` }, + { id: 52, name: `must not be admitted`, stable: `stable-52` }, + ) + }) + + fcTest.prop( + [ + fc.integer({ min: 1, max: 20 }), + fc.tuple(fc.string({ maxLength: 12 }), fc.string({ maxLength: 12 })), + ], + persistencePolicyPropertyOptions(), + )( + `generated adapter failures reject publicly and admit no later Electric work`, + async (id, names) => { + persistencePolicyRun++ + await runPersistenceFailureCrashOnlyWitness( + `terminal-persistence-error-generated-${persistencePolicyRun}`, + { id, name: names[0], stable: `stable-${id}` }, + { id: id + 100, name: names[1], stable: `stable-${id + 100}` }, + ) + }, + ) + fcTest.prop( [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], { numRuns: 24 }, @@ -2192,11 +2692,121 @@ describe(`Electric adapter laws`, () => { }, ) + it(`keeps pending row presence across callbacks until one persisted subset commit`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistedRows = new Map() + const persistedMetadata = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + const durabilityEntered = createDeferred() + const releaseDurability = createDeferred() + const durabilitySettled = createDeferred() + let persistedTx: PersistedTx | undefined + adapter.applyCommittedTx = async (...args) => { + persistedTx = structuredClone(args[1]) + durabilityEntered.resolve() + await releaseDurability.promise + await applyCommittedTx(...args) + durabilitySettled.resolve() + } + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `persisted-cross-callback-presence`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `eager`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`)) + + subscriber([change(`insert`, 1, ``)]) + subscriber([change(`update`, 1, `updated across callbacks`)]) + subscriber([subsetEnd]) + await atCheckpoint( + durabilityEntered.promise, + `cross-callback durability entered`, + ) + + expect({ + publicRows: rowsFromCollection(collection), + durableRows: rowsFromMap(persistedRows), + persistedMutations: persistedTx?.mutations.map((mutation) => ({ + type: mutation.type, + key: mutation.key, + value: mutation.type === `delete` ? undefined : mutation.value, + })), + }).toEqual({ + publicRows: [[1, `updated across callbacks`, `stable-1`]], + durableRows: [], + persistedMutations: [ + { + type: `update`, + key: 1, + value: { + id: 1, + name: ``, + stable: `stable-1`, + }, + }, + { + type: `update`, + key: 1, + value: { + id: 1, + name: `updated across callbacks`, + }, + }, + ], + }) + + releaseDurability.resolve() + await atCheckpoint( + durabilitySettled.promise, + `cross-callback durability settled`, + ) + expect(rowsFromMap(persistedRows)).toEqual([ + [1, `updated across callbacks`, `stable-1`], + ]) + }, [ + async () => { + releaseDurability.resolve() + await collection.cleanup() + }, + ]) + }) + fcTest.prop( [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], { numRuns: 20, examples: [ + [ + [ + { operation: `insert`, id: 1, name: `` }, + { operation: `update`, id: 1, name: ` ` }, + { operation: `insert`, id: 2, name: `` }, + ], + 14, + ], [ [ { operation: `insert`, id: 2, name: `` }, @@ -3600,6 +4210,62 @@ describe(`Electric adapter laws`, () => { await collection.cleanup() }) + it(`does not advance durable Electric resume metadata past a rejected row commit`, async () => { + await runRejectedDurablePrefixWitness(`durable-prefix-fixed`, [ + `first`, + `second`, + ]) + }) + + fcTest.prop( + [fc.tuple(fc.string({ maxLength: 12 }), fc.string({ maxLength: 12 }))], + // Replay the retained shrink directly with: + // TANSTACK_DB_ELECTRIC_DURABILITY_SEED=20260921 + // TANSTACK_DB_ELECTRIC_DURABILITY_PATH=0:0:0 + durablePrefixPropertyOptions(), + )( + `durable Electric resume metadata covers every generated source prefix after a rejected commit`, + async (names) => { + await runRejectedDurablePrefixWitness( + `durable-prefix-generated-${JSON.stringify(names)}`, + names, + ) + }, + ) + + fcTest.prop( + [ + fc.uniqueArray(fc.integer({ min: 1, max: 20 }), { + minLength: 2, + maxLength: 6, + }), + ], + { numRuns: 20, seed: 20260922 }, + )(`append-only resume judgment rejects a missing committed row`, (ids) => { + const events: Array = ids.map((id) => ({ + type: `source-commit`, + offset: `${id}_0`, + mutations: [ + { + type: `set`, + row: { id, name: `row-${id}`, stable: `stable-${id}` }, + }, + ], + })) + const expected = rowsFromMap( + foldSourceLedgerThrough(events, events.at(-1)!.offset), + ) + // An intermediate durable checkpoint contains exactly its source prefix. + const prefix = rowsFromMap( + foldSourceLedgerThrough(events, events[0]!.offset), + ) + expect(prefix).toEqual(expected.filter(([key]) => key === ids[0])) + // An offset outside the append-only ledger cannot justify resume state. + expect(() => foldSourceLedgerThrough(events, `absent_0`)).toThrow( + /absent from source ledger/, + ) + }) + it(`rehydrates persisted rows and resume metadata after cleanup and restart`, async () => { const subscribers: Array<(messages: Array>) => void> = [] mockSubscribe.mockImplementation((callback) => { diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 8a56b825ef..7147722915 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -20,15 +20,20 @@ import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' * Hydrated rows and resume metadata provide the last complete public snapshot. * A must-refetch starts a private replacement. Until that replacement is fully * applied, readers may see an earlier permitted snapshot but never a torn mix. - * Failure keeps the old public rows and records repair debt; later success may - * replace them atomically. + * Failure keeps the prior public rows and records repair debt; later success + * may replace them atomically. * - * A plain persisted row Map and metadata Map form the reference snapshots. The - * driver controls hydration, SDK callbacks, applied receipts, cleanup, restart, - * and eager or progressive mode through the real persistence coordinator and - * Electric adapter. It records every exposed cut, not only the final rows. + * Plain persisted row and metadata Maps form the reference snapshots. The + * history grammar varies hydration position, empty state, sync mode, external + * invalidation, failure, cleanup, and restart. The driver controls hydration, + * SDK callbacks, applied receipts, cleanup, restart, and eager or progressive + * mode through the real persistence coordinator and Electric adapter. It + * records every exposed publication cut rather than only final rows. + * + * Trace controls reject torn, reverted, or omitted intermediate publications. + * The controlled SDK boundary does not establish HTTP framing, PostgreSQL + * expression semantics, or live-service recovery; those have separate owners. */ - type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } @@ -227,9 +232,9 @@ describe(`persisted Electric recovery laws`, () => { f.record(`before`) subscribers[0]!([change(`update`, { id: 1, name: `wrong` }), upToDate]) subscribers[0]!([change(`update`, { id: 1, name: `correct` }), upToDate]) - f.record(`repaired`) const correct = [{ ...oldRow, name: `correct` }] - expect(f.publicRows()).toEqual(correct) + await vi.waitFor(() => expect(f.publicRows()).toEqual(correct)) + f.record(`repaired`) const entries = f.exposures.slice(cut) expect(entries[0]!.rows).toEqual([oldRow]) expect( diff --git a/packages/offline-transactions/tests/leadership-replay.property.test.ts b/packages/offline-transactions/tests/leadership-replay.property.test.ts index a762c20ad3..71f3ce30f3 100644 --- a/packages/offline-transactions/tests/leadership-replay.property.test.ts +++ b/packages/offline-transactions/tests/leadership-replay.property.test.ts @@ -15,17 +15,23 @@ import type { OfflineTransaction } from '../src/types' * * Only the current leader may read and schedule the outbox. Losing leadership * or disposing fences startup, provider work, acknowledgement, retry hooks, and - * stale reads. Regaining leadership may replay durable pending rows, but an ID - * already pending, running, completed, or permanently rejected in the current - * boundary must not execute twice. + * stale reads. Regaining leadership may replay durable pending rows, but an + * ID already pending, running, completed, permanently rejected, or durably + * removed at the current boundary must not execute twice. * * A fake durable outbox, explicit leadership callbacks, and gated provider and - * delete acknowledgements form the history grammar. The real OfflineExecutor, - * TransactionExecutor, scheduler, and transaction path run unchanged. Checks - * cover mutation calls and idempotency keys, outbox contents, scheduler counts, - * leadership ownership, retry records, restoration, disposal, and cleanup. + * delete acknowledgements form the history grammar. `foldReplayLedger` models + * stored and durably removed IDs without production revision counters. + * Generated histories hold a real storage-read delivery while removing a + * prefix. The real OfflineExecutor, TransactionExecutor, scheduler, outbox, and + * transaction path run over FakeStorageAdapter. Named checkpoints compare + * delivered IDs, provider calls and idempotency keys, durable outbox state, + * scheduler state, leadership, restoration, and cleanup. + * + * Fixed cases, random campaigns, and an append-only sensitivity control preserve + * path and checker evidence. This suite does not claim IndexedDB, LocalStorage, + * native transaction completion, or exactly-once network execution. */ - function gate() { let resolve!: () => void const promise = new Promise((done) => { @@ -48,6 +54,36 @@ const storedTransaction = (id: string): OfflineTransaction => ({ version: 1, }) +type ReplayLedgerEvent = + | { type: `stored`; id: string } + | { type: `read-captured`; ids: Array } + | { type: `durably-removed`; id: string } + | { type: `read-delivered` } + +// The model records durable facts instead of copying OutboxManager's version +// fences. A stale captured value cannot undo a later durable removal. These +// fixtures use FakeStorageAdapter and therefore establish executor/outbox +// behavior, not IndexedDB, LocalStorage, or native transaction completion. +function foldReplayLedger(events: ReadonlyArray): { + unfinished: Array + staleCapture: Array +} { + const stored = new Set() + const removed = new Set() + let staleCapture: Array = [] + + for (const event of events) { + if (event.type === `stored`) stored.add(event.id) + else if (event.type === `durably-removed`) removed.add(event.id) + else if (event.type === `read-captured`) staleCapture = [...event.ids] + } + + return { + unfinished: Array.from(stored).filter((id) => !removed.has(id)), + staleCapture, + } +} + it.each([`construction`, `leadership`, `outbox read`, `retry hook`] as const)( `does not revive a disposed executor after %s`, async (boundary) => { @@ -614,6 +650,105 @@ it(`does not readmit a permanently rejected row from a stale outbox read`, async } }) +it.each([20260923, undefined])( + `filters every durably removed transaction from a generated stale replay read (seed %s)`, + async (seed) => { + await fc.assert( + fc.asyncProperty( + fc.record({ + count: fc.integer({ min: 2, max: 6 }), + removedPrefix: fc.integer({ min: 1, max: 5 }), + }), + async ({ count, removedPrefix }) => { + const captured = gate() + const delivery = gate() + let hold = true + let capturedSnapshot: Array = [] + class Storage extends FakeStorageAdapter { + override async keys() { + const keys = await super.keys() + capturedSnapshot = [...keys] + return keys + } + override async get(key: string) { + const value = await super.get(key) + if (hold && key === `tx:stored-0`) { + captured.resolve() + await delivery.promise + } + return value + } + } + const storage = new Storage() + const outbox = new OutboxManager(storage, {}) + const transactions = Array.from({ length: count }, (_, index) => + storedTransaction(`stored-${index}`), + ) + const ledger: Array = [] + for (const transaction of transactions) { + await outbox.add(transaction) + ledger.push({ type: `stored`, id: transaction.id }) + } + const reading = outbox.getAll() + let hasPrimaryFailure = false + try { + await atOracleCheckpoint(captured.promise, `stale replay captured`) + ledger.push({ + type: `read-captured`, + ids: capturedSnapshot.map((key) => key.slice(`tx:`.length)), + }) + const removed = transactions.slice( + 0, + Math.min(count - 1, removedPrefix), + ) + await outbox.removeMany(removed.map(({ id }) => id)) + for (const transaction of removed) { + ledger.push({ type: `durably-removed`, id: transaction.id }) + } + hold = false + delivery.resolve() + ledger.push({ type: `read-delivered` }) + + const expected = foldReplayLedger(ledger) + const delivered = await atOracleCheckpoint( + reading, + `stale replay delivered`, + ) + expect(delivered.map(({ id }) => id)).toEqual(expected.unfinished) + expect(expected.staleCapture).not.toEqual(expected.unfinished) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hold = false + delivery.resolve() + await cleanupOfflineOracle([() => reading], hasPrimaryFailure) + } + }, + ), + { + seed, + numRuns: 20, + examples: [[{ count: 3, removedPrefix: 1 }]], + }, + ) + }, +) + +it(`append-only replay judgment rejects stale admission after durable removal`, () => { + const ledger: Array = [ + { type: `stored`, id: `removed` }, + { type: `stored`, id: `peer` }, + { type: `read-captured`, ids: [`removed`, `peer`] }, + { type: `durably-removed`, id: `removed` }, + { type: `read-delivered` }, + ] + const expected = foldReplayLedger(ledger) + + expect(expected.unfinished).toEqual([`peer`]) + expect(expected.staleCapture).not.toEqual(expected.unfinished) +}) + it.each([`remove`, `removeMany`, `clear`] as const)( `filters %s from an outbox read already in flight`, async (operation) => {