From 28affe624cba8af90e3a25fbcf4c7bb6f2e5a15a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 20 Sep 2026 23:41:14 +0100 Subject: [PATCH 01/10] fix(sqlite-persistence): race local and network readiness --- .github/workflows/e2e-tests.yml | 5 + .../electric-coordinator-readiness.opfs.html | 16 + ...lectric-coordinator-readiness.opfs.spec.ts | 94 ++ .../electric-coordinator-readiness.opfs.ts | 336 ++++ .../package.json | 5 +- .../playwright.readiness-opfs.config.ts | 29 + .../tsconfig.json | 3 +- .../vite.readiness-opfs.config.ts | 36 + .../src/persisted.ts | 442 +++++- .../tests/persisted.test.ts | 1404 +++++++++++++++-- .../electric-db-collection/src/electric.ts | 11 +- pnpm-lock.yaml | 6 + 12 files changed, 2161 insertions(+), 226 deletions(-) create mode 100644 packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.html create mode 100644 packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts create mode 100644 packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts create mode 100644 packages/browser-db-sqlite-persistence/playwright.readiness-opfs.config.ts create mode 100644 packages/browser-db-sqlite-persistence/vite.readiness-opfs.config.ts diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a340502b46..6e11a32ce0 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -74,6 +74,11 @@ jobs: cd examples/react/start-ssr-e2e pnpm exec playwright install --with-deps chromium + - name: Run browser SQLite readiness E2E tests + run: | + cd packages/browser-db-sqlite-persistence + pnpm test:e2e:readiness + - name: Run React Start SSR E2E tests run: | cd examples/react/start-ssr-e2e diff --git a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.html b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.html new file mode 100644 index 0000000000..d1962323f8 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.html @@ -0,0 +1,16 @@ + + + + + + + Electric coordinator readiness oracle + + + running + + + diff --git a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts new file mode 100644 index 0000000000..bcb5bb0aa6 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts @@ -0,0 +1,94 @@ +import { expect, test } from '@playwright/test' +import type { Page } from '@playwright/test' +import type { ReadinessOracleResult } from './electric-coordinator-readiness.opfs' + +type Mode = `non-empty` | `empty` | `network-wins` | `on-demand` + +async function readResult( + page: Page, + mode: Mode, +): Promise { + await page.goto(`/e2e/electric-coordinator-readiness.opfs.html?mode=${mode}`) + await page.waitForFunction( + () => window.__tanstackElectricCoordinatorReadiness !== undefined, + ) + return page.evaluate(() => window.__tanstackElectricCoordinatorReadiness!) +} + +function expectComplete( + result: ReadinessOracleResult, +): asserts result is Extract { + if (result.status !== `complete`) { + throw new Error(result.primaryFailure) + } +} + +const eagerCases = [ + { + mode: `non-empty` as const, + rows: [{ id: `persisted`, title: `Persisted while upstream is pending` }], + }, + { mode: `empty` as const, rows: [] }, +] as const + +for (const expected of eagerCases) { + test(`eager ${expected.mode} OPFS snapshot becomes ready while Electric remains pending`, async ({ + page, + }) => { + const result = await readResult(page, expected.mode) + + expectComplete(result) + expect(result.status).toBe(`complete`) + expect(result.provider).toBe( + `Chromium OPFSCoopSyncVFS + BrowserCollectionCoordinator + Electric ShapeStream`, + ) + expect(result.observation).toEqual({ + mode: expected.mode, + status: `ready`, + rows: expected.rows, + readyEvents: 1, + hydrationCalls: 1, + upstreamRequests: 1, + readyBeforeHydrationRelease: false, + }) + expect(result.cleanupFailures).toEqual([]) + }) +} + +test(`on-demand remains upstream-gated with the same browser stack`, async ({ + page, +}) => { + const result = await readResult(page, `on-demand`) + + expectComplete(result) + expect(result.status).toBe(`complete`) + expect(result.observation).toEqual({ + mode: `on-demand`, + status: `loading`, + rows: [], + readyEvents: 0, + hydrationCalls: 0, + upstreamRequests: 1, + readyBeforeHydrationRelease: false, + }) + expect(result.cleanupFailures).toEqual([]) +}) + +test(`an authoritative Electric snapshot wins before OPFS hydration finishes`, async ({ + page, +}) => { + const result = await readResult(page, `network-wins`) + + expectComplete(result) + expect(result.status).toBe(`complete`) + expect(result.observation.mode).toBe(`network-wins`) + expect(result.observation.status).toBe(`ready`) + expect(result.observation.rows).toEqual([ + { id: `network`, title: `Network winner` }, + ]) + expect(result.observation.readyEvents).toBe(1) + expect(result.observation.hydrationCalls).toBe(1) + expect(result.observation.upstreamRequests).toBeGreaterThanOrEqual(1) + expect(result.observation.readyBeforeHydrationRelease).toBe(true) + expect(result.cleanupFailures).toEqual([]) +}) diff --git a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts new file mode 100644 index 0000000000..009e0146e7 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts @@ -0,0 +1,336 @@ +import { createCollection } from '@tanstack/db' +import { electricCollectionOptions } from '@tanstack/electric-db-collection' +import { + BrowserCollectionCoordinator, + createBrowserWASQLitePersistence, + openBrowserWASQLiteOPFSDatabase, + persistedCollectionOptions, +} from '../src/index' +import type { ElectricCollectionUtils } from '@tanstack/electric-db-collection' + +type Row = { id: string; title: string } +type Mode = `non-empty` | `empty` | `network-wins` | `on-demand` + +type ReadinessObservation = { + mode: Mode + status: string + rows: Array + readyEvents: number + hydrationCalls: number + upstreamRequests: number + readyBeforeHydrationRelease: boolean +} + +export type ReadinessOracleResult = + | { + status: `complete` + provider: `Chromium OPFSCoopSyncVFS + BrowserCollectionCoordinator + Electric ShapeStream` + observation: ReadinessObservation + cleanupFailures: ReadonlyArray + } + | { + status: `failed-before-checkpoint` + provider: `Chromium OPFSCoopSyncVFS + BrowserCollectionCoordinator + Electric ShapeStream` + primaryFailure: string + cleanupFailures: ReadonlyArray + } + +declare global { + interface Window { + __tanstackElectricCoordinatorReadiness?: ReadinessOracleResult + } +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +function currentMode(): Mode { + const mode = new URL(location.href).searchParams.get(`mode`) + if (mode === `empty` || mode === `network-wins` || mode === `on-demand`) { + return mode + } + return `non-empty` +} + +function publicRows(rows: ReadonlyArray): Array { + return rows + .map((row) => ({ id: row.id, title: row.title })) + .sort((left, right) => left.id.localeCompare(right.id)) +} + +async function removeOPFSArtifacts( + databaseName: string, +): Promise> { + const failures: Array = [] + const root = await navigator.storage.getDirectory() + for (const suffix of [``, `-journal`, `-wal`]) { + try { + await root.removeEntry(`${databaseName}${suffix}`) + } catch (error) { + if (!(error instanceof DOMException && error.name === `NotFoundError`)) { + failures.push( + `${databaseName}${suffix}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + } + + const iterableRoot = root as FileSystemDirectoryHandle & { + entries: () => AsyncIterableIterator<[string, FileSystemHandle]> + } + for await (const [name, handle] of iterableRoot.entries()) { + if (handle.kind !== `directory` || !name.startsWith(`.ahp-`)) continue + try { + await root.removeEntry(name, { recursive: true }) + } catch (error) { + failures.push( + `${name}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + return failures +} + +async function nextFrame(): Promise { + await new Promise((resolve) => requestAnimationFrame(() => resolve())) +} + +async function observe(mode: Mode): Promise { + const databaseName = `r6-${crypto.randomUUID()}.sqlite` + const collectionId = `electric-${mode}` + const cleanupTasks: Array<() => void | Promise> = [] + const hydrationRelease = deferred() + let outcome: + | { ok: true; observation: ReadinessObservation } + | { ok: false; error: unknown } + try { + const database = await openBrowserWASQLiteOPFSDatabase({ databaseName }) + cleanupTasks.unshift(async () => { + await Promise.resolve(database.close?.()) + }) + const coordinator = new BrowserCollectionCoordinator({ + dbName: databaseName, + }) + cleanupTasks.unshift(() => coordinator.dispose()) + const rootPersistence = createBrowserWASQLitePersistence({ + database, + coordinator, + }) + const persistence = rootPersistence.resolvePersistenceForCollection?.({ + collectionId, + mode: `sync-present`, + }) + if (!persistence) { + throw new Error(`Browser persistence did not resolve sync-present mode`) + } + + const seededRows: Array = + mode === `non-empty` + ? [{ id: `persisted`, title: `Persisted while upstream is pending` }] + : mode === `network-wins` + ? [{ id: `stale`, title: `Late local row` }] + : [] + if (seededRows.length > 0) { + await persistence.adapter.applyCommittedTx(collectionId, { + txId: `seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: seededRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: row, + })), + }) + } + + let hydrationCalls = 0 + const hydrationReturned = deferred() + const loadSubset = persistence.adapter.loadSubset.bind(persistence.adapter) + persistence.adapter.loadSubset = async (...args) => { + hydrationCalls++ + const rows = await loadSubset(...args) + if (mode === `network-wins`) await hydrationRelease.promise + hydrationReturned.resolve() + return rows + } + + let upstreamRequests = 0 + let deliveredNetworkSnapshot = false + const upstreamRequested = deferred() + const fetchClient: typeof fetch = (_input, init) => { + upstreamRequests++ + upstreamRequested.resolve() + if (mode === `network-wins` && !deliveredNetworkSnapshot) { + deliveredNetworkSnapshot = true + return Promise.resolve( + new Response( + JSON.stringify([ + { + key: `network`, + value: { id: `network`, title: `Network winner` }, + headers: { operation: `insert` }, + }, + { + headers: { + control: `up-to-date`, + global_last_seen_lsn: `1`, + }, + }, + ]), + { + headers: { + 'electric-handle': `network-shape`, + 'electric-offset': `1_0`, + 'electric-schema': JSON.stringify({ + id: { type: `text` }, + title: { type: `text` }, + }), + }, + }, + ), + ) + } + return new Promise((_resolve, reject) => { + const signal = init?.signal + const abort = () => + reject( + signal?.reason ?? + new DOMException(`Electric request aborted`, `AbortError`), + ) + if (signal?.aborted) abort() + else signal?.addEventListener(`abort`, abort, { once: true }) + }) + } + + const collection = createCollection( + persistedCollectionOptions< + Row, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: collectionId, + shapeOptions: { + url: `http://electric.invalid/v1/shape`, + params: { table: `todos` }, + fetchClient, + }, + syncMode: mode === `on-demand` ? `on-demand` : `eager`, + startSync: true, + getKey: (row) => row.id, + }), + persistence, + }), + ) + cleanupTasks.unshift(() => collection.cleanup()) + let readyEvents = 0 + const readyObserved = deferred() + collection.on(`status:ready`, () => { + readyEvents++ + readyObserved.resolve() + }) + + await upstreamRequested.promise + let readyBeforeHydrationRelease = false + if (mode === `network-wins`) { + await readyObserved.promise + readyBeforeHydrationRelease = collection.status === `ready` + hydrationRelease.resolve() + await hydrationReturned.promise + } else if (mode !== `on-demand`) { + await hydrationReturned.promise + } + await nextFrame() + await nextFrame() + outcome = { + ok: true, + observation: { + mode, + status: collection.status, + rows: publicRows(collection.toArray), + readyEvents, + hydrationCalls, + upstreamRequests, + readyBeforeHydrationRelease, + }, + } + } catch (error) { + outcome = { ok: false, error } + } + + hydrationRelease.resolve() + const cleanupFailures: Array = [] + for (const cleanup of cleanupTasks) { + try { + await cleanup() + } catch (error) { + cleanupFailures.push(error) + } + } + try { + cleanupFailures.push(...(await removeOPFSArtifacts(databaseName))) + } catch (error) { + cleanupFailures.push(error) + } + + if (!outcome.ok) { + if (cleanupFailures.length > 0) { + throw new AggregateError( + cleanupFailures, + `Readiness observation failed`, + { + cause: outcome.error, + }, + ) + } + throw outcome.error + } + if (cleanupFailures.length > 0) { + throw new AggregateError(cleanupFailures, `OPFS cleanup failed`) + } + return outcome.observation +} + +async function run(): Promise { + const provider = + `Chromium OPFSCoopSyncVFS + BrowserCollectionCoordinator + Electric ShapeStream` as const + const status = document.querySelector(`#oracle-status`) + let cleanupFailures: ReadonlyArray = [] + try { + const observation = await observe(currentMode()) + window.__tanstackElectricCoordinatorReadiness = { + status: `complete`, + provider, + observation, + cleanupFailures, + } + if (status) status.value = `complete` + } catch (error) { + if (error instanceof AggregateError) { + cleanupFailures = error.errors.map(String) + } + const primaryError = + error instanceof AggregateError && error.cause !== undefined + ? error.cause + : error + window.__tanstackElectricCoordinatorReadiness = { + status: `failed-before-checkpoint`, + provider, + primaryFailure: + primaryError instanceof Error + ? primaryError.message + : String(primaryError), + cleanupFailures, + } + if (status) status.value = `failed-before-checkpoint` + } +} + +void run() diff --git a/packages/browser-db-sqlite-persistence/package.json b/packages/browser-db-sqlite-persistence/package.json index 9bae7bb83c..02ccd80170 100644 --- a/packages/browser-db-sqlite-persistence/package.json +++ b/packages/browser-db-sqlite-persistence/package.json @@ -23,7 +23,8 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:e2e": "pnpm --filter @tanstack/db-ivm build && pnpm --filter @tanstack/db build && pnpm --filter @tanstack/db-sqlite-persistence-core build && pnpm --filter @tanstack/browser-db-sqlite-persistence build && vitest --config vitest.e2e.config.ts --run" + "test:e2e": "pnpm --filter @tanstack/db-ivm build && pnpm --filter @tanstack/db build && pnpm --filter @tanstack/db-sqlite-persistence-core build && pnpm --filter @tanstack/browser-db-sqlite-persistence build && vitest --config vitest.e2e.config.ts --run", + "test:e2e:readiness": "playwright test --config playwright.readiness-opfs.config.ts" }, "type": "module", "main": "dist/cjs/index.cjs", @@ -56,6 +57,8 @@ }, "devDependencies": { "@journeyapps/wa-sqlite": "^1.4.1", + "@playwright/test": "^1.60.0", + "@tanstack/electric-db-collection": "workspace:*", "@types/better-sqlite3": "^7.6.13", "@vitest/coverage-istanbul": "^3.2.4", "better-sqlite3": "^12.6.2" diff --git a/packages/browser-db-sqlite-persistence/playwright.readiness-opfs.config.ts b/packages/browser-db-sqlite-persistence/playwright.readiness-opfs.config.ts new file mode 100644 index 0000000000..fd226ed5bc --- /dev/null +++ b/packages/browser-db-sqlite-persistence/playwright.readiness-opfs.config.ts @@ -0,0 +1,29 @@ +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { defineConfig } from '@playwright/test' + +const packageDirectory = dirname(fileURLToPath(import.meta.url)) +const baseURL = `http://127.0.0.1:4186` +const browserChannel = + process.env.PLAYWRIGHT_CHANNEL ?? (process.env.CI ? undefined : `chrome`) + +export default defineConfig({ + testDir: `./e2e`, + testMatch: `electric-coordinator-readiness.opfs.spec.ts`, + timeout: 60_000, + fullyParallel: false, + workers: 1, + use: { + baseURL, + ...(browserChannel ? { channel: browserChannel } : {}), + headless: true, + trace: `retain-on-failure`, + }, + webServer: { + command: `${resolve(packageDirectory, `../../node_modules/.bin/vite`)} --config vite.readiness-opfs.config.ts --host 127.0.0.1 --port 4186`, + cwd: packageDirectory, + reuseExistingServer: false, + timeout: 120_000, + url: `${baseURL}/e2e/electric-coordinator-readiness.opfs.html`, + }, +}) diff --git a/packages/browser-db-sqlite-persistence/tsconfig.json b/packages/browser-db-sqlite-persistence/tsconfig.json index 5b14f299c7..cc3ce70afe 100644 --- a/packages/browser-db-sqlite-persistence/tsconfig.json +++ b/packages/browser-db-sqlite-persistence/tsconfig.json @@ -14,11 +14,12 @@ "paths": { "@tanstack/db": ["../db/src"], "@tanstack/db-ivm": ["../db-ivm/src"], + "@tanstack/electric-db-collection": ["../electric-db-collection/src"], "@tanstack/db-sqlite-persistence-core": [ "../db-sqlite-persistence-core/src" ] } }, - "include": ["src", "tests", "e2e", "vite.config.ts", "vitest.e2e.config.ts"], + "include": ["src", "tests", "e2e", "*.config.ts"], "exclude": ["node_modules", "dist"] } diff --git a/packages/browser-db-sqlite-persistence/vite.readiness-opfs.config.ts b/packages/browser-db-sqlite-persistence/vite.readiness-opfs.config.ts new file mode 100644 index 0000000000..a99089e6dd --- /dev/null +++ b/packages/browser-db-sqlite-persistence/vite.readiness-opfs.config.ts @@ -0,0 +1,36 @@ +import { realpathSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vite' + +const packageDirectory = dirname(fileURLToPath(import.meta.url)) +const workspaceDirectory = resolve(packageDirectory, `../..`) +const dependencyDirectory = realpathSync( + resolve(workspaceDirectory, `node_modules`), +) + +export default defineConfig({ + base: `./`, + optimizeDeps: { + exclude: [`@journeyapps/wa-sqlite`], + }, + resolve: { + alias: { + '@tanstack/db': resolve(packageDirectory, `../db/src`), + '@tanstack/db-ivm': resolve(packageDirectory, `../db-ivm/src`), + '@tanstack/db-sqlite-persistence-core': resolve( + packageDirectory, + `../db-sqlite-persistence-core/src`, + ), + '@tanstack/electric-db-collection': resolve( + packageDirectory, + `../electric-db-collection/src`, + ), + }, + }, + server: { + fs: { + allow: [workspaceDirectory, dependencyDirectory], + }, + }, +}) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..c1f7b4f14e 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -579,6 +579,7 @@ type NormalizedSyncOperation = } type BufferedSyncTransaction = { + lifecycleGeneration: number operations: Array> rowMetadataWrites: Map< TKey, @@ -599,7 +600,13 @@ type OpenSyncTransaction< T extends object, TKey extends string | number, > = BufferedSyncTransaction & { + beginOptions?: { immediate?: boolean } queuedBecauseHydrating: boolean + supersededHydration: boolean +} + +class PersistedHydrationSupersededError extends Error { + override readonly name = `PersistedHydrationSupersededError` } type SyncWriteNormalization = { @@ -807,6 +814,7 @@ class PersistedCollectionRuntime< private startPromise: Promise | null = null private resumeBaselinePromise: Promise | null = null private lifecycleGeneration = 0 + private hydrationSupersessionGeneration: number | null = null private internalApplyDepth = 0 private appliedReceiptSequence = 0 private readonly pendingAppliedReceipts = new Map>() @@ -877,6 +885,41 @@ class PersistedCollectionRuntime< return this.hydratingGeneration === this.lifecycleGeneration } + getLifecycleGeneration(): number { + return this.lifecycleGeneration + } + + supersedeHydration(): boolean { + if (!this.isHydratingNow()) return false + this.hydrationSupersessionGeneration = this.lifecycleGeneration + this.hydratingGeneration = null + const error = new PersistedHydrationSupersededError( + `Persisted hydration was superseded by an upstream snapshot`, + ) + for (const transaction of this.queuedHydrationTransactions) { + transaction.rejectApplied?.(error) + } + this.queuedHydrationTransactions.length = 0 + // The authoritative snapshot replaces the queued commits' row effects, + // but their stream positions still precede the snapshot's persistence. + // Preserve that ordering so the snapshot cannot reuse an applied seq. + for (const txCommitted of this.queuedTxCommitted) { + this.observeStreamPosition( + txCommitted.term, + txCommitted.seq, + txCommitted.latestRowVersion, + ) + } + this.queuedTxCommitted.length = 0 + return true + } + + finishHydrationSupersession(lifecycleGeneration: number): void { + if (this.hydrationSupersessionGeneration === lifecycleGeneration) { + this.hydrationSupersessionGeneration = null + } + } + isApplyingInternally(): boolean { return this.internalApplyDepth > 0 } @@ -1048,14 +1091,32 @@ class PersistedCollectionRuntime< this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, - lifecycleGeneration, - }), - ) - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.waitForAppliedReceiptsAfter(appliedCursor) + let localFailure: unknown + let localFailed = false + try { + await this.applyMutex.run(() => + this.hydrateSubsetUnsafe(options, { + requestRemoteEnsure: this.mode === `sync-present`, + lifecycleGeneration, + }), + ) + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.waitForAppliedReceiptsAfter(appliedCursor) + } catch (error) { + localFailed = true + localFailure = error + } + + if ( + localFailed && + (options.signal?.aborted || + (typeof localFailure === `object` && + localFailure !== null && + `name` in localFailure && + localFailure.name === `AbortError`)) + ) { + throw localFailure + } if (upstreamLoadSubset) { try { @@ -1073,9 +1134,21 @@ class PersistedCollectionRuntime< } console.warn(`Failed to trigger remote subset load:`, error) this.queueRemoteSubsetEnsure(options) + if (localFailed) { + throw new AggregateError( + [localFailure, error], + `Persisted and upstream subset loading both failed`, + { cause: localFailure }, + ) + } // Hydration remains readable, but it does not satisfy remote demand. throw error } + return + } + + if (localFailed) { + throw localFailure } } @@ -1108,9 +1181,10 @@ class PersistedCollectionRuntime< async persistAndBroadcastExternalSyncTransaction( transaction: BufferedSyncTransaction, ): Promise { - await this.applyMutex.run(() => - this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction), - ) + await this.applyMutex.run(async () => { + if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return + await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) + }) } normalizeSyncWriteMessage( @@ -1243,6 +1317,7 @@ class PersistedCollectionRuntime< private advanceLifecycle(): void { this.lifecycleGeneration++ + this.hydrationSupersessionGeneration = null this.started = false this.startupMetadataPromise = null this.startPromise = null @@ -1303,23 +1378,52 @@ class PersistedCollectionRuntime< }, ): Promise { this.hydratingGeneration = config.lifecycleGeneration + let hydrationFailure: unknown + let hydrationFailed = false try { const rows = await this.loadSubsetRowsUnsafe(options) if (config.lifecycleGeneration !== this.lifecycleGeneration) return + if (this.hydratingGeneration !== config.lifecycleGeneration) { + throw new PersistedHydrationSupersededError( + `Persisted hydration was superseded by an upstream snapshot`, + ) + } this.applyRowsToCollection(rows) + } catch (error) { + hydrationFailed = true + hydrationFailure = error } finally { if (this.hydratingGeneration === config.lifecycleGeneration) { this.hydratingGeneration = null } } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() + if (hydrationFailed) { + for (const transaction of this.queuedHydrationTransactions) { + transaction.rejectApplied?.(hydrationFailure) + } + this.queuedHydrationTransactions.length = 0 + for (const txCommitted of this.queuedTxCommitted) { + this.observeLocalStreamPosition( + txCommitted.term, + txCommitted.seq, + txCommitted.latestRowVersion, + ) + } + this.queuedTxCommitted.length = 0 + } else { + await this.flushQueuedHydrationTransactionsUnsafe() + await this.flushQueuedTxCommittedUnsafe() + } if (config.requestRemoteEnsure) { this.queueRemoteSubsetEnsure(options) } + + if (hydrationFailed) { + throw hydrationFailure + } } private applyRowsToCollection( @@ -1480,7 +1584,11 @@ class PersistedCollectionRuntime< } if (!transaction.internal) { - await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) + if (transaction.lifecycleGeneration === this.lifecycleGeneration) { + await this.persistAndBroadcastExternalSyncTransactionUnsafe( + transaction, + ) + } } transaction.resolveApplied?.() } catch (error) { @@ -1829,6 +1937,14 @@ class PersistedCollectionRuntime< this.latestRowVersion = rowVersion } + this.observeLocalStreamPosition(term, seq, rowVersion) + } + + private observeLocalStreamPosition( + term: number, + seq: number, + rowVersion: number, + ): void { if (term > this.localTerm) { this.localTerm = term this.localSeq = seq @@ -1955,6 +2071,16 @@ class PersistedCollectionRuntime< // of both local and remote mutations. The seq dedup in // processCommittedTxUnsafe prevents double-processing of our own writes. if (isTxCommittedPayload(payload)) { + // Reserve the observed stream position synchronously. Row invalidation + // still runs under applyMutex, but a local write already queued ahead of + // it must not reuse a sequence that is durable in another context. + if (this.hydrationSupersessionGeneration === this.lifecycleGeneration) { + this.observeLocalStreamPosition( + payload.term, + payload.seq, + payload.latestRowVersion, + ) + } if (this.isHydratingNow()) { this.queuedTxCommitted.push(payload) return @@ -2321,6 +2447,7 @@ function createWrappedSyncConfig< >( sourceSyncConfig: SyncConfig, runtime: PersistedCollectionRuntime, + syncMode: `eager` | `on-demand`, ): SyncConfig { return { ...sourceSyncConfig, @@ -2329,7 +2456,131 @@ function createWrappedSyncConfig< const getOpenTransaction = () => transactionStack[transactionStack.length - 1] let fullStartPromise: Promise | null = null - const startupState = { cleanedUp: false } + const startupState: { + cleanedUp: boolean + signalled: `loading` | `ready` | `error` + local: `pending` | `unavailable` | `ready` | `failed` + localError?: unknown + upstream: `pending` | `ready` | `failed` + upstreamError?: unknown + } = { + cleanedUp: false, + signalled: `loading`, + local: syncMode === `on-demand` ? `unavailable` : `pending`, + upstream: `pending`, + } + const reconcileAvailability = () => { + if (startupState.cleanedUp) return + + if ( + startupState.local === `ready` || + startupState.upstream === `ready` + ) { + if (startupState.signalled === `ready`) return + startupState.signalled = `ready` + params.markReady() + return + } + + if ( + startupState.upstream !== `failed` || + startupState.local === `pending` || + startupState.signalled === `error` + ) { + return + } + + startupState.signalled = `error` + const error = + startupState.local === `failed` + ? new AggregateError( + [startupState.localError, startupState.upstreamError], + `Persisted collection startup failed locally and upstream`, + { cause: startupState.localError }, + ) + : startupState.upstreamError + params.markError(error) + } + const signalLocalReady = () => { + if (startupState.local !== `pending`) return + startupState.local = `ready` + reconcileAvailability() + } + const signalLocalFailure = (error: unknown) => { + if (startupState.local === `ready` || startupState.local === `failed`) { + return + } + if (error instanceof PersistedHydrationSupersededError) { + startupState.local = `unavailable` + reconcileAvailability() + return + } + startupState.local = `failed` + startupState.localError = error + reconcileAvailability() + } + const signalPersistenceFailure = (error: unknown) => { + if (startupState.cleanedUp) return + startupState.local = `failed` + startupState.localError = error + + if (startupState.signalled === `loading`) { + reconcileAvailability() + return + } + + if (startupState.signalled === `error`) { + if (startupState.upstream === `failed`) { + params.markError( + new AggregateError( + [error, startupState.upstreamError], + `Persisted collection failed locally and upstream`, + { cause: error }, + ), + ) + } + return + } + + startupState.signalled = `error` + params.markError(error) + } + const signalUpstreamReady = () => { + startupState.upstream = `ready` + startupState.upstreamError = undefined + reconcileAvailability() + } + const signalUpstreamFailure = (error: unknown) => { + const failedAfterReady = startupState.signalled === `ready` + startupState.upstream = `failed` + startupState.upstreamError = error + if (failedAfterReady) { + startupState.signalled = `error` + params.markError( + startupState.local === `failed` + ? new AggregateError( + [startupState.localError, error], + `Persisted collection failed locally and upstream`, + { cause: startupState.localError }, + ) + : error, + ) + return + } + if (startupState.signalled === `error`) { + if (startupState.local === `failed`) { + params.markError( + new AggregateError( + [startupState.localError, error], + `Persisted collection failed locally and upstream`, + { cause: startupState.localError }, + ), + ) + } + return + } + reconcileAvailability() + } const acquisitions = new Map() runtime.setSyncControls({ begin: params.begin, @@ -2344,32 +2595,21 @@ function createWrappedSyncConfig< const wrappedParams = { ...params, - markReady: () => { - if (startupState.cleanedUp) return - void (fullStartPromise ?? runtime.ensureStarted()) - .then(() => { - if (startupState.cleanedUp) return - params.markReady() - }) - .catch((error) => { - if (startupState.cleanedUp) return - console.warn( - `Failed persisted sync startup before markReady:`, - error, - ) - params.markReady() - }) - }, + markReady: signalUpstreamReady, + markError: signalUpstreamFailure, begin: (options?: { immediate?: boolean }) => { if (startupState.cleanedUp) return const transaction: OpenSyncTransaction = { + lifecycleGeneration: runtime.getLifecycleGeneration(), operations: [], rowMetadataWrites: new Map(), collectionMetadataWrites: new Map(), truncate: false, internal: runtime.isApplyingInternally(), + beginOptions: options, queuedBecauseHydrating: !runtime.isApplyingInternally() && runtime.isHydratingNow(), + supersededHydration: false, } transactionStack.push(transaction) @@ -2569,6 +2809,24 @@ function createWrappedSyncConfig< // collection-scoped metadata before truncating row data, and those // writes must commit atomically with the truncate transaction. openTransaction.truncate = true + if ( + openTransaction.queuedBecauseHydrating && + runtime.supersedeHydration() + ) { + openTransaction.queuedBecauseHydrating = false + openTransaction.supersededHydration = true + params.begin(openTransaction.beginOptions) + for (const [ + key, + metadataWrite, + ] of openTransaction.collectionMetadataWrites) { + if (metadataWrite.type === `delete`) { + params.metadata?.collection.delete(key) + } else { + params.metadata?.collection.set(key, metadataWrite.value) + } + } + } if (!openTransaction.queuedBecauseHydrating) { params.truncate() } @@ -2594,6 +2852,7 @@ function createWrappedSyncConfig< }) void applied.catch(() => undefined) runtime.queueHydrationBufferedTransaction({ + lifecycleGeneration: openTransaction.lifecycleGeneration, operations: openTransaction.operations, rowMetadataWrites: openTransaction.rowMetadataWrites, collectionMetadataWrites: @@ -2607,18 +2866,58 @@ function createWrappedSyncConfig< return applied } - const applied = params.commit(signal) + let applied: SyncAppliedReceipt + try { + applied = params.commit(signal) + } catch (error) { + if (openTransaction.supersededHydration) { + runtime.finishHydrationSupersession( + openTransaction.lifecycleGeneration, + ) + } + throw error + } if (!openTransaction.internal) { + const persist = async () => { + try { + await runtime.persistAndBroadcastExternalSyncTransaction({ + lifecycleGeneration: openTransaction.lifecycleGeneration, + operations: openTransaction.operations, + rowMetadataWrites: openTransaction.rowMetadataWrites, + collectionMetadataWrites: + openTransaction.collectionMetadataWrites, + truncate: openTransaction.truncate, + internal: false, + }) + } finally { + if (openTransaction.supersededHydration) { + runtime.finishHydrationSupersession( + openTransaction.lifecycleGeneration, + ) + } + } + } + if (openTransaction.supersededHydration) { + if (applied === true) { + signalUpstreamReady() + void persist().catch(signalPersistenceFailure) + } else { + void applied.then( + () => { + signalUpstreamReady() + return persist().catch(signalPersistenceFailure) + }, + () => + runtime.finishHydrationSupersession( + openTransaction.lifecycleGeneration, + ), + ) + } + return applied + } const persistAfterApplication = async () => { if (applied !== true) await applied - await runtime.persistAndBroadcastExternalSyncTransaction({ - operations: openTransaction.operations, - rowMetadataWrites: openTransaction.rowMetadataWrites, - collectionMetadataWrites: - openTransaction.collectionMetadataWrites, - truncate: openTransaction.truncate, - internal: false, - }) + await persist() } const persisted = persistAfterApplication() void persisted.catch(() => undefined) @@ -2631,7 +2930,11 @@ function createWrappedSyncConfig< let sourceResult: SyncConfigRes = {} fullStartPromise = runtime.ensureStarted() const sourceResultPromise = (async () => { - await runtime.ensureStartupMetadataLoaded() + try { + await runtime.ensureStartupMetadataLoaded() + } catch (error) { + signalLocalFailure(error) + } if (startupState.cleanedUp) { return sourceResult @@ -2642,6 +2945,10 @@ function createWrappedSyncConfig< ) return sourceResult })() + void fullStartPromise.then(() => { + if (syncMode !== `on-demand`) signalLocalReady() + }, signalLocalFailure) + void sourceResultPromise.catch(signalUpstreamFailure) return { cleanup: () => { @@ -2654,7 +2961,14 @@ function createWrappedSyncConfig< loadSubset: async (options: LoadSubsetOptions) => { const acquisition = { forwarded: false } acquisitions.set(options, acquisition) - await fullStartPromise + let localStartupFailure: unknown + let localStartupFailed = false + try { + await fullStartPromise + } catch (error) { + localStartupFailed = true + localStartupFailure = error + } const resolvedSourceResult = await sourceResultPromise if ( startupState.cleanedUp || @@ -2662,7 +2976,7 @@ function createWrappedSyncConfig< ) { return } - return runtime.loadSubset(options, (loadOptions) => { + const loadFromUpstream = (loadOptions: LoadSubsetOptions) => { // Hydration is another async boundary. A release before this // point owns no upstream lease and must not start one later. if ( @@ -2681,7 +2995,41 @@ function createWrappedSyncConfig< acquisition.forwarded = false throw error } - }) + } + if (localStartupFailed) { + if ( + options.signal?.aborted || + (typeof localStartupFailure === `object` && + localStartupFailure !== null && + `name` in localStartupFailure && + localStartupFailure.name === `AbortError`) + ) { + throw localStartupFailure + } + if (!resolvedSourceResult.loadSubset) { + throw localStartupFailure + } + try { + await loadFromUpstream(options) + } catch (upstreamError) { + if ( + options.signal?.aborted || + (typeof upstreamError === `object` && + upstreamError !== null && + `name` in upstreamError && + upstreamError.name === `AbortError`) + ) { + throw upstreamError + } + throw new AggregateError( + [localStartupFailure, upstreamError], + `Persisted and upstream subset startup both failed`, + { cause: localStartupFailure }, + ) + } + return + } + return runtime.loadSubset(options, loadFromUpstream) }, unloadSubset: (options: LoadSubsetOptions) => { const acquisition = acquisitions.get(options) @@ -2805,7 +3153,11 @@ export function persistedCollectionOptions< const result = { ...syncOptions, id: collectionId, - sync: createWrappedSyncConfig(syncOptions.sync, runtime), + sync: createWrappedSyncConfig( + syncOptions.sync, + runtime, + syncOptions.syncMode ?? `eager`, + ), persistence, } diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 0895e52375..8d10c9f911 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -27,7 +27,11 @@ import type { PullSinceResponse, TxCommitted, } from '../src' -import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { + LoadSubsetOptions, + SyncAppliedReceipt, + SyncConfig, +} from '@tanstack/db' type Todo = { id: string @@ -258,6 +262,20 @@ async function flushAsyncWork(delayMs: number = 0): Promise { await new Promise((resolve) => setTimeout(resolve, delayMs)) } +function deferred(): { + promise: Promise + resolve: (value: T | PromiseLike) => void + reject: (reason?: unknown) => void +} { + let resolve!: (value: T | PromiseLike) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + describe(`persistedCollectionOptions`, () => { it(`provides a sync-absent loopback configuration with persisted utils`, async () => { const adapter = createRecordingAdapter() @@ -1413,10 +1431,15 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) - it(`marks ready even when persisted startup fails before markReady`, async () => { + it(`falls back to upstream readiness when persisted startup fails`, async () => { const adapter = createRecordingAdapter() - adapter.loadSubset = async () => { - throw new Error(`startup failure`) + const startupError = new Error(`startup failure`) + const localAttempted = deferred() + const upstreamStarted = deferred() + let markUpstreamReady: (() => void) | undefined + adapter.loadSubset = () => { + localAttempted.resolve() + return Promise.reject(startupError) } const collection = createCollection( @@ -1425,7 +1448,8 @@ describe(`persistedCollectionOptions`, () => { getKey: (item) => item.id, sync: { sync: ({ markReady }) => { - markReady() + markUpstreamReady = markReady + upstreamStarted.resolve() return {} }, }, @@ -1435,237 +1459,1220 @@ describe(`persistedCollectionOptions`, () => { }), ) - await collection.stateWhenReady() + const preload = collection.preload() + await Promise.all([localAttempted.promise, upstreamStarted.promise]) await flushAsyncWork() + expect(collection.status).toBe(`loading`) + + expect(markUpstreamReady).toBeTypeOf(`function`) + markUpstreamReady!() + await preload expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() }) - it(`reads staged metadata writes during hydration-queued transactions`, async () => { - const adapter = createRecordingAdapter([ - { - id: `cached-1`, - title: `Cached row`, - }, - ]) - adapter.rowMetadata.set(`cached-1`, { source: `persisted` }) - adapter.collectionMetadata.set(`startup:key`, { ready: true }) + it.each([ + { + name: `non-empty`, + rows: [{ id: `1`, title: `Offline Todo` }], + }, + { name: `empty`, rows: [] }, + ])( + `marks an eager $name local snapshot ready without upstream readiness`, + async ({ rows }) => { + const adapter = createRecordingAdapter(rows) + const collection = createCollection( + persistedCollectionOptions({ + id: `local-first-${rows.length}`, + getKey: (item) => item.id, + sync: { sync: () => ({}) }, + persistence: { adapter }, + }), + ) - let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [ - { - key: `cached-1`, - value: { - id: `cached-1`, - title: `Cached row`, - }, - metadata: adapter.rowMetadata.get(`cached-1`), - }, - ] - } + await collection.preload() - let remoteBegin: (() => void) | undefined - let remoteCommit: (() => void) | undefined - let remoteTruncate: (() => void) | undefined - let remoteMetadata: - | Parameters[`sync`]>[0][`metadata`] - | undefined + expect(collection.status).toBe(`ready`) + expect(collection.toArray.map(stripVirtualProps)).toEqual(rows) + expect(adapter.loadSubsetCalls).toHaveLength(1) + await collection.cleanup() + }, + ) + it(`keeps on-demand readiness gated on the upstream`, async () => { + const upstreamStarted = deferred() + let markUpstreamReady: (() => void) | undefined const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-metadata-read`, + id: `on-demand-readiness`, + syncMode: `on-demand`, getKey: (item) => item.id, sync: { - sync: ({ begin, commit, truncate, markReady, metadata }) => { - remoteBegin = begin - remoteCommit = commit - remoteTruncate = truncate - remoteMetadata = metadata - markReady() - return {} + sync: ({ markReady }) => { + markUpstreamReady = markReady + upstreamStarted.resolve() }, }, - persistence: { - adapter, - }, + persistence: { adapter: createRecordingAdapter() }, }), ) - const readyPromise = collection.stateWhenReady() - for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { - await flushAsyncWork() - } - - expect(resolveLoadSubset).toBeDefined() - expect(remoteBegin).toBeDefined() - expect(remoteMetadata).toBeDefined() + collection.startSyncImmediate() + await upstreamStarted.promise + expect(collection.status).toBe(`loading`) - remoteBegin?.() - remoteMetadata?.row.set(`cached-1`, { source: `staged` }) - remoteMetadata?.collection.set(`runtime:key`, { persisted: true }) + markUpstreamReady?.() + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + await collection.cleanup() + }) - expect(remoteMetadata?.row.get(`cached-1`)).toEqual({ source: `staged` }) - expect(remoteMetadata?.collection.get(`runtime:key`)).toEqual({ - persisted: true, - }) - expect(remoteMetadata?.collection.list()).toContainEqual({ - key: `runtime:key`, - value: { persisted: true }, - }) + it(`preserves upstream ready-error-ready transitions for on-demand sync`, async () => { + const upstreamError = new Error(`rebuild failed`) + const upstreamStarted = deferred() + let failUpstream: ((error: unknown) => void) | undefined + let recoverUpstream: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `on-demand-upstream-recovery`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markError, markReady }) => { + failUpstream = markError + recoverUpstream = markReady + markReady() + upstreamStarted.resolve() + return { loadSubset: () => true } + }, + }, + persistence: { adapter: createRecordingAdapter() }, + }), + ) - remoteTruncate?.() + collection.startSyncImmediate() + await upstreamStarted.promise + expect(collection.status).toBe(`ready`) - expect(remoteMetadata?.row.get(`cached-1`)).toBeUndefined() - expect(remoteMetadata?.collection.get(`startup:key`)).toEqual({ - ready: true, - }) + failUpstream?.(upstreamError) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(upstreamError) - remoteCommit?.() - resolveLoadSubset?.() - await readyPromise + recoverUpstream?.() + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() }) - it(`persists truncate transactions and preserves intended collection metadata`, async () => { + it(`loads an on-demand subset upstream when persisted startup fails`, async () => { + const localError = new Error(`persisted metadata unavailable`) const adapter = createRecordingAdapter() - - let remoteBegin: (() => void) | undefined - let remoteWrite: - | ((message: { type: `insert`; value: Todo }) => void) - | undefined - let remoteCommit: (() => void) | undefined - let remoteTruncate: (() => void) | undefined - let remoteMetadata: - | Parameters[`sync`]>[0][`metadata`] - | undefined - + adapter.getStreamPosition = () => Promise.reject(localError) + const upstreamStarted = deferred() + let upstreamLoads = 0 const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-truncate`, + id: `on-demand-upstream-fallback`, + syncMode: `on-demand`, getKey: (item) => item.id, sync: { - sync: ({ begin, write, commit, truncate, markReady, metadata }) => { - remoteBegin = begin - remoteWrite = write as (message: { - type: `insert` - value: Todo - }) => void - remoteCommit = commit - remoteTruncate = truncate - remoteMetadata = metadata + sync: ({ begin, write, commit, markReady }) => { markReady() - return {} + upstreamStarted.resolve() + return { + loadSubset: async () => { + upstreamLoads++ + begin() + write({ + type: `insert`, + value: { id: `network`, title: `Loaded from network` }, + }) + await commit() + }, + } }, }, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) - await collection.stateWhenReady() - await flushAsyncWork() + collection.startSyncImmediate() + await upstreamStarted.promise + expect(collection.status).toBe(`ready`) + await collection._sync.loadSubset({}) - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { - id: `pre-truncate`, - title: `Pre truncate`, - }, - }) - remoteMetadata?.collection.set(`electric:resume`, { - kind: `reset`, - updatedAt: 1, + expect(upstreamLoads).toBe(1) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Loaded from network`, }) - remoteTruncate?.() - remoteWrite?.({ - type: `insert`, - value: { - id: `post-truncate`, - title: `Post truncate`, - }, - }) - remoteCommit?.() - await flushAsyncWork() - - expect(adapter.applyCommittedTxCalls.at(-1)?.tx.truncate).toBe(true) + expect(collection.status).toBe(`ready`) + await collection.cleanup() + }) - const reloadedCollection = createCollection( + it(`loads an on-demand subset upstream when local hydration fails`, async () => { + const localError = new Error(`persisted rows unavailable`) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => Promise.reject(localError) + const upstreamStarted = deferred() + let upstreamLoads = 0 + const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-truncate`, + id: `on-demand-hydration-fallback`, + syncMode: `on-demand`, getKey: (item) => item.id, sync: { - sync: ({ markReady }) => { + sync: ({ begin, write, commit, markReady }) => { markReady() + upstreamStarted.resolve() + return { + loadSubset: async () => { + upstreamLoads++ + begin() + write({ + type: `insert`, + value: { id: `network`, title: `Loaded from network` }, + }) + await commit() + }, + } }, }, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) - await reloadedCollection.preload() - await flushAsyncWork() + collection.startSyncImmediate() + await upstreamStarted.promise + expect(collection.status).toBe(`ready`) + await collection._sync.loadSubset({}) - expect(reloadedCollection.get(`pre-truncate`)).toBeUndefined() - expect(stripVirtualProps(reloadedCollection.get(`post-truncate`))).toEqual({ - id: `post-truncate`, - title: `Post truncate`, - }) - expect( - reloadedCollection._state.syncedCollectionMetadata.get(`electric:resume`), - ).toEqual({ - kind: `reset`, - updatedAt: 1, + expect(upstreamLoads).toBe(1) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Loaded from network`, }) + await collection.cleanup() }) - it(`uses pullSince recovery when tx sequence gaps are detected`, async () => { - const adapter = createRecordingAdapter([ - { - id: `1`, - title: `Initial`, - }, + it(`reports an error only after local hydration and upstream both fail`, async () => { + const localError = new Error(`local startup failed`) + const upstreamError = new Error(`upstream startup failed`) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => Promise.reject(localError) + const collection = createCollection( + persistedCollectionOptions({ + id: `both-startup-paths-fail`, + getKey: (item) => item.id, + sync: { + sync: ({ markError }) => { + markError(upstreamError) + return {} + }, + }, + persistence: { adapter }, + }), + ) + + const outcome = await collection.preload().catch((error) => error) + expect(outcome).toBeInstanceOf(AggregateError) + expect((outcome as AggregateError).errors).toEqual([ + localError, + upstreamError, ]) - const coordinator = createCoordinatorHarness() - coordinator.setPullSinceResponse({ - type: `rpc:pullSince:res`, - rpcId: `pull-1`, - ok: true, - latestTerm: 1, - latestSeq: 3, - latestRowVersion: 3, - requiresFullReload: false, - changedKeys: [`2`], - deletedKeys: [], - }) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(outcome) + await collection.cleanup() + }) + it(`recovers when upstream becomes ready after both startup paths fail`, async () => { + const localError = new Error(`local startup failed`) + const upstreamError = new Error(`upstream startup failed`) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => Promise.reject(localError) + let recoverUpstream: (() => void) | undefined const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, + id: `startup-recovery-after-both-fail`, getKey: (item) => item.id, sync: { - sync: ({ markReady }) => { - markReady() + sync: ({ markError, markReady }) => { + recoverUpstream = markReady + markError(upstreamError) + return {} }, }, - persistence: { - adapter, - coordinator, - }, + persistence: { adapter }, }), ) - await collection.preload() - - adapter.rows.set(`2`, { - id: `2`, - title: `Recovered`, - }) + await expect(collection.preload()).rejects.toBeInstanceOf(AggregateError) + expect(collection.status).toBe(`error`) + + expect(recoverUpstream).toBeTypeOf(`function`) + recoverUpstream!() + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() + }) + + it(`uses a successful local snapshot when upstream fails`, async () => { + const upstreamError = new Error(`upstream startup failed`) + const rows = [{ id: `1`, title: `Offline Todo` }] + const collection = createCollection( + persistedCollectionOptions({ + id: `local-success-upstream-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ markError }) => { + markError(upstreamError) + return {} + }, + }, + persistence: { adapter: createRecordingAdapter(rows) }, + }), + ) + + await collection.preload() + expect(collection.status).toBe(`ready`) + expect(collection.toArray.map(stripVirtualProps)).toEqual(rows) + await collection.cleanup() + }) + + it(`keeps waiting for upstream after local hydration fails`, async () => { + const localAttempted = deferred() + const localError = new Error(`local startup failed`) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + localAttempted.resolve() + return Promise.reject(localError) + } + const collection = createCollection( + persistedCollectionOptions({ + id: `local-failure-upstream-pending`, + getKey: (item) => item.id, + sync: { sync: () => ({}) }, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await localAttempted.promise + await flushAsyncWork() + + expect(collection.status).toBe(`loading`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() + }) + + it(`uses an authoritative upstream snapshot before local hydration finishes`, async () => { + const hydrationStarted = deferred() + const hydration = deferred>() + const upstreamDone = deferred() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + const collection = createCollection( + persistedCollectionOptions({ + id: `upstream-snapshot-wins`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markReady }) => { + void (async () => { + await hydrationStarted.promise + begin() + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + await commit() + markReady() + })().then(upstreamDone.resolve, upstreamDone.reject) + return {} + }, + }, + persistence: { adapter }, + }), + ) + + const preload = collection.preload() + await upstreamDone.promise + await preload + + expect(collection.status).toBe(`ready`) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Network winner`, + }) + + hydration.resolve([ + { key: `stale`, value: { id: `stale`, title: `Late local row` } }, + { key: `network`, value: { id: `network`, title: `Stale local row` } }, + ]) + await hydration.promise + await flushAsyncWork() + + expect(collection.get(`stale`)).toBeUndefined() + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Network winner`, + }) + await collection.cleanup() + }) + + it(`persists a network winner after coordinator activity races hydration`, async () => { + const hydrationStarted = deferred() + const hydration = deferred>() + const startNetwork = deferred() + const upstreamDone = deferred() + const adapter = createRecordingAdapter() + const loadPersistedRows = adapter.loadSubset.bind(adapter) + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let persistedSeq = 0 + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.getStreamPosition = () => + Promise.resolve({ + latestTerm: 1, + latestSeq: persistedSeq, + latestRowVersion: persistedSeq, + }) + adapter.applyCommittedTx = (collectionId, transaction) => { + if (transaction.term === 1 && transaction.seq <= persistedSeq) { + return Promise.resolve() + } + persistedSeq = transaction.seq + return applyCommittedTx(collectionId, transaction) + } + const coordinator = createCoordinatorHarness() + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markReady }) => { + void (async () => { + await startNetwork.promise + begin() + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + await commit() + markReady() + })().then(upstreamDone.resolve, upstreamDone.reject) + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + const preload = collection.preload() + await hydrationStarted.promise + startNetwork.resolve() + await upstreamDone.promise + await preload + + // The authoritative snapshot is live, but its persistence is still parked + // behind hydration. A peer can commit the next stream position in this + // window before the snapshot's queued persistence acquires the mutex. + adapter.rows.set(`peer`, { id: `peer`, title: `Peer transaction` }) + persistedSeq = 1 + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `peer-during-hydration`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `peer`, value: { id: `peer`, title: `Peer transaction` } }, + ], + deletedKeys: [], + }) + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + + hydration.resolve([ + { key: `stale`, value: { id: `stale`, title: `Late local row` } }, + ]) + await hydration.promise + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + expect(adapter.applyCommittedTxCalls[0]?.tx.seq).toBe(2) + expect(adapter.rows.get(`peer`)).toBeUndefined() + expect(adapter.rows.get(`network`)).toEqual({ + id: `network`, + title: `Network winner`, + }) + await collection.cleanup() + + adapter.loadSubset = loadPersistedRows + const reopened = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { sync: () => ({}) }, + persistence: { adapter }, + }), + ) + await reopened.preload() + + expect(stripVirtualProps(reopened.get(`network`))).toEqual({ + id: `network`, + title: `Network winner`, + }) + expect(reopened.get(`peer`)).toBeUndefined() + await reopened.cleanup() + }) + + it(`reports a network winner persistence failure after becoming ready`, async () => { + const hydrationStarted = deferred() + const hydration = deferred>() + const upstreamDone = deferred() + const persistenceAttempted = deferred() + const persistenceError = new Error(`network winner persistence failed`) + let recoverUpstream: (() => void) | undefined + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + adapter.applyCommittedTx = () => { + persistenceAttempted.resolve() + return Promise.reject(persistenceError) + } + const collection = createCollection( + persistedCollectionOptions({ + id: `network-winner-persistence-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markReady }) => { + recoverUpstream = markReady + void (async () => { + await hydrationStarted.promise + begin() + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + await commit() + markReady() + })().then(upstreamDone.resolve, upstreamDone.reject) + return {} + }, + }, + persistence: { adapter }, + }), + ) + + const preload = collection.preload() + await upstreamDone.promise + await preload + expect(collection.status).toBe(`ready`) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Network winner`, + }) + + hydration.resolve([]) + await persistenceAttempted.promise + await flushAsyncWork() + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(persistenceError) + + expect(recoverUpstream).toBeTypeOf(`function`) + recoverUpstream!() + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() + }) + + it(`recovers when explicit upstream readiness follows network winner persistence failure`, async () => { + const hydrationStarted = deferred() + const hydration = deferred>() + const upstreamCommitted = deferred() + const persistenceAttempted = deferred() + const persistenceError = new Error(`network winner persistence failed`) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + adapter.applyCommittedTx = () => { + persistenceAttempted.resolve() + return Promise.reject(persistenceError) + } + let markUpstreamReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `delayed-upstream-ready-after-persistence-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markReady }) => { + markUpstreamReady = markReady + void (async () => { + await hydrationStarted.promise + begin() + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + await commit() + })().then(upstreamCommitted.resolve, upstreamCommitted.reject) + return {} + }, + }, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await upstreamCommitted.promise + expect(collection.status).toBe(`ready`) + hydration.resolve([]) + await persistenceAttempted.promise + await flushAsyncWork() + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(persistenceError) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Network winner`, + }) + + expect(markUpstreamReady).toBeTypeOf(`function`) + markUpstreamReady!() + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() + }) + + it(`aggregates upstream failure after network winner persistence fails`, async () => { + const hydrationStarted = deferred() + const hydration = deferred>() + const upstreamCommitted = deferred() + const persistenceAttempted = deferred() + const persistenceError = new Error(`network winner persistence failed`) + const upstreamError = new Error(`upstream startup failed`) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + adapter.applyCommittedTx = () => { + persistenceAttempted.resolve() + return Promise.reject(persistenceError) + } + let failUpstream: ((error: unknown) => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `delayed-upstream-failure-after-persistence-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markError }) => { + failUpstream = markError + void (async () => { + await hydrationStarted.promise + begin() + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + await commit() + })().then(upstreamCommitted.resolve, upstreamCommitted.reject) + return {} + }, + }, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await upstreamCommitted.promise + expect(collection.status).toBe(`ready`) + hydration.resolve([]) + await persistenceAttempted.promise + await flushAsyncWork() + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(persistenceError) + + expect(failUpstream).toBeTypeOf(`function`) + failUpstream!(upstreamError) + expect(collection.status).toBe(`error`) + const failure = collection._lifecycle.getSyncError() + expect(failure).toBeInstanceOf(AggregateError) + expect((failure as AggregateError).errors).toEqual([ + persistenceError, + upstreamError, + ]) + await collection.cleanup() + }) + + it(`does not persist a network winner after cleanup starts a new lifecycle`, async () => { + const firstHydrationStarted = deferred() + const firstHydration = deferred>() + const firstUpstreamDone = deferred() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + firstHydrationStarted.resolve() + return firstHydration.promise + } + let syncRun = 0 + const collection = createCollection( + persistedCollectionOptions({ + id: `cleanup-fences-network-winner-persistence`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markReady }) => { + syncRun++ + if (syncRun === 1) { + void (async () => { + await firstHydrationStarted.promise + begin() + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + await commit() + markReady() + })().then(firstUpstreamDone.resolve, firstUpstreamDone.reject) + } else { + markReady() + } + return {} + }, + }, + persistence: { adapter }, + }), + ) + + const firstPreload = collection.preload() + await firstUpstreamDone.promise + await firstPreload + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + + await collection.cleanup() + adapter.loadSubset = () => + Promise.resolve([ + { key: `fresh`, value: { id: `fresh`, title: `Fresh restart` } }, + ]) + collection.startSyncImmediate() + firstHydration.resolve([ + { key: `stale`, value: { id: `stale`, title: `Late local row` } }, + ]) + await firstHydration.promise + await collection.stateWhenReady() + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + expect(stripVirtualProps(collection.get(`fresh`))).toEqual({ + id: `fresh`, + title: `Fresh restart`, + }) + expect(collection.get(`stale`)).toBeUndefined() + await collection.cleanup() + }) + + it(`does not persist a transaction after its commit listener cleans up`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `cleanup-during-sync-commit`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return {} + }, + }, + persistence: { adapter }, + }), + ) + + await collection.preload() + let cleanupPromise: Promise | undefined + collection.subscribeChanges( + () => { + cleanupPromise ??= collection.cleanup() + }, + { includeInitialState: false }, + ) + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `retired`, title: `Retired lifecycle` }, + }) + const applied = remoteCommit?.() + if (applied !== true) await applied + await cleanupPromise + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + collection.startSyncImmediate() + await collection.stateWhenReady() + expect(collection.get(`retired`)).toBeUndefined() + await collection.cleanup() + }) + + it(`does not persist buffered replay after its commit listener cleans up`, async () => { + const hydrationStarted = deferred() + const hydration = deferred>() + const upstreamDone = deferred() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let syncRun = 0 + const collection = createCollection( + persistedCollectionOptions({ + id: `cleanup-during-buffered-replay`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + syncRun++ + if (syncRun === 1) { + void (async () => { + await hydrationStarted.promise + begin() + write({ + type: `insert`, + value: { id: `retired`, title: `Retired lifecycle` }, + }) + await commit() + markReady() + })().then(upstreamDone.resolve, upstreamDone.reject) + } else { + markReady() + } + return {} + }, + }, + persistence: { adapter }, + }), + ) + let cleanupPromise: Promise | undefined + collection.subscribeChanges( + () => { + cleanupPromise ??= collection.cleanup() + }, + { includeInitialState: false }, + ) + + collection.startSyncImmediate() + await hydrationStarted.promise + hydration.resolve([]) + await upstreamDone.promise + await cleanupPromise + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + adapter.loadSubset = () => Promise.resolve([]) + collection.startSyncImmediate() + await collection.stateWhenReady() + expect(collection.get(`retired`)).toBeUndefined() + await collection.cleanup() + }) + + it(`rejects buffered upstream rows when eager local hydration fails`, async () => { + const localError = new Error(`persisted rows unavailable`) + const hydrationStarted = deferred() + const hydration = deferred>() + const upstreamDone = deferred() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + const collection = createCollection( + persistedCollectionOptions({ + id: `buffered-upstream-after-hydration-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + void (async () => { + await hydrationStarted.promise + begin() + write({ + type: `insert`, + value: { id: `network`, title: `Loaded from network` }, + }) + await commit() + markReady() + })().then(upstreamDone.resolve, upstreamDone.reject) + return {} + }, + }, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await hydrationStarted.promise + hydration.reject(localError) + const upstreamOutcome = await upstreamDone.promise.catch((error) => error) + await flushAsyncWork() + + expect(upstreamOutcome).toBe(localError) + expect(collection.status).toBe(`loading`) + expect(collection.get(`network`)).toBeUndefined() + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + await collection.cleanup() + }) + + it(`signals readiness once when eager hydration and upstream readiness race`, async () => { + const hydrationStarted = deferred() + const upstreamStarted = deferred() + const hydration = deferred>() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let markUpstreamReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `readiness-race`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markUpstreamReady = markReady + upstreamStarted.resolve() + }, + }, + persistence: { adapter }, + }), + ) + let readyEvents = 0 + collection.on(`status:ready`, () => readyEvents++) + + collection.startSyncImmediate() + await Promise.all([hydrationStarted.promise, upstreamStarted.promise]) + expect(markUpstreamReady).toBeTypeOf(`function`) + markUpstreamReady!() + expect(collection.status).toBe(`ready`) + expect(readyEvents).toBe(1) + + hydration.resolve([]) + await collection.stateWhenReady() + await flushAsyncWork() + + expect(readyEvents).toBe(1) + await collection.cleanup() + }) + + it(`cleanup fences old readiness before a fresh restart snapshot`, async () => { + const hydrationStarted = deferred() + const upstreamStarted = deferred() + const hydration = deferred>() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let markUpstreamReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `cleanup-during-hydration`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markUpstreamReady = markReady + upstreamStarted.resolve() + }, + }, + persistence: { adapter }, + }), + ) + let readyEvents = 0 + collection.on(`status:ready`, () => readyEvents++) + + collection.startSyncImmediate() + await Promise.all([hydrationStarted.promise, upstreamStarted.promise]) + await collection.cleanup() + expect(markUpstreamReady).toBeTypeOf(`function`) + markUpstreamReady!() + hydration.resolve([ + { key: `late`, value: { id: `late`, title: `Must not apply` } }, + ]) + await hydration.promise + await flushAsyncWork() + + expect(readyEvents).toBe(0) + expect(collection.get(`late`)).toBeUndefined() + + adapter.loadSubset = () => + Promise.resolve([ + { key: `fresh`, value: { id: `fresh`, title: `Fresh restart` } }, + ]) + collection.on(`status:ready`, () => readyEvents++) + collection.startSyncImmediate() + await collection.stateWhenReady() + + expect(readyEvents).toBe(1) + expect(stripVirtualProps(collection.get(`fresh`))).toEqual({ + id: `fresh`, + title: `Fresh restart`, + }) + expect(collection.get(`late`)).toBeUndefined() + await collection.cleanup() + }) + + it(`reads staged metadata writes during hydration-queued transactions`, async () => { + const adapter = createRecordingAdapter([ + { + id: `cached-1`, + title: `Cached row`, + }, + ]) + adapter.rowMetadata.set(`cached-1`, { source: `persisted` }) + adapter.collectionMetadata.set(`startup:key`, { ready: true }) + + let resolveLoadSubset: (() => void) | undefined + adapter.loadSubset = async () => { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + return [ + { + key: `cached-1`, + value: { + id: `cached-1`, + title: `Cached row`, + }, + metadata: adapter.rowMetadata.get(`cached-1`), + }, + ] + } + + let remoteBegin: (() => void) | undefined + let remoteCommit: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteMetadata: + | Parameters[`sync`]>[0][`metadata`] + | undefined + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-metadata-read`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, commit, truncate, markReady, metadata }) => { + remoteBegin = begin + remoteCommit = commit + remoteTruncate = truncate + remoteMetadata = metadata + markReady() + return {} + }, + }, + persistence: { + adapter, + }, + }), + ) + + const readyPromise = collection.stateWhenReady() + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } + + expect(resolveLoadSubset).toBeDefined() + expect(remoteBegin).toBeDefined() + expect(remoteMetadata).toBeDefined() + + remoteBegin?.() + remoteMetadata?.row.set(`cached-1`, { source: `staged` }) + remoteMetadata?.collection.set(`runtime:key`, { persisted: true }) + + expect(remoteMetadata?.row.get(`cached-1`)).toEqual({ source: `staged` }) + expect(remoteMetadata?.collection.get(`runtime:key`)).toEqual({ + persisted: true, + }) + expect(remoteMetadata?.collection.list()).toContainEqual({ + key: `runtime:key`, + value: { persisted: true }, + }) + + remoteTruncate?.() + + expect(remoteMetadata?.row.get(`cached-1`)).toBeUndefined() + expect(remoteMetadata?.collection.get(`startup:key`)).toEqual({ + ready: true, + }) + + remoteCommit?.() + resolveLoadSubset?.() + await readyPromise + }) + + it(`persists truncate transactions and preserves intended collection metadata`, async () => { + const adapter = createRecordingAdapter() + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteMetadata: + | Parameters[`sync`]>[0][`metadata`] + | undefined + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-truncate`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, truncate, markReady, metadata }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + remoteTruncate = truncate + remoteMetadata = metadata + markReady() + return {} + }, + }, + persistence: { + adapter, + }, + }), + ) + + await collection.stateWhenReady() + await flushAsyncWork() + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { + id: `pre-truncate`, + title: `Pre truncate`, + }, + }) + remoteMetadata?.collection.set(`electric:resume`, { + kind: `reset`, + updatedAt: 1, + }) + remoteTruncate?.() + remoteWrite?.({ + type: `insert`, + value: { + id: `post-truncate`, + title: `Post truncate`, + }, + }) + remoteCommit?.() + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls.at(-1)?.tx.truncate).toBe(true) + + const reloadedCollection = createCollection( + persistedCollectionOptions({ + id: `sync-present-truncate`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + }, + }), + ) + + await reloadedCollection.preload() + await flushAsyncWork() + + expect(reloadedCollection.get(`pre-truncate`)).toBeUndefined() + expect(stripVirtualProps(reloadedCollection.get(`post-truncate`))).toEqual({ + id: `post-truncate`, + title: `Post truncate`, + }) + expect( + reloadedCollection._state.syncedCollectionMetadata.get(`electric:resume`), + ).toEqual({ + kind: `reset`, + updatedAt: 1, + }) + }) + + it(`uses pullSince recovery when tx sequence gaps are detected`, async () => { + const adapter = createRecordingAdapter([ + { + id: `1`, + title: `Initial`, + }, + ]) + const coordinator = createCoordinatorHarness() + coordinator.setPullSinceResponse({ + type: `rpc:pullSince:res`, + rpcId: `pull-1`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: false, + changedKeys: [`2`], + deletedKeys: [], + }) + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + coordinator, + }, + }), + ) + + await collection.preload() + + adapter.rows.set(`2`, { + id: `2`, + title: `Recovered`, + }) coordinator.emit({ type: `tx:committed`, @@ -2181,6 +3188,53 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`does not acquire an upstream lease after released hydration fails`, async () => { + const localError = new Error(`persisted rows unavailable`) + const hydrationStarted = deferred() + const hydration = deferred>() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let loads = 0 + let unloads = 0 + const collection = createCollection( + persistedCollectionOptions({ + id: `released-failed-hydration`, + syncMode: `on-demand`, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + loads++ + return true + }, + unloadSubset: () => { + unloads++ + }, + } + }, + }, + persistence: { adapter }, + }), + ) + const options: LoadSubsetOptions = { limit: 1 } + + collection.startSyncImmediate() + const pending = collection._sync.loadSubset(options) + await hydrationStarted.promise + collection._sync.unloadSubset(options) + hydration.reject(localError) + await pending + + expect(loads).toBe(0) + expect(unloads).toBe(0) + await collection.cleanup() + }) + it.each([`abort`, `release`, `offline`] as const)( `handles remote ensure after %s without resurrecting cancelled demand`, async (action) => { diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 8152fd0a54..d80f2981e6 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1984,10 +1984,13 @@ function createElectricSync>( signal: abortController.signal, }) - const resumeKeysPromise = - requiresCompleteResume || freshSnapshotPending - ? whenHydrated?.() - : undefined + // A resumed incremental stream needs the persisted baseline before its + // partial updates can be interpreted. A fresh authoritative snapshot + // does not: its truncate transaction can supersede hydration and make + // the network snapshot visible immediately. + const resumeKeysPromise = requiresCompleteResume + ? whenHydrated?.() + : undefined let areResumeKeysReady = !resumeKeysPromise const pendingResumeBatches: Array>> = [] let unsubscribeStream: () => void = () => {} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 328d4fe5ad..47340d5460 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1072,6 +1072,12 @@ importers: '@journeyapps/wa-sqlite': specifier: ^1.4.1 version: 1.5.0 + '@playwright/test': + specifier: ^1.60.0 + version: 1.60.0 + '@tanstack/electric-db-collection': + specifier: workspace:* + version: link:../electric-db-collection '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 From 5580d776033970696f562e6d57799d465b2035fb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 01:00:04 +0100 Subject: [PATCH 02/10] fix(sqlite-persistence): harden dual-source readiness --- .../fix-persisted-dual-source-readiness.md | 7 + .../src/browser-coordinator.ts | 343 ++++++++++++++---- .../tests/browser-coordinator.test.ts | 333 ++++++++++++++++- .../src/persisted.ts | 240 +++++++----- .../tests/persisted.test.ts | 309 ++++++++++++++++ .../tests/electric-recovery-oracle.test.ts | 6 +- 6 files changed, 1071 insertions(+), 167 deletions(-) create mode 100644 .changeset/fix-persisted-dual-source-readiness.md diff --git a/.changeset/fix-persisted-dual-source-readiness.md b/.changeset/fix-persisted-dual-source-readiness.md new file mode 100644 index 0000000000..1c132d91af --- /dev/null +++ b/.changeset/fix-persisted-dual-source-readiness.md @@ -0,0 +1,7 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/browser-db-sqlite-persistence': patch +'@tanstack/electric-db-collection': patch +--- + +Allow eager persisted collections to become ready from either compatible SQLite hydration or an authoritative upstream snapshot. Serialize browser source snapshots with coordinator mutations and preserve exact startup, supersession, and durability failure behavior. diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 1babddc5a7..73776a0622 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -1,10 +1,13 @@ import { safeRandomUUID } from '@tanstack/db-sqlite-persistence-core' import type { ApplyLocalMutationsResponse, + ApplyPersistedTransactionResponse, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, + PersistedTx, PersistenceAdapter, + PositionlessPersistedTx, ProtocolEnvelope, PullSinceResponse, } from '@tanstack/db-sqlite-persistence-core' @@ -20,6 +23,7 @@ const RPC_RETRY_ATTEMPTS = 2 const RPC_RETRY_DELAY_MS = 200 const WRITER_LOCK_BUSY_RETRY_MS = 50 const WRITER_LOCK_MAX_RETRIES = 20 +const TARGETED_INVALIDATION_KEY_LIMIT = 128 // --------------------------------------------------------------------------- // Internal types @@ -43,6 +47,11 @@ type RPCRequest = envelopeId: string mutations: Array } + | { + type: `rpc:applyPersistedTransaction:req` + rpcId: string + transaction: PositionlessPersistedTx + } | { type: `rpc:pullSince:req` rpcId: string @@ -63,6 +72,7 @@ type RPCResponse = error?: string } | ApplyLocalMutationsResponse + | ApplyPersistedTransactionResponse | PullSinceResponse type PendingRPC = { @@ -105,6 +115,8 @@ type AdapterWithPullSince = PersistenceAdapter & { }> } +class LostLeadershipError extends Error {} + // --------------------------------------------------------------------------- // Options // --------------------------------------------------------------------------- @@ -126,6 +138,15 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private readonly collections = new Map() private readonly pendingRPCs = new Map() private readonly appliedEnvelopeIds = new Map() + private readonly appliedPersistedTransactions = new Map< + string, + { + timestamp: number + term: number + seq: number + latestRowVersion: number + } + >() private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -179,7 +200,8 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } } - publish(_collectionId: string, message: ProtocolEnvelope): void { + publish(collectionId: string, message: ProtocolEnvelope): void { + this.observeEnvelopePosition(collectionId, message) this.channel.postMessage(message) } @@ -267,6 +289,25 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }) } + async requestApplyPersistedTransaction( + collectionId: string, + transaction: PositionlessPersistedTx, + ): Promise { + if (this.isLeader(collectionId)) { + return this.handleApplyPersistedTransaction(collectionId, { + type: `rpc:applyPersistedTransaction:req`, + rpcId: safeRandomUUID(), + transaction, + }) + } + + return this.sendRPC(collectionId, { + type: `rpc:applyPersistedTransaction:req`, + rpcId: safeRandomUUID(), + transaction, + }) + } + async pullSince( collectionId: string, fromRowVersion: number, @@ -439,6 +480,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (!isProtocolEnvelope(data)) return const envelope = data + this.observeEnvelopePosition(envelope.collectionId, envelope) // Ignore own messages if (envelope.senderId === this.nodeId) return @@ -559,6 +601,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina case `rpc:applyLocalMutations:req`: response = await this.handleApplyLocalMutations(collectionId, request) break + case `rpc:applyPersistedTransaction:req`: + response = await this.handleApplyPersistedTransaction( + collectionId, + request, + ) + break case `rpc:pullSince:req`: response = await this.handlePullSince(collectionId, request) break @@ -632,97 +680,205 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina mutations: Array }, ): Promise { - // Dedupe by envelopeId - if (this.appliedEnvelopeIds.has(request.envelopeId)) { + try { + return await this.withWriterLock(async () => { + if (this.appliedEnvelopeIds.has(request.envelopeId)) { + return { + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: false as const, + code: `CONFLICT` as const, + error: `envelope ${request.envelopeId} already applied`, + } + } + + const tx = await this.applyPositionlessTransactionWithWriterLock( + collectionId, + { + txId: safeRandomUUID(), + mutations: request.mutations.map((mutation) => ({ + type: mutation.type, + key: mutation.key, + value: mutation.value, + })), + }, + ) + + this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) + this.pruneAppliedEnvelopeIds() + this.publishCommittedTransaction(collectionId, tx) + + return { + type: `rpc:applyLocalMutations:res` as const, + rpcId: request.rpcId, + ok: true as const, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + acceptedMutationIds: request.mutations.map( + (mutation) => mutation.mutationId, + ), + } + }) + } catch (error) { + if (!(error instanceof LostLeadershipError)) throw error return { type: `rpc:applyLocalMutations:res`, rpcId: request.rpcId, ok: false, - code: `CONFLICT`, - error: `envelope ${request.envelopeId} already applied`, + code: `NOT_LEADER`, + error: `not the leader for ${collectionId}`, } } + } - const state = this.collections.get(collectionId) - if (!state || !state.isLeader) { + private async handleApplyPersistedTransaction( + collectionId: string, + request: { + type: `rpc:applyPersistedTransaction:req` + rpcId: string + transaction: PositionlessPersistedTx + }, + ): Promise { + try { + return await this.withWriterLock(async () => { + const transactionKey = `${collectionId}:${request.transaction.txId}` + const prior = this.appliedPersistedTransactions.get(transactionKey) + if (prior) { + return { + type: `rpc:applyPersistedTransaction:res`, + rpcId: request.rpcId, + ok: true, + txId: request.transaction.txId, + term: prior.term, + seq: prior.seq, + latestRowVersion: prior.latestRowVersion, + } + } + + const tx = await this.applyPositionlessTransactionWithWriterLock( + collectionId, + request.transaction, + ) + this.appliedPersistedTransactions.set(transactionKey, { + timestamp: Date.now(), + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + }) + this.pruneAppliedEnvelopeIds() + this.publishCommittedTransaction(collectionId, tx) + return { + type: `rpc:applyPersistedTransaction:res`, + rpcId: request.rpcId, + ok: true, + txId: tx.txId, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + }) + } catch (error) { + if (!(error instanceof LostLeadershipError)) throw error return { - type: `rpc:applyLocalMutations:res`, + type: `rpc:applyPersistedTransaction:res`, rpcId: request.rpcId, ok: false, code: `NOT_LEADER`, error: `not the leader for ${collectionId}`, } } + } - // Assign stream position - state.latestSeq++ - state.latestRowVersion++ - - const term = state.latestTerm - const seq = state.latestSeq - const rowVersion = state.latestRowVersion + /** Called only from inside the database writer lock. */ + private async applyPositionlessTransactionWithWriterLock( + collectionId: string, + transaction: PositionlessPersistedTx, + ): Promise { + const state = this.collections.get(collectionId) + if (!state?.isLeader) throw new LostLeadershipError() - // Build and apply the persisted transaction - const tx = { - txId: safeRandomUUID(), - term, - seq, - rowVersion, - mutations: request.mutations.map((m) => ({ - type: m.type, - key: m.key, - value: m.value, - })), + const adapter = this.requireAdapter() + if (adapter.getStreamPosition) { + const durablePosition = await adapter.getStreamPosition(collectionId) + if (!this.isLeader(collectionId)) throw new LostLeadershipError() + this.observeCollectionPosition( + state, + durablePosition.latestTerm, + durablePosition.latestSeq, + durablePosition.latestRowVersion, + ) } - await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), - ) - - // Track envelope for dedup - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) - this.pruneAppliedEnvelopeIds() + if (!this.isLeader(collectionId)) throw new LostLeadershipError() + const tx: PersistedTx = { + ...transaction, + term: state.latestTerm, + seq: state.latestSeq + 1, + rowVersion: state.latestRowVersion + 1, + } - // Broadcast tx:committed to all tabs - const changedRows = request.mutations - .filter((m) => m.type !== `delete`) - .map((m) => ({ key: m.key, value: m.value })) - const deletedKeys = request.mutations - .filter((m) => m.type === `delete`) - .map((m) => m.key) + await adapter.applyCommittedTx(collectionId, tx) + this.observeCollectionPosition(state, tx.term, tx.seq, tx.rowVersion) + return tx + } - const txCommitted: ProtocolEnvelope = { + private publishCommittedTransaction( + collectionId: string, + tx: PersistedTx, + ): void { + const changedRows = tx.mutations + .filter((mutation) => mutation.type !== `delete`) + .map((mutation) => ({ key: mutation.key, value: mutation.value })) + const deletedKeys = tx.mutations + .filter((mutation) => mutation.type === `delete`) + .map((mutation) => mutation.key) + const rowMetadataMutations = tx.rowMetadataMutations ?? [] + const collectionMetadataMutations = tx.collectionMetadataMutations ?? [] + const changedKeyCount = + changedRows.length + + deletedKeys.length + + rowMetadataMutations.length + + collectionMetadataMutations.length + const requiresFullReload = + tx.truncate === true || + changedKeyCount === 0 || + changedKeyCount > TARGETED_INVALIDATION_KEY_LIMIT + const payload = requiresFullReload + ? { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + requiresFullReload: true as const, + } + : { + type: `tx:committed` as const, + term: tx.term, + seq: tx.seq, + txId: tx.txId, + latestRowVersion: tx.rowVersion, + requiresFullReload: false as const, + changedRows, + deletedKeys, + rowMetadataMutations, + collectionMetadataMutations, + } + const envelope: ProtocolEnvelope = { v: 1, dbName: this.dbName, collectionId, senderId: this.nodeId, ts: Date.now(), - payload: { - type: `tx:committed`, - term, - seq, - txId: tx.txId, - latestRowVersion: rowVersion, - requiresFullReload: false, - changedRows, - deletedKeys, - }, - } - this.channel.postMessage(txCommitted) - - // Deliver to local subscribers too - for (const subscriber of state.subscribers) { - subscriber(txCommitted) + payload, } + this.observeEnvelopePosition(collectionId, envelope) + this.channel.postMessage(envelope) - return { - type: `rpc:applyLocalMutations:res`, - rpcId: request.rpcId, - ok: true, - term, - seq, - latestRowVersion: rowVersion, - acceptedMutationIds: request.mutations.map((m) => m.mutationId), + const state = this.collections.get(collectionId) + for (const subscriber of state?.subscribers ?? []) { + subscriber(envelope) } } @@ -784,9 +940,17 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina const lockName = `tsdb:writer:${this.dbName}` for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const lockAttempt = { enteredCallback: false } try { - return await navigator.locks.request(lockName, async () => fn()) + return await navigator.locks.request(lockName, async () => { + lockAttempt.enteredCallback = true + return fn() + }) } catch (error) { + // The lock request may transiently fail before entering the callback, + // but adapter and application failures must never be replayed. + if (lockAttempt.enteredCallback) throw error + if (error instanceof DOMException && error.name === `AbortError`) { throw error } @@ -808,6 +972,48 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina // Helpers // ----------------------------------------------------------------------- + private observeEnvelopePosition( + collectionId: string, + envelope: ProtocolEnvelope, + ): void { + const payload = envelope.payload + if (!payload || typeof payload !== `object`) return + const record = payload as Record + if ( + record.type !== `tx:committed` || + typeof record.term !== `number` || + typeof record.seq !== `number` || + typeof record.latestRowVersion !== `number` + ) { + return + } + + const state = this.collections.get(collectionId) + if (!state) return + this.observeCollectionPosition( + state, + record.term, + record.seq, + record.latestRowVersion, + ) + } + + private observeCollectionPosition( + state: CollectionState, + term: number, + seq: number, + rowVersion: number, + ): void { + if ( + term > state.latestTerm || + (term === state.latestTerm && seq > state.latestSeq) + ) { + state.latestTerm = term + state.latestSeq = seq + } + state.latestRowVersion = Math.max(state.latestRowVersion, rowVersion) + } + private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 @@ -816,6 +1022,11 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina this.appliedEnvelopeIds.delete(id) } } + for (const [txId, applied] of this.appliedPersistedTransactions) { + if (applied.timestamp < cutoff) { + this.appliedPersistedTransactions.delete(txId) + } + } } } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 5f554006e2..c14a7c749d 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,7 +1,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createCollection } from '../../db/src' +import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' +import type { + PersistedTx, + PersistenceAdapter, +} from '../../db-sqlite-persistence-core/src' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' -import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' // --------------------------------------------------------------------------- // BroadcastChannel mock @@ -228,6 +233,20 @@ async function flush(ms: number = 10): Promise { await new Promise((resolve) => setTimeout(resolve, ms)) } +function deferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolve!: (value: T) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -409,6 +428,221 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + it(`does not reuse a directly published source transaction position`, async () => { + const adapter = createStubAdapter() + const durableRows = new Map() + const occupiedPositions = new Set() + adapter.applyCommittedTx = (collectionId, tx) => { + const position = `${collectionId}:${tx.term}:${tx.seq}` + if (occupiedPositions.has(position)) return Promise.resolve() + occupiedPositions.add(position) + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + if (tx.truncate) durableRows.clear() + for (const mutation of tx.mutations) { + if (mutation.type === `delete`) { + durableRows.delete(mutation.key) + } else { + durableRows.set(mutation.key, mutation.value) + } + } + return Promise.resolve() + } + const coord = createCoordinator(adapter) + coord.subscribe(`todos`, () => {}) + await flush(50) + expect(coord.isLeader(`todos`)).toBe(true) + + await adapter.applyCommittedTx(`todos`, { + txId: `authoritative-source-snapshot`, + term: 1, + seq: 1, + rowVersion: 1, + truncate: true, + mutations: [ + { + type: `insert`, + key: `network`, + value: { id: `network`, title: `Network snapshot` }, + }, + ], + }) + coord.publish(`todos`, { + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: coord.getNodeId(), + ts: Date.now(), + payload: { + type: `tx:committed`, + term: 1, + seq: 1, + txId: `authoritative-source-snapshot`, + latestRowVersion: 1, + requiresFullReload: true, + }, + }) + + const response = await coord.requestApplyLocalMutations(`todos`, [ + { + mutationId: `local-after-network`, + type: `insert`, + key: `local`, + value: { id: `local`, title: `Local mutation` }, + }, + ]) + + expect({ + ok: response.ok, + seq: response.ok ? response.seq : undefined, + durableKeys: Array.from(durableRows.keys()).sort(), + }).toEqual({ + ok: true, + seq: 2, + durableKeys: [`local`, `network`], + }) + coord.dispose() + }) + + it(`serializes a network winner with a competing coordinator mutation`, async () => { + type Todo = { id: string; title: string } + + const adapter = createStubAdapter() + const hydrationStarted = deferred() + const hydration = + deferred< + Array<{ key: string | number; value: Record }> + >() + const startNetwork = deferred() + const upstreamDone = deferred() + const snapshotApplyEntered = deferred() + const releaseSnapshotApply = deferred() + const durableRows = new Map>() + const occupiedPositions = new Set() + + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + adapter.applyCommittedTx = async (collectionId, tx) => { + if (tx.truncate) { + snapshotApplyEntered.resolve(tx) + await releaseSnapshotApply.promise + } + + const position = `${collectionId}:${tx.term}:${tx.seq}` + if (occupiedPositions.has(position)) return + occupiedPositions.add(position) + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + if (tx.truncate) durableRows.clear() + for (const mutation of tx.mutations) { + if (mutation.type === `delete`) { + durableRows.delete(mutation.key) + } else { + durableRows.set(mutation.key, mutation.value) + } + } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + const collection = createCollection( + persistedCollectionOptions({ + id: `todos`, + getKey: (todo) => todo.id, + sync: { + sync: ({ begin, truncate, write, commit, markReady, metadata }) => { + void (async () => { + await startNetwork.promise + begin() + metadata?.collection.set(`cursor`, `network-cursor`) + truncate() + write({ + type: `insert`, + value: { id: `network`, title: `Network snapshot` }, + metadata: { owner: `network` }, + }) + const applied = commit() + if (applied !== true) void applied.catch(() => undefined) + markReady() + })().then(upstreamDone.resolve, upstreamDone.reject) + return {} + }, + }, + persistence: { adapter, coordinator: leader }, + }), + ) + + try { + const preload = collection.preload() + await hydrationStarted.promise + await flush(50) + expect(leader.isLeader(`todos`)).toBe(true) + follower.subscribe(`todos`, () => {}) + await flush(50) + expect(follower.isLeader(`todos`)).toBe(false) + + startNetwork.resolve() + await upstreamDone.promise + hydration.resolve([]) + await preload + const sourceTx = await snapshotApplyEntered.promise + expect(sourceTx).toMatchObject({ + truncate: true, + mutations: [ + { + key: `network`, + value: { id: `network`, title: `Network snapshot` }, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `network`, value: { owner: `network` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `network-cursor` }, + ], + }) + + let responseSettled = false + const responsePromise = follower + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `peer-while-source-apply-is-held`, + type: `insert`, + key: `peer`, + value: { id: `peer`, title: `Peer mutation` }, + }, + ]) + .then((response) => { + responseSettled = true + return response + }) + await flush() + const responseSettledBeforeSourceApply = responseSettled + releaseSnapshotApply.resolve() + const response = await responsePromise + await flush() + await collection.cleanup() + + expect({ + responseSettledBeforeSourceApply, + responseSeq: response.ok ? response.seq : undefined, + sourceSeq: sourceTx.seq, + durableKeys: Array.from(durableRows.keys()).sort(), + }).toEqual({ + responseSettledBeforeSourceApply: false, + responseSeq: sourceTx.seq + 1, + sourceSeq: sourceTx.seq, + durableKeys: [`network`, `peer`], + }) + } finally { + releaseSnapshotApply.resolve() + hydration.resolve([]) + await collection.cleanup() + leader.dispose() + follower.dispose() + } + }) + it(`follower routes mutations to leader via RPC`, async () => { const adapter = createStubAdapter() const leader = createCoordinator(adapter) @@ -440,6 +674,103 @@ describe(`BrowserCollectionCoordinator`, () => { follower.dispose() }) + it(`routes full persisted transactions through the leader without losing data`, async () => { + const adapter = createStubAdapter() + let appliedTx: PersistedTx | undefined + adapter.applyCommittedTx = (collectionId, tx) => { + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + appliedTx = tx + return Promise.resolve() + } + const leader = createCoordinator(adapter) + leader.subscribe(`todos`, () => {}) + await flush(50) + const follower = createCoordinator(adapter) + follower.subscribe(`todos`, () => {}) + await flush(50) + + const response = await follower.requestApplyPersistedTransaction( + `todos`, + { + txId: `source-with-metadata`, + truncate: true, + mutations: [ + { + type: `insert`, + key: `network`, + value: { id: `network`, title: `Network snapshot` }, + metadata: { owner: `source` }, + metadataChanged: true, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `network`, value: { owner: `source` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `cursor-1` }, + ], + }, + ) + + expect(response.ok).toBe(true) + expect(appliedTx).toMatchObject({ + txId: `source-with-metadata`, + truncate: true, + mutations: [ + { + type: `insert`, + key: `network`, + value: { id: `network`, title: `Network snapshot` }, + metadata: { owner: `source` }, + metadataChanged: true, + }, + ], + rowMetadataMutations: [ + { type: `set`, key: `network`, value: { owner: `source` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `cursor`, value: `cursor-1` }, + ], + }) + expect(appliedTx?.term).toBeGreaterThan(0) + expect(appliedTx?.seq).toBeGreaterThan(0) + expect(appliedTx?.rowVersion).toBeGreaterThan(0) + + leader.dispose() + follower.dispose() + }) + + it(`does not retry or consume a position when adapter application fails`, async () => { + const adapter = createStubAdapter() + const persistenceError = new Error(`write failed`) + let applyAttempts = 0 + adapter.applyCommittedTx = (collectionId, tx) => { + applyAttempts++ + if (applyAttempts === 1) return Promise.reject(persistenceError) + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + return Promise.resolve() + } + const coord = createCoordinator(adapter) + coord.subscribe(`todos`, () => {}) + await flush(50) + + await expect( + coord.requestApplyPersistedTransaction(`todos`, { + txId: `failed-source`, + mutations: [], + }), + ).rejects.toBe(persistenceError) + + const response = await coord.requestApplyPersistedTransaction(`todos`, { + txId: `successful-source`, + mutations: [], + }) + expect(response).toMatchObject({ ok: true, seq: 1, latestRowVersion: 1 }) + expect(applyAttempts).toBe(2) + + coord.dispose() + }) + it(`deduplicates envelope ids`, async () => { const adapter = createStubAdapter() const coord = createCoordinator(adapter) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index c1f7b4f14e..b2c6a1b7e5 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -249,6 +249,29 @@ export type PersistedTx< collectionMetadataMutations?: Array } +export type PositionlessPersistedTx< + T extends object = Record, + TKey extends string | number = string | number, +> = Omit, `term` | `seq` | `rowVersion`> + +export type ApplyPersistedTransactionResponse = + | { + type: `rpc:applyPersistedTransaction:res` + rpcId: string + ok: true + txId: string + term: number + seq: number + latestRowVersion: number + } + | { + type: `rpc:applyPersistedTransaction:res` + rpcId: string + ok: false + code: `NOT_LEADER` | `VALIDATION_ERROR` | `CONFLICT` | `TIMEOUT` + error: string + } + export interface PersistenceAdapter { loadSubset: ( collectionId: string, @@ -319,6 +342,10 @@ export interface PersistedCollectionCoordinator { collectionId: string, mutations: Array, ) => Promise + requestApplyPersistedTransaction?: ( + collectionId: string, + transaction: PositionlessPersistedTx, + ) => Promise pullSince?: ( collectionId: string, fromRowVersion: number, @@ -554,6 +581,26 @@ function isRecord(value: unknown): value is Record { return typeof value === `object` && value !== null } +function isAbortFailure(error: unknown, signal?: AbortSignal): boolean { + return ( + signal?.aborted === true || + (typeof error === `object` && + error !== null && + `name` in error && + error.name === `AbortError`) + ) +} + +function createLocalUpstreamAggregateError( + localError: unknown, + upstreamError: unknown, + message: string, +): AggregateError { + return new AggregateError([localError, upstreamError], message, { + cause: localError, + }) +} + function isValidSyncConfig(value: unknown): value is SyncConfig { if (!isRecord(value)) { return false @@ -606,6 +653,10 @@ type OpenSyncTransaction< } class PersistedHydrationSupersededError extends Error { + constructor() { + super(`Persisted hydration was superseded by an upstream snapshot`) + } + override readonly name = `PersistedHydrationSupersededError` } @@ -893,9 +944,7 @@ class PersistedCollectionRuntime< if (!this.isHydratingNow()) return false this.hydrationSupersessionGeneration = this.lifecycleGeneration this.hydratingGeneration = null - const error = new PersistedHydrationSupersededError( - `Persisted hydration was superseded by an upstream snapshot`, - ) + const error = new PersistedHydrationSupersededError() for (const transaction of this.queuedHydrationTransactions) { transaction.rejectApplied?.(error) } @@ -1107,14 +1156,7 @@ class PersistedCollectionRuntime< localFailure = error } - if ( - localFailed && - (options.signal?.aborted || - (typeof localFailure === `object` && - localFailure !== null && - `name` in localFailure && - localFailure.name === `AbortError`)) - ) { + if (localFailed && isAbortFailure(localFailure, options.signal)) { throw localFailure } @@ -1122,23 +1164,17 @@ class PersistedCollectionRuntime< try { await upstreamLoadSubset(options) } catch (error) { - if ( - options.signal?.aborted || - (typeof error === `object` && - error !== null && - `name` in error && - error.name === `AbortError`) - ) { + if (isAbortFailure(error, options.signal)) { this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) throw error } console.warn(`Failed to trigger remote subset load:`, error) this.queueRemoteSubsetEnsure(options) if (localFailed) { - throw new AggregateError( - [localFailure, error], + throw createLocalUpstreamAggregateError( + localFailure, + error, `Persisted and upstream subset loading both failed`, - { cause: localFailure }, ) } // Hydration remains readable, but it does not satisfy remote demand. @@ -1384,9 +1420,7 @@ class PersistedCollectionRuntime< const rows = await this.loadSubsetRowsUnsafe(options) if (config.lifecycleGeneration !== this.lifecycleGeneration) return if (this.hydratingGeneration !== config.lifecycleGeneration) { - throw new PersistedHydrationSupersededError( - `Persisted hydration was superseded by an upstream snapshot`, - ) + throw new PersistedHydrationSupersededError() } this.applyRowsToCollection(rows) @@ -1417,7 +1451,10 @@ class PersistedCollectionRuntime< await this.flushQueuedTxCommittedUnsafe() } - if (config.requestRemoteEnsure) { + const hydrationAborted = + hydrationFailed && isAbortFailure(hydrationFailure, options.signal) + + if (config.requestRemoteEnsure && !hydrationAborted) { this.queueRemoteSubsetEnsure(options) } @@ -1583,12 +1620,11 @@ class PersistedCollectionRuntime< await applied } - if (!transaction.internal) { - if (transaction.lifecycleGeneration === this.lifecycleGeneration) { - await this.persistAndBroadcastExternalSyncTransactionUnsafe( - transaction, - ) - } + if ( + !transaction.internal && + transaction.lifecycleGeneration === this.lifecycleGeneration + ) { + await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) } transaction.resolveApplied?.() } catch (error) { @@ -1604,6 +1640,27 @@ class PersistedCollectionRuntime< return } + if (this.persistence.coordinator.requestApplyPersistedTransaction) { + const response = + await this.persistence.coordinator.requestApplyPersistedTransaction( + this.collectionId, + this.createPositionlessPersistedTxFromOperations(transaction), + ) + + if (!response.ok) { + throw new Error( + `failed to apply external sync transaction through coordinator: ${response.error}`, + ) + } + + this.observeStreamPosition( + response.term, + response.seq, + response.latestRowVersion, + ) + return + } + const streamPosition = this.nextLocalStreamPosition() if ( @@ -1656,10 +1713,18 @@ class PersistedCollectionRuntime< streamPosition: { term: number; seq: number; rowVersion: number }, ): PersistedTx { return { - txId: safeRandomUUID(), + ...this.createPositionlessPersistedTxFromOperations(transaction), term: streamPosition.term, seq: streamPosition.seq, rowVersion: streamPosition.rowVersion, + } + } + + private createPositionlessPersistedTxFromOperations( + transaction: BufferedSyncTransaction, + ): PositionlessPersistedTx { + return { + txId: safeRandomUUID(), truncate: transaction.truncate, mutations: transaction.operations.map((operation) => operation.type === `update` @@ -2328,6 +2393,8 @@ class PersistedCollectionRuntime< } } + if (this.hydratingGeneration !== lifecycleGeneration) return + this.replaceCollectionSnapshot( Array.from(mergedRows.entries()).map(([key, row]) => ({ key, @@ -2455,7 +2522,6 @@ function createWrappedSyncConfig< const transactionStack: Array> = [] const getOpenTransaction = () => transactionStack[transactionStack.length - 1] - let fullStartPromise: Promise | null = null const startupState: { cleanedUp: boolean signalled: `loading` | `ready` | `error` @@ -2493,10 +2559,10 @@ function createWrappedSyncConfig< startupState.signalled = `error` const error = startupState.local === `failed` - ? new AggregateError( - [startupState.localError, startupState.upstreamError], + ? createLocalUpstreamAggregateError( + startupState.localError, + startupState.upstreamError, `Persisted collection startup failed locally and upstream`, - { cause: startupState.localError }, ) : startupState.upstreamError params.markError(error) @@ -2532,10 +2598,10 @@ function createWrappedSyncConfig< if (startupState.signalled === `error`) { if (startupState.upstream === `failed`) { params.markError( - new AggregateError( - [error, startupState.upstreamError], + createLocalUpstreamAggregateError( + error, + startupState.upstreamError, `Persisted collection failed locally and upstream`, - { cause: error }, ), ) } @@ -2551,17 +2617,17 @@ function createWrappedSyncConfig< reconcileAvailability() } const signalUpstreamFailure = (error: unknown) => { - const failedAfterReady = startupState.signalled === `ready` + const failedAfterUpstreamReady = startupState.upstream === `ready` startupState.upstream = `failed` startupState.upstreamError = error - if (failedAfterReady) { + if (failedAfterUpstreamReady) { startupState.signalled = `error` params.markError( startupState.local === `failed` - ? new AggregateError( - [startupState.localError, error], + ? createLocalUpstreamAggregateError( + startupState.localError, + error, `Persisted collection failed locally and upstream`, - { cause: startupState.localError }, ) : error, ) @@ -2570,10 +2636,10 @@ function createWrappedSyncConfig< if (startupState.signalled === `error`) { if (startupState.local === `failed`) { params.markError( - new AggregateError( - [startupState.localError, error], + createLocalUpstreamAggregateError( + startupState.localError, + error, `Persisted collection failed locally and upstream`, - { cause: startupState.localError }, ), ) } @@ -2866,35 +2932,28 @@ function createWrappedSyncConfig< return applied } - let applied: SyncAppliedReceipt - try { - applied = params.commit(signal) - } catch (error) { + const finishSupersession = () => { if (openTransaction.supersededHydration) { runtime.finishHydrationSupersession( openTransaction.lifecycleGeneration, ) } + } + let applied: SyncAppliedReceipt + try { + applied = params.commit(signal) + } catch (error) { + finishSupersession() throw error } if (!openTransaction.internal) { const persist = async () => { try { - await runtime.persistAndBroadcastExternalSyncTransaction({ - lifecycleGeneration: openTransaction.lifecycleGeneration, - operations: openTransaction.operations, - rowMetadataWrites: openTransaction.rowMetadataWrites, - collectionMetadataWrites: - openTransaction.collectionMetadataWrites, - truncate: openTransaction.truncate, - internal: false, - }) + await runtime.persistAndBroadcastExternalSyncTransaction( + openTransaction, + ) } finally { - if (openTransaction.supersededHydration) { - runtime.finishHydrationSupersession( - openTransaction.lifecycleGeneration, - ) - } + finishSupersession() } } if (openTransaction.supersededHydration) { @@ -2902,33 +2961,27 @@ function createWrappedSyncConfig< signalUpstreamReady() void persist().catch(signalPersistenceFailure) } else { - void applied.then( - () => { - signalUpstreamReady() - return persist().catch(signalPersistenceFailure) - }, - () => - runtime.finishHydrationSupersession( - openTransaction.lifecycleGeneration, - ), - ) + void applied.then(() => { + signalUpstreamReady() + return persist().catch(signalPersistenceFailure) + }, finishSupersession) } return applied } const persistAfterApplication = async () => { if (applied !== true) await applied - await persist() + const persisted = persist() + void persisted.catch(signalPersistenceFailure) + return persisted } - const persisted = persistAfterApplication() - void persisted.catch(() => undefined) - return persisted + return persistAfterApplication() } return applied }, } let sourceResult: SyncConfigRes = {} - fullStartPromise = runtime.ensureStarted() + const fullStartPromise = runtime.ensureStarted() const sourceResultPromise = (async () => { try { await runtime.ensureStartupMetadataLoaded() @@ -2997,13 +3050,7 @@ function createWrappedSyncConfig< } } if (localStartupFailed) { - if ( - options.signal?.aborted || - (typeof localStartupFailure === `object` && - localStartupFailure !== null && - `name` in localStartupFailure && - localStartupFailure.name === `AbortError`) - ) { + if (isAbortFailure(localStartupFailure, options.signal)) { throw localStartupFailure } if (!resolvedSourceResult.loadSubset) { @@ -3012,19 +3059,13 @@ function createWrappedSyncConfig< try { await loadFromUpstream(options) } catch (upstreamError) { - if ( - options.signal?.aborted || - (typeof upstreamError === `object` && - upstreamError !== null && - `name` in upstreamError && - upstreamError.name === `AbortError`) - ) { + if (isAbortFailure(upstreamError, options.signal)) { throw upstreamError } - throw new AggregateError( - [localStartupFailure, upstreamError], + throw createLocalUpstreamAggregateError( + localStartupFailure, + upstreamError, `Persisted and upstream subset startup both failed`, - { cause: localStartupFailure }, ) } return @@ -3133,6 +3174,7 @@ export function persistedCollectionOptions< const { schemaVersion, ...syncOptions } = options const collectionId = syncOptions.id ?? `persisted-collection:${safeRandomUUID()}` + const syncMode = syncOptions.syncMode ?? `eager` const persistence = resolvePersistenceForCollection( syncOptions.persistence, { @@ -3146,7 +3188,7 @@ export function persistedCollectionOptions< `sync-present`, collectionId, persistence, - syncOptions.syncMode ?? `eager`, + syncMode, collectionId, ) @@ -3156,7 +3198,7 @@ export function persistedCollectionOptions< sync: createWrappedSyncConfig( syncOptions.sync, runtime, - syncOptions.syncMode ?? `eager`, + syncMode, ), persistence, } diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 8d10c9f911..13806014b2 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -3,6 +3,7 @@ import { BasicIndex, DbClient, IR, + SyncTransactionAbortedError, collectionOptions, createCollection, createTransaction, @@ -885,6 +886,85 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`does not report an application receipt rejection as a persistence failure`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-application-rejection`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const mutationGate = deferred() + const transaction = createTransaction({ + mutationFn: () => mutationGate.promise, + }) + + try { + await collection.stateWhenReady() + await flushAsyncWork() + transaction.mutate(() => { + collection.insert({ id: `local`, title: `Optimistic gate` }) + }) + expect(transaction.state).toBe(`persisting`) + + const abortController = new AbortController() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `remote`, title: `Canceled before application` }, + }) + const receipt = remoteCommit?.(abortController.signal) + expect(receipt).toBeInstanceOf(Promise) + if (receipt === true || receipt === undefined) { + throw new Error(`Persisting optimistic work did not hold remote sync`) + } + + const applicationReceipt = + collection._state.pendingSyncedTransactions.at(-1)?.applied.promise + if (!applicationReceipt) { + throw new Error(`Expected a pending application receipt`) + } + const applicationRejection = applicationReceipt.catch((error) => error) + + abortController.abort() + const [applicationError, callerError] = await Promise.all([ + applicationRejection, + receipt.catch((error) => error), + ]) + await flushAsyncWork() + + expect(applicationError).toBeInstanceOf(SyncTransactionAbortedError) + expect(callerError).toBe(applicationError) + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + expect(collection.get(`remote`)).toBeUndefined() + } finally { + mutationGate.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }) + it(`persists a wrapped sync transaction when abort follows application`, async () => { const adapter = createRecordingAdapter() let remoteBegin: (() => void) | undefined @@ -1741,6 +1821,37 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`keeps a successful local snapshot ready when the upstream first fails later`, async () => { + const upstreamError = new Error(`delayed upstream startup failure`) + const rows = [{ id: `1`, title: `Offline Todo` }] + let failUpstream: ((error: unknown) => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `local-success-before-upstream-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ markError }) => { + failUpstream = markError + return {} + }, + }, + persistence: { adapter: createRecordingAdapter(rows) }, + }), + ) + + await collection.preload() + expect(collection.status).toBe(`ready`) + expect(collection.toArray.map(stripVirtualProps)).toEqual(rows) + + expect(failUpstream).toBeTypeOf(`function`) + failUpstream!(upstreamError) + + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + expect(collection.toArray.map(stripVirtualProps)).toEqual(rows) + await collection.cleanup() + }) + it(`keeps waiting for upstream after local hydration fails`, async () => { const localAttempted = deferred() const localError = new Error(`local startup failed`) @@ -2000,6 +2111,68 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`reports persistence failure when a network snapshot follows local readiness`, async () => { + const persistenceError = new Error(`post-local-ready persistence failed`) + const persistenceAttempted = deferred() + const adapter = createRecordingAdapter([ + { id: `local`, title: `Local snapshot` }, + ]) + let remoteBegin: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + let recoverUpstream: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `local-first-network-persistence-failure`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, truncate, write, commit, markReady }) => { + remoteBegin = begin + remoteTruncate = truncate + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + recoverUpstream = markReady + return {} + }, + }, + persistence: { adapter }, + }), + ) + + await collection.preload() + expect(collection.status).toBe(`ready`) + adapter.applyCommittedTx = () => { + persistenceAttempted.resolve() + return Promise.reject(persistenceError) + } + + remoteBegin!() + remoteTruncate!() + remoteWrite!({ + type: `insert`, + value: { id: `network`, title: `Network snapshot` }, + }) + const applied = remoteCommit!() + if (applied !== true) void applied.catch(() => undefined) + await persistenceAttempted.promise + await flushAsyncWork() + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(persistenceError) + + recoverUpstream!() + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + await collection.cleanup() + }) + it(`recovers when explicit upstream readiness follows network winner persistence failure`, async () => { const hydrationStarted = deferred() const hydration = deferred>() @@ -2836,6 +3009,90 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`does not let a stale invalidation reload overwrite an authoritative snapshot`, async () => { + const adapter = createRecordingAdapter([ + { id: `cached`, title: `Cached snapshot` }, + ]) + const coordinator = createCoordinatorHarness() + const reloadStarted = deferred() + const releaseReload = deferred() + const originalLoadSubset = adapter.loadSubset.bind(adapter) + let loadCalls = 0 + adapter.loadSubset = async (...args) => { + loadCalls++ + if (loadCalls !== 2) return originalLoadSubset(...args) + const staleRows = [ + { + key: `stale`, + value: { id: `stale`, title: `Stale reload` }, + }, + ] + reloadStarted.resolve() + await releaseReload.promise + return staleRows + } + let remoteBegin: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, truncate, write, commit }) => { + remoteBegin = begin + remoteTruncate = truncate + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `full-reload-before-authoritative-snapshot`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await reloadStarted.promise + + remoteBegin!() + remoteTruncate!() + remoteWrite!({ + type: `insert`, + value: { id: `network`, title: `Authoritative network snapshot` }, + }) + const applied = remoteCommit!() + if (applied !== true) void applied.catch(() => undefined) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Authoritative network snapshot`, + }) + + releaseReload.resolve() + await flushAsyncWork() + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Authoritative network snapshot`, + }) + expect(collection.get(`stale`)).toBeUndefined() + await collection.cleanup() + }) + it(`does not let stale reload metadata start row loading after restart`, async () => { const adapter = createRecordingAdapter([{ id: `1`, title: `Initial` }]) const coordinator = createCoordinatorHarness() @@ -3300,6 +3557,58 @@ describe(`persistedCollectionOptions`, () => { }, ) + it(`does not start or retry remote demand after local hydration aborts`, async () => { + vi.useFakeTimers() + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const abortError = Object.assign(new Error(`local hydration aborted`), { + name: `AbortError`, + }) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => Promise.reject(abortError) + const ensure = vi.fn(async () => { + throw new Error(`remote ensure must not run for cancelled demand`) + }) + const upstreamLoadSubset = vi.fn(async (): Promise => {}) + const coordinator: PersistedCollectionCoordinator = { + getNodeId: () => `local-abort-ensure`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => true, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestEnsureRemoteSubset: ensure, + } + const collection = createCollection( + persistedCollectionOptions({ + id: `local-hydration-abort`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset: upstreamLoadSubset } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + try { + collection.startSyncImmediate() + await expect(collection._sync.loadSubset({ limit: 1 })).rejects.toBe( + abortError, + ) + await vi.advanceTimersByTimeAsync(500) + + expect(upstreamLoadSubset).not.toHaveBeenCalled() + expect(ensure).not.toHaveBeenCalled() + } finally { + await collection.cleanup() + warning.mockRestore() + vi.useRealTimers() + } + }) + it(`retries queued remote subset ensure after transient failures`, async () => { const adapter = createRecordingAdapter() let ensureCalls = 0 diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 00b3fc09b4..225fecec7e 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -463,7 +463,11 @@ describe(`persisted Electric recovery laws`, () => { upToDate, ]) f.record(`after invalid resume`) - await vi.waitFor(() => expect(f.collection.status).toBe(`error`)) + await vi.waitFor(() => + expect(f.collection.status).toBe( + syncMode === `eager` ? `ready` : `error`, + ), + ) await vi.waitFor(() => expect(f.metadata.get(`electric:resume`)).toMatchObject({ kind: `reset`, From 1fc0ab789ab36a5ed2204ca661e018ffb8076a13 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 09:49:59 +0100 Subject: [PATCH 03/10] docs: explain persisted readiness oracles --- ...lectric-coordinator-readiness.opfs.spec.ts | 18 ++++++++++ .../electric-coordinator-readiness.opfs.ts | 24 ++++++++++++++ .../tests/browser-coordinator.test.ts | 27 +++++++++++++++ .../tests/persisted.test.ts | 31 +++++++++++++++++ .../tests/electric-recovery-oracle.test.ts | 33 +++++++++++++++++++ 5 files changed, 133 insertions(+) diff --git a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts index bcb5bb0aa6..13f49bed35 100644 --- a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts +++ b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts @@ -2,6 +2,24 @@ import { expect, test } from '@playwright/test' import type { Page } from '@playwright/test' import type { ReadinessOracleResult } from './electric-coordinator-readiness.opfs' +/** + * # Does the Chromium driver refine dual-source readiness? + * + * The contract is the same independent dual-source law as the core suite: an + * eager Collection becomes ready from a compatible local snapshot or an + * authoritative upstream source snapshot, while on-demand remains upstream + * gated. The four literal expected cells are independent of the browser driver + * and are bounded exhaustiveness over this declared matrix. + * + * Each test reads the fixture only after its observation checkpoint, then + * compares exact public rows, Collection status, ready-event and work counters, + * provider identity, race ordering, and cleanup results. Replay by Playwright + * title (the eager title includes its mode). A pre-fix network winner remains + * loading or exposes the stale OPFS row; a pre-fix eager local case remains + * blocked on Electric. The fixture limits still apply: one Chromium context, + * controlled HTTP responses, no live Electric service, and no PowerSync path. + */ + type Mode = `non-empty` | `empty` | `network-wins` | `on-demand` async function readResult( diff --git a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts index 009e0146e7..4ecaa8b794 100644 --- a/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts +++ b/packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.ts @@ -8,6 +8,30 @@ import { } from '../src/index' import type { ElectricCollectionUtils } from '@tanstack/electric-db-collection' +/** + * # Which browser path supplies persisted readiness evidence? + * + * This fixture drives a real Chromium `OPFSCoopSyncVFS`, + * `BrowserCollectionCoordinator`, persisted Collection, and Electric + * ShapeStream. A controlled fetch transport holds or supplies upstream data so + * the four declared histories are deterministic: eager empty and non-empty + * SQLite snapshots, on-demand pending upstream, and an authoritative network + * snapshot that wins while hydration is held. + * + * The companion spec supplies the independent expected observations. This + * driver records exact public rows, Collection status, ready-event count, + * hydration calls, upstream requests, and whether publication preceded the + * hydration-release checkpoint. `failed-before-checkpoint` distinguishes setup + * failure from completed evidence, and cleanup diagnostics remain separate from + * the primary observation. + * + * Replay by the companion Playwright test title and mode. The held-hydration + * network case kills the pre-fix stale-local/indefinite-loading behavior; the + * eager cases kill upstream-only readiness. Scope is one Chromium page/context + * with synthetic HTTP responses: it is neither a live Electric service nor + * multi-tab or PowerSync evidence. + */ + type Row = { id: string; title: string } type Mode = `non-empty` | `empty` | `network-wins` | `on-demand` diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index c14a7c749d..a6b9787ce1 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -8,6 +8,33 @@ import type { } from '../../db-sqlite-persistence-core/src' import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordinator' +/** + * # Which transaction owns the next durable coordinator position? + * + * Contract: the leader-owned writer lock serializes position allocation, + * durable application, coordinator-state advancement, and publication. A + * direct authoritative source transaction and a competing follower RPC must + * therefore receive distinct term/sequence positions without losing rows, + * metadata, truncate intent, or failure identity. + * + * The history grammar covers direct source application, competing follower + * mutation RPC, leader routing, full transaction fidelity, and adapter failure. + * Expected results come from the public ordering law: positions are distinct, + * durable rows and metadata equal the submitted transactions, and a failed + * application neither retries nor consumes the next position. + * + * The driver calls the real `BrowserCollectionCoordinator` methods with + * controlled BroadcastChannel and Web Locks implementations. Checkpoints observe + * RPC settlement, term/sequence values, durable adapter input, and publication. + * These simulated browser primitives do not prove native lock or multi-tab + * behavior; the single-context Chromium/OPFS evidence lives in the readiness + * E2E suite and is not a two-context browser proof. + * + * Replay by exact Vitest title. The source-position and held-lock schedules are + * hostile pre-fix witnesses: reusing sequence 1 dropped the network winner, and + * releasing the competing mutation early violated source-before-follower order. + */ + // --------------------------------------------------------------------------- // BroadcastChannel mock // --------------------------------------------------------------------------- diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 13806014b2..fd8c8ca215 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -34,6 +34,37 @@ import type { SyncConfig, } from '@tanstack/db' +/** + * # Which startup source makes a persisted Collection ready? + * + * Contract and source: the sync-mode contract recorded in + * `.changeset/fix-persisted-dual-source-readiness.md`. An eager Collection + * becomes ready after either compatible SQLite hydration or an authoritative + * upstream source snapshot succeeds. On-demand Collections remain + * upstream-gated, and startup enters error only after every available startup + * path fails. + * + * The deterministic history grammar below controls local hydration, upstream + * readiness and sync transactions, durable application, cleanup, and restart. + * It covers empty and non-empty local snapshots, either source winning, dual + * failure and recovery, post-ready durability failure, and stale work. The + * expected relation is independent of the implementation queues: the first + * usable startup result establishes Collection readiness, an authoritative + * upstream winner cannot be overwritten by late hydration, and cleanup fences + * the prior sync run. + * + * The production driver is `persistedCollectionOptions` through real Collection + * status, reads, applied receipts, and persistence/coordinator boundaries. + * Checkpoints compare exact public rows, Collection status and errors, durable + * calls, and settlement before controlled gates are released. + * + * These are pinned schedules, not a generated lifecycle model. Replay one with + * its exact Vitest title. The schedules retain pre-fix kills for stale overwrite, + * false readiness errors, misclassified receipt rejection, remote-ensure retry, + * and hidden durability failure. PowerSync does not currently use this + * authoritative-truncate path. + */ + type Todo = { id: string title: string diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 225fecec7e..1ab3f1e68d 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -14,6 +14,39 @@ import type { } from '../../db-sqlite-persistence-core/src' import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' +/** + * # Does persisted Electric recovery publish only complete source snapshots? + * + * Contract and source: persisted dual-source readiness, authoritative truncate + * replacement, and Electric resume/reset semantics. SQLite hydration and the + * upstream source are independent startup authorities; a valid resume may merge + * deltas, while an invalid resume must replace omitted cached rows only when the + * new source snapshot is complete. + * + * The fixture's `rows` and `metadata` Maps model durable state independently of + * the Collection. The generated property's `expected` Map models complete public + * and durable rows. `expectWholeRecoveryTrace` allows only monotonic movement + * through the explicitly listed public snapshots; it does not copy production's + * replay state machine. + * + * Histories cross eager, progressive, and on-demand sync modes; external + * coordinator publication and ShapeStream deltas; full reload and delta paths; + * insert, update, and delete; valid and invalid resume; empty and non-empty + * replacement; and hydration before or after the final source commit. + * + * The production driver is `persistedCollectionOptions` composed with + * `electricCollectionOptions` and mocked installed ShapeStream callbacks. + * Observation cuts include coordinator metadata publication, `up-to-date`, + * restart, resume metadata, and exact public and durable rows. + * + * Replay matrices by exact Vitest title; replay the generated property with the + * fast-check seed and path printed on failure. The trace fault control rejects a + * missing intermediate publication, while exact row checks reject stale cached + * rows with fresh metadata and incomplete replacement. This is a fixed matrix + * plus a bounded generated property over a mocked ShapeStream, not a live + * Electric service or PowerSync authority. + */ + type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } From 3f6aa449ef2f825ef2e9c83e56d3debb2fca3d43 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 14:22:01 +0100 Subject: [PATCH 04/10] fix(sqlite-persistence): fence queued hydration races --- .../fix-persisted-dual-source-readiness.md | 2 +- docs/contributing/oracle-coverage.md | 2 +- .../src/persisted.ts | 109 ++++-- .../tests/persisted.test.ts | 312 +++++++++++++++++- 4 files changed, 396 insertions(+), 29 deletions(-) diff --git a/.changeset/fix-persisted-dual-source-readiness.md b/.changeset/fix-persisted-dual-source-readiness.md index 1c132d91af..1da709e20c 100644 --- a/.changeset/fix-persisted-dual-source-readiness.md +++ b/.changeset/fix-persisted-dual-source-readiness.md @@ -4,4 +4,4 @@ '@tanstack/electric-db-collection': patch --- -Allow eager persisted collections to become ready from either compatible SQLite hydration or an authoritative upstream snapshot. Serialize browser source snapshots with coordinator mutations and preserve exact startup, supersession, and durability failure behavior. +Allow eager persisted collections to become ready from either a compatible SQLite snapshot, including an empty snapshot, or an authoritative upstream snapshot. Keep usable local rows visible after a later upstream failure, while durable-write failures still enter the Collection error state and a later upstream ready signal may recover it. Serialize browser source snapshots with coordinator mutations and preserve exact startup, supersession, and durability failure behavior. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 763060ab54..7bb1edb66c 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -54,7 +54,7 @@ comment and the current API/architecture contract before extending its model. | Opaque backend pagination | [window oracle](../../packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](../../packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](../../packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](../../packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](../../packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](../../packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [browser OPFS lifecycle](../../packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](../../packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, exact driver results, controlled page/worker ownership, and diagnostic-cause retention. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [Electric recovery](../../packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [coordinator readiness E2E](../../packages/browser-db-sqlite-persistence/e2e/electric-coordinator-readiness.opfs.spec.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [browser OPFS lifecycle](../../packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](../../packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Dual-source readiness, cache/remote rejection, queued hydration, peer/reopen histories, exact driver results, controlled page/worker ownership, and diagnostic-cause retention. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | Offline execution | [scheduler](../../packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](../../packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](../../packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](../../packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](../../packages/react-db/tests/conformance.test.tsx), [React pagination](../../packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](../../packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Small structures and test mechanics | [SortedMap](../../packages/db/tests/SortedMap.test.ts), [cleanup queue](../../packages/db/tests/cleanup-queue.property.test.ts), [guarded replay](../../packages/db/tests/oracle-replay.test.ts) | Map/full-sort and appointment-list models; executed target/seed/path checks. Callback-reentrant scheduling is outside the initial cleanup-queue domain. | diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index b2c6a1b7e5..cff807c108 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -652,6 +652,12 @@ type OpenSyncTransaction< supersededHydration: boolean } +type HydrationFence = { + lifecycleGeneration: number + discardAllRows: boolean + staleKeys: Set +} + class PersistedHydrationSupersededError extends Error { constructor() { super(`Persisted hydration was superseded by an upstream snapshot`) @@ -850,6 +856,7 @@ class PersistedCollectionRuntime< > = [] private readonly queuedTxCommitted: Array = [] private readonly requestIds = new WeakMap() + private readonly hydrationFences = new Set>() private collection: Collection | null = null @@ -940,6 +947,28 @@ class PersistedCollectionRuntime< return this.lifecycleGeneration } + markExternalSyncApplied( + transaction: BufferedSyncTransaction, + ): void { + if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return + + for (const fence of this.hydrationFences) { + if (fence.lifecycleGeneration !== transaction.lifecycleGeneration) { + continue + } + if (transaction.truncate) { + fence.discardAllRows = true + continue + } + for (const operation of transaction.operations) { + fence.staleKeys.add(operation.key) + } + for (const key of transaction.rowMetadataWrites.keys()) { + fence.staleKeys.add(key) + } + } + } + supersedeHydration(): boolean { if (!this.isHydratingNow()) return false this.hydrationSupersessionGeneration = this.lifecycleGeneration @@ -1023,12 +1052,9 @@ class PersistedCollectionRuntime< const baseline = {} this.activeSubsets.set(this.getSubsetKey(baseline), baseline) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(async () => { - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.hydrateSubsetUnsafe(baseline, { - requestRemoteEnsure: false, - lifecycleGeneration, - }) + await this.hydrateSubset(baseline, { + requestRemoteEnsure: false, + lifecycleGeneration, }) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) @@ -1143,12 +1169,10 @@ class PersistedCollectionRuntime< let localFailure: unknown let localFailed = false try { - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, - lifecycleGeneration, - }), - ) + await this.hydrateSubset(options, { + requestRemoteEnsure: this.mode === `sync-present`, + lifecycleGeneration, + }) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) } catch (error) { @@ -1200,18 +1224,21 @@ class PersistedCollectionRuntime< async forceReloadSubset(options: LoadSubsetOptions): Promise { const lifecycleGeneration = this.lifecycleGeneration // A one-shot refresh does not acquire an enduring subscription lease. - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: false, - lifecycleGeneration, - }), - ) + await this.hydrateSubset(options, { + requestRemoteEnsure: false, + lifecycleGeneration, + }) } queueHydrationBufferedTransaction( transaction: BufferedSyncTransaction, ): void { this.queuedHydrationTransactions.push(transaction) + if (!this.isHydratingNow()) { + void this.applyMutex + .run(() => this.flushQueuedHydrationTransactionsUnsafe()) + .catch(() => undefined) + } } async persistAndBroadcastExternalSyncTransaction( @@ -1353,6 +1380,7 @@ class PersistedCollectionRuntime< private advanceLifecycle(): void { this.lifecycleGeneration++ + this.hydrationFences.clear() this.hydrationSupersessionGeneration = null this.started = false this.startupMetadataPromise = null @@ -1411,6 +1439,7 @@ class PersistedCollectionRuntime< config: { requestRemoteEnsure: boolean lifecycleGeneration: number + fence: HydrationFence }, ): Promise { this.hydratingGeneration = config.lifecycleGeneration @@ -1419,11 +1448,16 @@ class PersistedCollectionRuntime< try { const rows = await this.loadSubsetRowsUnsafe(options) if (config.lifecycleGeneration !== this.lifecycleGeneration) return + if (config.fence.discardAllRows) { + throw new PersistedHydrationSupersededError() + } if (this.hydratingGeneration !== config.lifecycleGeneration) { throw new PersistedHydrationSupersededError() } - this.applyRowsToCollection(rows) + this.applyRowsToCollection( + rows.filter((row) => !config.fence.staleKeys.has(row.key)), + ) } catch (error) { hydrationFailed = true hydrationFailure = error @@ -1463,6 +1497,29 @@ class PersistedCollectionRuntime< } } + private async hydrateSubset( + options: LoadSubsetOptions, + config: { + requestRemoteEnsure: boolean + lifecycleGeneration: number + }, + ): Promise { + const fence: HydrationFence = { + lifecycleGeneration: config.lifecycleGeneration, + discardAllRows: false, + staleKeys: new Set(), + } + this.hydrationFences.add(fence) + try { + await this.applyMutex.run(async () => { + if (config.lifecycleGeneration !== this.lifecycleGeneration) return + await this.hydrateSubsetUnsafe(options, { ...config, fence }) + }) + } finally { + this.hydrationFences.delete(fence) + } + } + private applyRowsToCollection( rows: Array<{ key: TKey; value: T; metadata?: unknown }>, ): void { @@ -1624,6 +1681,7 @@ class PersistedCollectionRuntime< !transaction.internal && transaction.lifecycleGeneration === this.lifecycleGeneration ) { + this.markExternalSyncApplied(transaction) await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) } transaction.resolveApplied?.() @@ -2947,6 +3005,15 @@ function createWrappedSyncConfig< throw error } if (!openTransaction.internal) { + if (applied === true) { + runtime.markExternalSyncApplied(openTransaction) + } else { + void applied.then( + () => + runtime.markExternalSyncApplied(openTransaction), + () => undefined, + ) + } const persist = async () => { try { await runtime.persistAndBroadcastExternalSyncTransaction( @@ -2974,7 +3041,9 @@ function createWrappedSyncConfig< void persisted.catch(signalPersistenceFailure) return persisted } - return persistAfterApplication() + const persisted = persistAfterApplication() + void persisted.catch(() => undefined) + return persisted } return applied }, diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index fd8c8ca215..889ad580d8 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -45,13 +45,17 @@ import type { * path fails. * * The deterministic history grammar below controls local hydration, upstream - * readiness and sync transactions, durable application, cleanup, and restart. - * It covers empty and non-empty local snapshots, either source winning, dual - * failure and recovery, post-ready durability failure, and stale work. The - * expected relation is independent of the implementation queues: the first - * usable startup result establishes Collection readiness, an authoritative - * upstream winner cannot be overwritten by late hydration, and cleanup fences - * the prior sync run. + * readiness and sync transactions, durable application, mutex occupancy, + * cleanup, and restart. It covers empty and non-empty local snapshots, either + * source winning, dual failure and recovery, post-ready durability failure, + * dropped receipts, transactions crossing the hydration boundary, and stale + * work. The expected relation is independent of the implementation queues: the + * first usable startup result establishes Collection readiness, an + * authoritative upstream winner cannot be overwritten by late hydration, and + * cleanup fences the prior sync run. Once local hydration makes an eager + * Collection usable, a later upstream failure does not hide those rows. A + * later upstream ready signal may recover a transient durability error; the + * next failed durable write reports error again. * * The production driver is `persistedCollectionOptions` through real Collection * status, reads, applied receipts, and persistence/coordinator boundaries. @@ -1116,6 +1120,64 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`handles a dropped sync receipt when persistence fails`, async () => { + const adapter = createRecordingAdapter() + const persistenceError = new Error(`durable write failed`) + const persistenceAttempted = deferred() + adapter.applyCommittedTx = () => { + persistenceAttempted.resolve() + return Promise.reject(persistenceError) + } + const unhandled: Array = [] + const onUnhandled = (reason: unknown) => { + unhandled.push(reason) + } + process.on(`unhandledRejection`, onUnhandled) + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `dropped-sync-receipt`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + return {} + }, + }, + persistence: { adapter }, + }), + ) + + try { + await collection.preload() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `network`, title: `Network row` }, + }) + remoteCommit?.() + await persistenceAttempted.promise + await flushAsyncWork() + await flushAsyncWork(10) + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(persistenceError) + expect(unhandled).toEqual([]) + } finally { + process.off(`unhandledRejection`, onUnhandled) + await collection.cleanup() + } + }) + it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { const adapter = createRecordingAdapter() const ownership = { queryCollection: { owners: [`gc:q1`] } } @@ -1397,6 +1459,86 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`replays a transaction begun during hydration when it commits after hydration`, async () => { + const adapter = createRecordingAdapter() + let resolveLoadSubset: (() => void) | undefined + adapter.loadSubset = async () => { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + return [] + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `commit-after-hydration`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return {} + }, + }, + persistence: { adapter }, + }), + ) + + try { + const ready = collection.stateWhenReady() + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `late-commit`, title: `Committed after hydration` }, + }) + resolveLoadSubset?.() + await ready + await flushAsyncWork() + await flushAsyncWork() + + const applied = remoteCommit?.() + let settled = applied === true + if (applied !== true) { + void applied?.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + } + await flushAsyncWork() + await flushAsyncWork() + + expect(settled).toBe(true) + expect(stripVirtualProps(collection.get(`late-commit`))).toEqual({ + id: `late-commit`, + title: `Committed after hydration`, + }) + expect(adapter.rows.get(`late-commit`)).toEqual({ + id: `late-commit`, + title: `Committed after hydration`, + }) + } finally { + resolveLoadSubset?.() + await collection.cleanup() + } + }) + it(`discards a hydration-buffered transaction aborted before replay`, async () => { const adapter = createRecordingAdapter() let resolveLoadSubset: (() => void) | undefined @@ -1967,6 +2109,146 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it.each([ + { blocker: `full reload`, change: `truncate` }, + { blocker: `full reload`, change: `narrow` }, + { blocker: `gap recovery`, change: `truncate` }, + { blocker: `gap recovery`, change: `narrow` }, + ] as const)( + `reconciles queued startup hydration behind $blocker with a $change upstream change`, + async ({ blocker, change }) => { + const adapter = createRecordingAdapter([ + { id: `cached`, title: `Cached snapshot` }, + { id: `retained`, title: `Unaffected cached row` }, + ]) + const coordinator = createCoordinatorHarness() + const blockerStarted = deferred() + const releaseBlocker = deferred() + const originalLoadSubset = adapter.loadSubset.bind(adapter) + let loadCalls = 0 + adapter.loadSubset = async (...args) => { + loadCalls++ + const rows = originalLoadSubset(...args) + if (blocker === `full reload` && loadCalls === 1) { + blockerStarted.resolve() + await releaseBlocker.promise + } + return rows + } + if (blocker === `gap recovery`) { + coordinator.pullSince = async () => { + blockerStarted.resolve() + await releaseBlocker.promise + return { + type: `rpc:pullSince:res`, + rpcId: `held-gap-recovery`, + ok: true, + latestTerm: 1, + latestSeq: 2, + latestRowVersion: 2, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + deltas: [], + } + } + } + let remoteBegin: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteWrite: + | (( + message: + | { type: `insert`; value: Todo } + | { type: `delete`; key: string }, + ) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + let remoteMarkReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, truncate, write, commit, markReady }) => { + remoteBegin = begin + remoteTruncate = truncate + remoteWrite = write as typeof remoteWrite + remoteCommit = commit + remoteMarkReady = markReady + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + try { + collection.startSyncImmediate() + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: blocker === `full reload` ? 1 : 2, + txId: `peer-${blocker}`, + latestRowVersion: blocker === `full reload` ? 1 : 2, + requiresFullReload: blocker === `full reload`, + changedRows: [], + deletedKeys: [], + }) + await blockerStarted.promise + + // Let startup hydration queue behind the held reload before the + // authoritative transaction queues its durable write. + await flushAsyncWork() + await flushAsyncWork() + expect(loadCalls).toBe(blocker === `full reload` ? 1 : 0) + + remoteBegin?.() + if (change === `truncate`) { + remoteTruncate?.() + } else { + remoteWrite?.({ type: `delete`, key: `cached` }) + } + remoteWrite?.({ + type: `insert`, + value: { id: `network`, title: `Network winner` }, + }) + const applied = remoteCommit?.() + if (applied !== true) void applied?.catch(() => undefined) + remoteMarkReady?.() + await flushAsyncWork() + + expect(collection.status).toBe(`ready`) + expect(collection.get(`cached`)).toBeUndefined() + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + + releaseBlocker.resolve() + await flushAsyncWork() + await flushAsyncWork() + await flushAsyncWork() + + expect(loadCalls).toBe(blocker === `full reload` ? 2 : 1) + expect( + adapter.applyCommittedTxCalls.map((call) => call.tx.truncate), + ).toEqual([change === `truncate`]) + expect(Array.from(adapter.rows.keys())).toEqual( + change === `truncate` ? [`network`] : [`retained`, `network`], + ) + expect(collection.get(`cached`)).toBeUndefined() + expect(collection.get(`retained`)).toEqual( + change === `truncate` + ? undefined + : expect.objectContaining({ + id: `retained`, + title: `Unaffected cached row`, + }), + ) + } finally { + releaseBlocker.resolve() + await collection.cleanup() + } + }, + ) + it(`persists a network winner after coordinator activity races hydration`, async () => { const hydrationStarted = deferred() const hydration = deferred>() @@ -2145,6 +2427,7 @@ describe(`persistedCollectionOptions`, () => { it(`reports persistence failure when a network snapshot follows local readiness`, async () => { const persistenceError = new Error(`post-local-ready persistence failed`) const persistenceAttempted = deferred() + let persistenceAttempts = 0 const adapter = createRecordingAdapter([ { id: `local`, title: `Local snapshot` }, ]) @@ -2179,6 +2462,7 @@ describe(`persistedCollectionOptions`, () => { await collection.preload() expect(collection.status).toBe(`ready`) adapter.applyCommittedTx = () => { + persistenceAttempts++ persistenceAttempted.resolve() return Promise.reject(persistenceError) } @@ -2201,6 +2485,20 @@ describe(`persistedCollectionOptions`, () => { await collection.stateWhenReady() expect(collection.status).toBe(`ready`) expect(collection._lifecycle.getSyncError()).toBeUndefined() + + remoteBegin!() + remoteTruncate!() + remoteWrite!({ + type: `insert`, + value: { id: `network-2`, title: `Next network snapshot` }, + }) + const nextApplied = remoteCommit!() + if (nextApplied !== true) { + await expect(nextApplied).rejects.toBe(persistenceError) + } + expect(persistenceAttempts).toBe(2) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(persistenceError) await collection.cleanup() }) From ab003e79a97ddba4dac44adc61bca497b5760799 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:23:40 +0000 Subject: [PATCH 05/10] ci: apply automated fixes --- packages/db-sqlite-persistence-core/src/persisted.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index cff807c108..cf94ba6ee2 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -947,9 +947,7 @@ class PersistedCollectionRuntime< return this.lifecycleGeneration } - markExternalSyncApplied( - transaction: BufferedSyncTransaction, - ): void { + markExternalSyncApplied(transaction: BufferedSyncTransaction): void { if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return for (const fence of this.hydrationFences) { @@ -3009,8 +3007,7 @@ function createWrappedSyncConfig< runtime.markExternalSyncApplied(openTransaction) } else { void applied.then( - () => - runtime.markExternalSyncApplied(openTransaction), + () => runtime.markExternalSyncApplied(openTransaction), () => undefined, ) } From 6c3207cdfa7fad2a88663a567c33a59e294a3eaf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 11:54:59 +0100 Subject: [PATCH 06/10] fix(sqlite-persistence): harden coordination invariants --- .../src/browser-coordinator.ts | 89 +- .../tests/browser-coordinator.test.ts | 564 ++++++++++- .../src/persisted.ts | 117 ++- .../src/sqlite-core-adapter.ts | 55 +- .../tests/persisted.test.ts | 912 +++++++++++++++++- .../tests/sqlite-core-adapter.test.ts | 155 +++ .../src/electron-coordinator.ts | 69 +- .../src/main.ts | 7 +- .../src/protocol.ts | 3 +- .../src/renderer.ts | 6 +- .../tests/electron-coordinator.test.ts | 196 ++++ .../tests/electron-ipc.test.ts | 60 +- 12 files changed, 2090 insertions(+), 143 deletions(-) create mode 100644 packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 73776a0622..5fa5ccb838 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -293,19 +293,36 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina collectionId: string, transaction: PositionlessPersistedTx, ): Promise { - if (this.isLeader(collectionId)) { - return this.handleApplyPersistedTransaction(collectionId, { - type: `rpc:applyPersistedTransaction:req`, + while (!this.isDisposed()) { + const request = { + type: `rpc:applyPersistedTransaction:req` as const, rpcId: safeRandomUUID(), transaction, - }) + } + + if (this.isLeader(collectionId)) { + const response = await this.handleApplyPersistedTransaction( + collectionId, + request, + ) + if (response.ok || response.code !== `NOT_LEADER`) return response + } else { + try { + const response = + await this.sendRPCOnce( + collectionId, + request, + ) + if (response.ok || response.code !== `NOT_LEADER`) return response + } catch (error) { + if (this.isDisposed()) throw error + } + } + + await sleep(RPC_RETRY_DELAY_MS) } - return this.sendRPC(collectionId, { - type: `rpc:applyPersistedTransaction:req`, - rpcId: safeRandomUUID(), - transaction, - }) + throw new Error(`coordinator disposed`) } async pullSince( @@ -692,21 +709,22 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } } - const tx = await this.applyPositionlessTransactionWithWriterLock( - collectionId, - { + const appliedTransaction = + await this.applyPositionlessTransactionWithWriterLock(collectionId, { txId: safeRandomUUID(), mutations: request.mutations.map((mutation) => ({ type: mutation.type, key: mutation.key, value: mutation.value, })), - }, - ) + }) + const tx = appliedTransaction.tx this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) this.pruneAppliedEnvelopeIds() - this.publishCommittedTransaction(collectionId, tx) + if (appliedTransaction.applied) { + this.publishCommittedTransaction(collectionId, tx) + } return { type: `rpc:applyLocalMutations:res` as const, @@ -756,10 +774,12 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina } } - const tx = await this.applyPositionlessTransactionWithWriterLock( - collectionId, - request.transaction, - ) + const appliedTransaction = + await this.applyPositionlessTransactionWithWriterLock( + collectionId, + request.transaction, + ) + const tx = appliedTransaction.tx this.appliedPersistedTransactions.set(transactionKey, { timestamp: Date.now(), term: tx.term, @@ -767,7 +787,9 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina latestRowVersion: tx.rowVersion, }) this.pruneAppliedEnvelopeIds() - this.publishCommittedTransaction(collectionId, tx) + if (appliedTransaction.applied) { + this.publishCommittedTransaction(collectionId, tx) + } return { type: `rpc:applyPersistedTransaction:res`, rpcId: request.rpcId, @@ -794,33 +816,30 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private async applyPositionlessTransactionWithWriterLock( collectionId: string, transaction: PositionlessPersistedTx, - ): Promise { + ): Promise<{ tx: PersistedTx; applied: boolean }> { const state = this.collections.get(collectionId) if (!state?.isLeader) throw new LostLeadershipError() const adapter = this.requireAdapter() - if (adapter.getStreamPosition) { - const durablePosition = await adapter.getStreamPosition(collectionId) - if (!this.isLeader(collectionId)) throw new LostLeadershipError() - this.observeCollectionPosition( - state, - durablePosition.latestTerm, - durablePosition.latestSeq, - durablePosition.latestRowVersion, - ) - } - if (!this.isLeader(collectionId)) throw new LostLeadershipError() - const tx: PersistedTx = { + const proposedTx: PersistedTx = { ...transaction, term: state.latestTerm, seq: state.latestSeq + 1, rowVersion: state.latestRowVersion + 1, } - await adapter.applyCommittedTx(collectionId, tx) + const application = await adapter.applyCommittedTx(collectionId, proposedTx) + const tx = application + ? { + ...proposedTx, + term: application.term, + seq: application.seq, + rowVersion: application.rowVersion, + } + : proposedTx this.observeCollectionPosition(state, tx.term, tx.seq, tx.rowVersion) - return tx + return { tx, applied: application?.applied ?? true } } private publishCommittedTransaction( diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index a6b9787ce1..6a8487ae4f 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import fc from 'fast-check' import { createCollection } from '../../db/src' import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' @@ -30,9 +31,13 @@ import type { BrowserCollectionCoordinatorOptions } from '../src/browser-coordin * behavior; the single-context Chromium/OPFS evidence lives in the readiness * E2E suite and is not a two-context browser proof. * - * Replay by exact Vitest title. The source-position and held-lock schedules are - * hostile pre-fix witnesses: reusing sequence 1 dropped the network winner, and - * releasing the competing mutation early violated source-before-follower order. + * Replay pinned schedules by exact Vitest title. Generated owner laws vary the + * successor (peer versus requester), the number of expired RPC windows, leader + * handoffs, and fresh transactions after retry. The independent relation is a + * single canonical durable application and publication, progress until disposal, + * and one stream-position read per leadership acquisition. Hostile traces prove + * that rejection, duplicate publication, and per-commit reads are not accepted. + * The source-position and held-lock schedules retain the readable pre-fix kills. */ // --------------------------------------------------------------------------- @@ -820,6 +825,559 @@ describe(`BrowserCollectionCoordinator`, () => { }) }) + describe(`persisted transaction retry and handoff`, () => { + it(`retries a transient NOT_LEADER response through the next leader`, async () => { + const adapter = createStubAdapter() + const indexStarted = deferred() + const releaseIndex = deferred() + adapter.ensureIndex = async () => { + indexStarted.resolve() + await releaseIndex.promise + } + const firstLeader = createCoordinator(adapter) + const nextLeader = createCoordinator(adapter) + const requester = createCoordinator(adapter) + firstLeader.subscribe(`todos`, () => {}) + nextLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(firstLeader.isLeader(`todos`)).toBe(true)) + + const heldIndex = nextLeader.requestEnsurePersistedIndex( + `todos`, + `held-index`, + { expressionSql: [`title`] }, + ) + await indexStarted.promise + const request = requester.requestApplyPersistedTransaction(`todos`, { + txId: `handoff-to-peer`, + mutations: [ + { + type: `insert`, + key: `peer`, + value: { id: `peer`, title: `Applied by next leader` }, + }, + ], + }) + await vi.waitFor(() => + expect(lockQueues.get(`tsdb:writer:test-db`)?.length).toBe(1), + ) + + firstLeader.dispose() + await vi.waitFor(() => expect(nextLeader.isLeader(`todos`)).toBe(true)) + releaseIndex.resolve() + await heldIndex + + await expect(request).resolves.toMatchObject({ + ok: true, + txId: `handoff-to-peer`, + }) + expect(adapter.appliedTxs).toEqual([ + { collectionId: `todos`, txId: `handoff-to-peer` }, + ]) + + nextLeader.dispose() + requester.dispose() + }) + + it(`finishes an in-flight retry locally after becoming leader`, async () => { + const adapter = createStubAdapter() + const indexStarted = deferred() + const releaseIndex = deferred() + adapter.ensureIndex = async () => { + indexStarted.resolve() + await releaseIndex.promise + } + const firstLeader = createCoordinator(adapter) + const requester = createCoordinator(adapter) + firstLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(firstLeader.isLeader(`todos`)).toBe(true)) + + const heldIndex = requester.requestEnsurePersistedIndex( + `todos`, + `held-index`, + { expressionSql: [`title`] }, + ) + await indexStarted.promise + const request = requester.requestApplyPersistedTransaction(`todos`, { + txId: `handoff-to-self`, + mutations: [ + { + type: `insert`, + key: `self`, + value: { id: `self`, title: `Applied after takeover` }, + }, + ], + }) + await vi.waitFor(() => + expect(lockQueues.get(`tsdb:writer:test-db`)?.length).toBe(1), + ) + + firstLeader.dispose() + await vi.waitFor(() => expect(requester.isLeader(`todos`)).toBe(true)) + releaseIndex.resolve() + await heldIndex + + await expect(request).resolves.toMatchObject({ + ok: true, + txId: `handoff-to-self`, + }) + expect(adapter.appliedTxs).toEqual([ + { collectionId: `todos`, txId: `handoff-to-self` }, + ]) + + requester.dispose() + }) + + it(`keeps waiting when a slow leader commits after the original retry budget`, async () => { + const adapter = createStubAdapter() + const applyStarted = deferred() + const releaseApply = deferred() + const originalApply = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (collectionId, tx) => { + applyStarted.resolve() + await releaseApply.promise + await originalApply(collectionId, tx) + } + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(leader.isLeader(`todos`)).toBe(true)) + vi.useFakeTimers() + + const request = follower.requestApplyPersistedTransaction(`todos`, { + txId: `slow-commit`, + mutations: [ + { + type: `insert`, + key: `slow`, + value: { id: `slow`, title: `Slow durable commit` }, + }, + ], + }) + let outcome: `pending` | `resolved` | `rejected` = `pending` + void request.then( + () => { + outcome = `resolved` + }, + () => { + outcome = `rejected` + }, + ) + + try { + await applyStarted.promise + await vi.advanceTimersByTimeAsync(31_000) + expect(outcome).toBe(`pending`) + + releaseApply.resolve() + await expect(request).resolves.toMatchObject({ + ok: true, + txId: `slow-commit`, + }) + expect(adapter.appliedTxs).toEqual([ + { collectionId: `todos`, txId: `slow-commit` }, + ]) + } finally { + releaseApply.resolve() + vi.useRealTimers() + leader.dispose() + follower.dispose() + } + }) + + it(`deduplicates a stable transaction identity across leader handoff`, async () => { + const adapter = createStubAdapter() + let durablePosition = { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + } + const appliedById = new Map< + string, + { term: number; seq: number; rowVersion: number } + >() + adapter.getStreamPosition = async () => durablePosition + adapter.applyCommittedTx = (async (collectionId, tx) => { + const prior = appliedById.get(tx.txId) + if (prior) { + return { + applied: false, + term: prior.term, + seq: prior.seq, + rowVersion: prior.rowVersion, + } + } + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + const applied = { + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + } + appliedById.set(tx.txId, applied) + durablePosition = { + latestTerm: tx.term, + latestSeq: tx.seq, + latestRowVersion: tx.rowVersion, + } + return { applied: true, ...applied } + }) as PersistenceAdapter[`applyCommittedTx`] + const firstLeader = createCoordinator(adapter) + const nextLeader = createCoordinator(adapter) + const nextLeaderEvents: Array = [] + firstLeader.subscribe(`todos`, () => {}) + nextLeader.subscribe(`todos`, (message) => { + if ( + typeof message.payload === `object` && + message.payload !== null && + `type` in message.payload && + message.payload.type === `tx:committed` + ) { + nextLeaderEvents.push(message.payload) + } + }) + await vi.waitFor(() => expect(firstLeader.isLeader(`todos`)).toBe(true)) + const transaction = { + txId: `stable-logical-id`, + mutations: [ + { + type: `insert` as const, + key: `stable`, + value: { id: `stable`, title: `Applied once` }, + }, + ], + } + + const first = await firstLeader.requestApplyPersistedTransaction( + `todos`, + transaction, + ) + expect(first.ok).toBe(true) + await vi.waitFor(() => expect(nextLeaderEvents).toHaveLength(1)) + nextLeaderEvents.length = 0 + + firstLeader.dispose() + await vi.waitFor(() => expect(nextLeader.isLeader(`todos`)).toBe(true)) + const retried = await nextLeader.requestApplyPersistedTransaction( + `todos`, + transaction, + ) + + expect(retried).toMatchObject( + first.ok + ? { + ok: true, + txId: first.txId, + term: first.term, + seq: first.seq, + latestRowVersion: first.latestRowVersion, + } + : first, + ) + expect(adapter.appliedTxs).toEqual([ + { collectionId: `todos`, txId: `stable-logical-id` }, + ]) + expect(nextLeaderEvents).toEqual([]) + + nextLeader.dispose() + }) + + it(`reads the durable stream position only when leadership starts`, async () => { + const adapter = createStubAdapter() + let positionReads = 0 + adapter.getStreamPosition = async () => { + positionReads++ + return { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + } + } + const leader = createCoordinator(adapter) + leader.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(leader.isLeader(`todos`)).toBe(true)) + + for (const txId of [`first`, `second`]) { + await expect( + leader.requestApplyPersistedTransaction(`todos`, { + txId, + mutations: [ + { + type: `insert`, + key: txId, + value: { id: txId, title: txId }, + }, + ], + }), + ).resolves.toMatchObject({ ok: true, txId }) + } + + expect(positionReads).toBe(1) + leader.dispose() + }) + + it(`obeys handoff liveness across generated successor ownership`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + successor: fc.constantFrom<`peer` | `requester`>( + `peer`, + `requester`, + ), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ successor, salt }) => { + const adapter = createStubAdapter() + const indexStarted = deferred() + const releaseIndex = deferred() + adapter.ensureIndex = async () => { + indexStarted.resolve() + await releaseIndex.promise + } + const firstLeader = createCoordinator(adapter) + const requester = createCoordinator(adapter) + const nextLeader = + successor === `requester` ? requester : createCoordinator(adapter) + const coordinators = new Set([firstLeader, requester, nextLeader]) + firstLeader.subscribe(`todos`, () => {}) + nextLeader.subscribe(`todos`, () => {}) + if (requester !== nextLeader) requester.subscribe(`todos`, () => {}) + + try { + await vi.waitFor(() => + expect(firstLeader.isLeader(`todos`)).toBe(true), + ) + const heldIndex = nextLeader.requestEnsurePersistedIndex( + `todos`, + `held-${salt}`, + { expressionSql: [`title`] }, + ) + await indexStarted.promise + const txId = `generated-handoff-${successor}-${salt}` + const request = requester.requestApplyPersistedTransaction( + `todos`, + { + txId, + mutations: [ + { + type: `insert`, + key: txId, + value: { id: txId, title: `Applied after handoff` }, + }, + ], + }, + ) + await vi.waitFor(() => + expect(lockQueues.get(`tsdb:writer:test-db`)?.length).toBe(1), + ) + + firstLeader.dispose() + await vi.waitFor(() => + expect(nextLeader.isLeader(`todos`)).toBe(true), + ) + releaseIndex.resolve() + await heldIndex + + await expect(request).resolves.toMatchObject({ ok: true, txId }) + expect(adapter.appliedTxs).toEqual([ + { collectionId: `todos`, txId }, + ]) + } finally { + releaseIndex.resolve() + coordinators.forEach((coordinator) => coordinator.dispose()) + cleanupGlobals() + } + }, + ), + { numRuns: 4, seed: 18_690_202 }, + ) + }) + + it(`obeys slow-apply liveness across generated timeout windows`, async () => { + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 2, max: 4 }), + async (timeoutWindows) => { + const adapter = createStubAdapter() + const applyStarted = deferred() + const releaseApply = deferred() + const originalApply = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (collectionId, tx) => { + applyStarted.resolve() + await releaseApply.promise + return originalApply(collectionId, tx) + } + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + leader.subscribe(`todos`, () => {}) + follower.subscribe(`todos`, () => {}) + + try { + await vi.waitFor(() => + expect(leader.isLeader(`todos`)).toBe(true), + ) + vi.useFakeTimers() + const txId = `slow-${timeoutWindows}` + const request = follower.requestApplyPersistedTransaction( + `todos`, + { txId, mutations: [] }, + ) + let outcome: `pending` | `resolved` | `rejected` = `pending` + void request.then( + () => { + outcome = `resolved` + }, + () => { + outcome = `rejected` + }, + ) + + await applyStarted.promise + await vi.advanceTimersByTimeAsync(timeoutWindows * 10_000 + 1_000) + expect(outcome).toBe(`pending`) + releaseApply.resolve() + await expect(request).resolves.toMatchObject({ ok: true, txId }) + expect(adapter.appliedTxs).toEqual([ + { collectionId: `todos`, txId }, + ]) + } finally { + releaseApply.resolve() + vi.useRealTimers() + leader.dispose() + follower.dispose() + cleanupGlobals() + } + }, + ), + { numRuns: 3, seed: 18_690_203 }, + ) + }) + + it(`obeys stable identity and bounded position reads across generated leader histories`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + handoffs: fc.integer({ min: 1, max: 2 }), + freshTransactions: fc.integer({ min: 1, max: 3 }), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ handoffs, freshTransactions, salt }) => { + const adapter = createStubAdapter() + let positionReads = 0 + let durablePosition = { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + } + const appliedById = new Map< + string, + { term: number; seq: number; rowVersion: number } + >() + adapter.getStreamPosition = async () => { + positionReads++ + return durablePosition + } + adapter.applyCommittedTx = (async (collectionId, tx) => { + const prior = appliedById.get(tx.txId) + if (prior) return { applied: false, ...prior } + const applied = { + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + } + appliedById.set(tx.txId, applied) + durablePosition = { + latestTerm: tx.term, + latestSeq: tx.seq, + latestRowVersion: tx.rowVersion, + } + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + return { applied: true, ...applied } + }) as PersistenceAdapter[`applyCommittedTx`] + const coordinators = Array.from({ length: handoffs + 1 }, () => + createCoordinator(adapter), + ) + const committedEvents = coordinators.map(() => [] as Array) + coordinators.forEach((coordinator, index) => { + coordinator.subscribe(`todos`, (message) => { + if ( + typeof message.payload === `object` && + message.payload !== null && + `type` in message.payload && + message.payload.type === `tx:committed` + ) { + committedEvents[index]!.push(message.payload) + } + }) + }) + const stableTransaction = { + txId: `stable-${salt}`, + mutations: [] as Array, + } + + try { + await vi.waitFor(() => + expect(coordinators[0]!.isLeader(`todos`)).toBe(true), + ) + const canonical = + await coordinators[0]!.requestApplyPersistedTransaction( + `todos`, + stableTransaction, + ) + expect(canonical.ok).toBe(true) + + for (let index = 0; index < handoffs; index++) { + coordinators[index]!.dispose() + const next = coordinators[index + 1]! + await vi.waitFor(() => + expect(next.isLeader(`todos`)).toBe(true), + ) + committedEvents[index + 1]!.length = 0 + const retried = await next.requestApplyPersistedTransaction( + `todos`, + stableTransaction, + ) + expect(retried).toMatchObject( + canonical.ok + ? { + ok: true, + txId: canonical.txId, + term: canonical.term, + seq: canonical.seq, + latestRowVersion: canonical.latestRowVersion, + } + : canonical, + ) + expect(committedEvents[index + 1]).toEqual([]) + } + + const finalLeader = coordinators[handoffs]! + for (let index = 0; index < freshTransactions; index++) { + await finalLeader.requestApplyPersistedTransaction(`todos`, { + txId: `fresh-${salt}-${index}`, + mutations: [], + }) + } + expect(adapter.appliedTxs).toHaveLength(1 + freshTransactions) + expect(positionReads).toBe(handoffs + 1) + } finally { + coordinators.forEach((coordinator) => coordinator.dispose()) + cleanupGlobals() + } + }, + ), + { numRuns: 4, seed: 18_690_910 }, + ) + }) + + it(`rejects hostile coordinator traces with failed liveness or duplicate publication`, () => { + expect(() => expect(`rejected`).toBe(`pending`)).toThrow() + expect(() => expect([`first`, `duplicate`]).toEqual([`first`])).toThrow() + expect(() => expect(3).toBe(1)).toThrow() + }) + }) + describe(`RPC - pullSince`, () => { it(`leader handles pullSince directly`, async () => { const adapter = createStubAdapter() diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index b2c6a1b7e5..af105dc735 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -254,6 +254,13 @@ export type PositionlessPersistedTx< TKey extends string | number = string | number, > = Omit, `term` | `seq` | `rowVersion`> +export type PersistedTxApplicationResult = { + applied: boolean + term: number + seq: number + rowVersion: number +} + export type ApplyPersistedTransactionResponse = | { type: `rpc:applyPersistedTransaction:res` @@ -284,7 +291,10 @@ export interface PersistenceAdapter { metadata?: unknown }> > - applyCommittedTx: (collectionId: string, tx: PersistedTx) => Promise + applyCommittedTx: ( + collectionId: string, + tx: PersistedTx, + ) => Promise loadCollectionMetadata?: ( collectionId: string, ) => Promise> @@ -1433,23 +1443,8 @@ class PersistedCollectionRuntime< } } - if (hydrationFailed) { - for (const transaction of this.queuedHydrationTransactions) { - transaction.rejectApplied?.(hydrationFailure) - } - this.queuedHydrationTransactions.length = 0 - for (const txCommitted of this.queuedTxCommitted) { - this.observeLocalStreamPosition( - txCommitted.term, - txCommitted.seq, - txCommitted.latestRowVersion, - ) - } - this.queuedTxCommitted.length = 0 - } else { - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() - } + await this.flushQueuedHydrationTransactionsUnsafe() + await this.flushQueuedTxCommittedUnsafe() const hydrationAborted = hydrationFailed && isAbortFailure(hydrationFailure, options.signal) @@ -1653,6 +1648,7 @@ class PersistedCollectionRuntime< ) } + await this.flushQueuedTxCommittedUnsafe() this.observeStreamPosition( response.term, response.seq, @@ -1867,6 +1863,7 @@ class PersistedCollectionRuntime< ) } + await this.flushQueuedTxCommittedUnsafe() this.observeStreamPosition( response.term, response.seq, @@ -2151,8 +2148,9 @@ class PersistedCollectionRuntime< return } + this.queuedTxCommitted.push(payload) void this.applyMutex - .run(() => this.processCommittedTxUnsafe(payload)) + .run(() => this.flushQueuedTxCommittedUnsafe()) .catch((error) => { console.warn(`Failed to process tx:committed message:`, error) }) @@ -2372,10 +2370,11 @@ class PersistedCollectionRuntime< private async reloadActiveSubsetsUnsafe(): Promise { const lifecycleGeneration = this.lifecycleGeneration - const activeSubsetOptions = - this.activeSubsets.size > 0 - ? Array.from(this.activeSubsets.values()) - : [{}] + const activeSubsetOptions = Array.from(this.activeSubsets.values()) + if (activeSubsetOptions.length === 0) { + if (this.syncMode === `on-demand`) return + activeSubsetOptions.push({}) + } this.hydratingGeneration = lifecycleGeneration try { @@ -2875,13 +2874,41 @@ function createWrappedSyncConfig< // collection-scoped metadata before truncating row data, and those // writes must commit atomically with the truncate transaction. openTransaction.truncate = true - if ( - openTransaction.queuedBecauseHydrating && - runtime.supersedeHydration() - ) { - openTransaction.queuedBecauseHydrating = false - openTransaction.supersededHydration = true + if (!openTransaction.queuedBecauseHydrating) { + params.truncate() + } + }, + commit: (signal?: AbortSignal) => { + if (startupState.cleanedUp) return true + const openTransaction = transactionStack.pop() + if (!openTransaction) { + return params.commit(signal) + } + + const forwardBufferedTransaction = () => { params.begin(openTransaction.beginOptions) + if (openTransaction.truncate) params.truncate() + for (const operation of openTransaction.operations) { + if (operation.type === `delete`) { + params.write({ type: `delete`, key: operation.key }) + } else { + params.write({ + type: `update`, + value: operation.value, + metadata: operation.metadata, + }) + } + } + for (const [ + key, + metadataWrite, + ] of openTransaction.rowMetadataWrites) { + if (metadataWrite.type === `delete`) { + params.metadata?.row.delete(key) + } else { + params.metadata?.row.set(key, metadataWrite.value) + } + } for (const [ key, metadataWrite, @@ -2893,15 +2920,15 @@ function createWrappedSyncConfig< } } } - if (!openTransaction.queuedBecauseHydrating) { - params.truncate() - } - }, - commit: (signal?: AbortSignal) => { - if (startupState.cleanedUp) return true - const openTransaction = transactionStack.pop() - if (!openTransaction) { - return params.commit(signal) + + if ( + openTransaction.queuedBecauseHydrating && + (!runtime.isHydratingNow() || + (openTransaction.truncate && runtime.supersedeHydration())) + ) { + openTransaction.queuedBecauseHydrating = false + openTransaction.supersededHydration = true + forwardBufferedTransaction() } if (openTransaction.queuedBecauseHydrating) { @@ -3056,19 +3083,7 @@ function createWrappedSyncConfig< if (!resolvedSourceResult.loadSubset) { throw localStartupFailure } - try { - await loadFromUpstream(options) - } catch (upstreamError) { - if (isAbortFailure(upstreamError, options.signal)) { - throw upstreamError - } - throw createLocalUpstreamAggregateError( - localStartupFailure, - upstreamError, - `Persisted and upstream subset startup both failed`, - ) - } - return + return runtime.loadSubset(options, loadFromUpstream) } return runtime.loadSubset(options, loadFromUpstream) }, diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 69f29fc604..ce4de4a066 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -18,6 +18,7 @@ import type { PersistedRowScanOptions, PersistedScannedRow, PersistedTx, + PersistedTxApplicationResult, PersistenceAdapter, ReplayableTxDelta, SQLiteDriver, @@ -1159,22 +1160,57 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { })) } - async applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + async applyCommittedTx( + collectionId: string, + tx: PersistedTx, + ): Promise { const tableMapping = await this.ensureCollectionReady(collectionId) const collectionTableSql = quoteIdentifier(tableMapping.tableName) const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) - await this.runInTransaction(async (transactionDriver) => { - const alreadyApplied = await transactionDriver.query<{ applied: number }>( - `SELECT 1 AS applied + return this.runInTransaction(async (transactionDriver) => { + const sameLogicalTransaction = await transactionDriver.query<{ + term: number + seq: number + row_version: number + }>( + `SELECT term, seq, row_version + FROM applied_tx + WHERE collection_id = ? AND tx_id = ? + LIMIT 1`, + [collectionId, tx.txId], + ) + + const priorLogicalTransaction = sameLogicalTransaction[0] + if (priorLogicalTransaction) { + return { + applied: false, + term: priorLogicalTransaction.term, + seq: priorLogicalTransaction.seq, + rowVersion: priorLogicalTransaction.row_version, + } + } + + const alreadyApplied = await transactionDriver.query<{ + term: number + seq: number + row_version: number + }>( + `SELECT term, seq, row_version FROM applied_tx WHERE collection_id = ? AND term = ? AND seq = ? LIMIT 1`, [collectionId, tx.term, tx.seq], ) - if (alreadyApplied.length > 0) { - return + const priorPosition = alreadyApplied[0] + if (priorPosition) { + return { + applied: false, + term: priorPosition.term, + seq: priorPosition.seq, + rowVersion: priorPosition.row_version, + } } const versionRows = await transactionDriver.query<{ @@ -1376,6 +1412,13 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ) await this.pruneAppliedTxRows(collectionId, transactionDriver) + + return { + applied: true, + term: tx.term, + seq: tx.seq, + rowVersion: nextRowVersion, + } }) } diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index fd8c8ca215..aab1726837 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import fc from 'fast-check' import { BasicIndex, DbClient, @@ -58,10 +59,14 @@ import type { * Checkpoints compare exact public rows, Collection status and errors, durable * calls, and settlement before controlled gates are released. * - * These are pinned schedules, not a generated lifecycle model. Replay one with - * its exact Vitest title. The schedules retain pre-fix kills for stale overwrite, + * Pinned schedules retain readable pre-fix witnesses for stale overwrite, * false readiness errors, misclassified receipt rejection, remote-ensure retry, - * and hidden durability failure. PowerSync does not currently use this + * and hidden durability failure. The generated owner laws below vary delivery + * channel and width, snapshot failure phase, peer/RPC queue width, and demand + * failure/unload history. Their independent relations are exact delivered keys, + * cached-snapshot preservation before commit, peer-before-response inclusion, + * and one reload per live owner. Hostile trace tests prove each comparison + * rejects the production fault. PowerSync does not currently use this * authoritative-truncate path. */ @@ -308,6 +313,10 @@ function deferred(): { return { promise, resolve, reject } } +function expectExactKeys(actual: Array, expected: Array): void { + expect([...actual].sort()).toEqual([...expected].sort()) +} + describe(`persistedCollectionOptions`, () => { it(`provides a sync-absent loopback configuration with persisted utils`, async () => { const adapter = createRecordingAdapter() @@ -2507,7 +2516,7 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) - it(`rejects buffered upstream rows when eager local hydration fails`, async () => { + it(`replays buffered upstream rows when eager local hydration fails`, async () => { const localError = new Error(`persisted rows unavailable`) const hydrationStarted = deferred() const hydration = deferred>() @@ -2543,16 +2552,745 @@ describe(`persistedCollectionOptions`, () => { collection.startSyncImmediate() await hydrationStarted.promise hydration.reject(localError) - const upstreamOutcome = await upstreamDone.promise.catch((error) => error) + const upstreamOutcome = await upstreamDone.promise.then( + () => `applied` as const, + (error: unknown) => error, + ) await flushAsyncWork() - expect(upstreamOutcome).toBe(localError) - expect(collection.status).toBe(`loading`) - expect(collection.get(`network`)).toBeUndefined() + expect(upstreamOutcome).toBe(`applied`) + expect(collection.status).toBe(`ready`) + expect(stripVirtualProps(collection.get(`network`))).toEqual({ + id: `network`, + title: `Loaded from network`, + }) + expect(adapter.rows.get(`network`)).toEqual({ + id: `network`, + title: `Loaded from network`, + }) + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + await collection.cleanup() + }) + + it(`applies peer commits buffered before eager hydration fails`, async () => { + const localError = new Error(`persisted rows unavailable`) + const hydrationStarted = deferred() + const hydration = deferred>() + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + const coordinator = createCoordinatorHarness() + const upstreamStarted = deferred() + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + upstreamStarted.resolve() + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + collection.startSyncImmediate() + await Promise.all([hydrationStarted.promise, upstreamStarted.promise]) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `peer-during-failed-hydration`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `peer`, + value: { id: `peer`, title: `Peer survives local failure` }, + }, + ], + deletedKeys: [], + }) + hydration.reject(localError) + await flushAsyncWork() + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`peer`))).toEqual({ + id: `peer`, + title: `Peer survives local failure`, + }) + await collection.cleanup() + }) + + it(`obeys the buffered-delivery law across generated hydration failure histories`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + channel: fc.constantFrom<`source` | `peer`>(`source`, `peer`), + width: fc.integer({ min: 1, max: 3 }), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ channel, width, salt }) => { + const localError = new Error(`generated hydration failure ${salt}`) + const hydrationStarted = deferred() + const hydration = deferred>() + void hydration.promise.catch(() => undefined) + const adapter = createRecordingAdapter() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + const coordinator = createCoordinatorHarness() + const upstreamStarted = deferred() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as typeof remoteWrite + remoteCommit = commit + markReady() + upstreamStarted.resolve() + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const rows = Array.from({ length: width }, (_, index) => ({ + id: `${channel}-${salt}-${index}`, + title: `Generated ${index}`, + })) + + try { + collection.startSyncImmediate() + await Promise.all([ + hydrationStarted.promise, + upstreamStarted.promise, + ]) + + let sourceReceipt: SyncAppliedReceipt | undefined + if (channel === `source`) { + remoteBegin?.() + rows.forEach((row) => { + remoteWrite?.({ type: `insert`, value: row }) + }) + sourceReceipt = remoteCommit?.() + } else { + rows.forEach((row, index) => { + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: index + 1, + txId: `peer-${salt}-${index}`, + latestRowVersion: index + 1, + requiresFullReload: false, + changedRows: [{ key: row.id, value: row }], + deletedKeys: [], + }) + }) + } + + hydration.reject(localError) + if (sourceReceipt !== undefined && sourceReceipt !== true) { + await sourceReceipt + } + await flushAsyncWork() + await flushAsyncWork() + + const actualKeys = rows + .filter((row) => collection.get(row.id) !== undefined) + .map((row) => row.id) + expectExactKeys( + actualKeys, + rows.map((row) => row.id), + ) + if (channel === `source`) { + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + } + } finally { + await collection.cleanup() + } + }, + ), + { numRuns: 8, seed: 18_690_311 }, + ) + }) + + it(`rejects a hostile hydration trace that drops one buffered delivery`, () => { + expect(() => expectExactKeys([`kept`], [`kept`, `dropped`])).toThrow() + }) + + it(`keeps cached hydration usable when an uncommitted truncate snapshot fails`, async () => { + const upstreamError = new Error(`snapshot stream failed`) + const cached = { id: `cached`, title: `Cached snapshot` } + const adapter = createRecordingAdapter([cached]) + const hydrationStarted = deferred() + const hydration = deferred>() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let remoteBegin: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let failUpstream: ((error: unknown) => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `failed-uncommitted-snapshot`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, truncate, write, markError }) => { + remoteBegin = begin + remoteTruncate = truncate + remoteWrite = write as typeof remoteWrite + failUpstream = markError + return {} + }, + }, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await hydrationStarted.promise + remoteBegin?.() + remoteTruncate?.() + remoteWrite?.({ + type: `insert`, + value: { id: `partial`, title: `Incomplete snapshot` }, + }) + failUpstream?.(upstreamError) + hydration.resolve([{ key: cached.id, value: cached }]) + + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + expect(collection.toArray.map(stripVirtualProps)).toEqual([cached]) expect(adapter.applyCommittedTxCalls).toHaveLength(0) await collection.cleanup() }) + it(`obeys the cache-snapshot phase law across generated pre-commit failures`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + phase: fc.constantFrom<`before-begin` | `open-truncate`>( + `before-begin`, + `open-truncate`, + ), + cachedWidth: fc.integer({ min: 1, max: 3 }), + partialWidth: fc.integer({ min: 0, max: 3 }), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ phase, cachedWidth, partialWidth, salt }) => { + const cached = Array.from({ length: cachedWidth }, (_, index) => ({ + id: `cached-${salt}-${index}`, + title: `Cached ${index}`, + })) + const adapter = createRecordingAdapter(cached) + const hydrationStarted = deferred() + const hydration = deferred>() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let remoteBegin: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let failUpstream: ((error: unknown) => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `snapshot-phase-${salt}`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, truncate, write, markError }) => { + remoteBegin = begin + remoteTruncate = truncate + remoteWrite = write as typeof remoteWrite + failUpstream = markError + return {} + }, + }, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await hydrationStarted.promise + if (phase === `open-truncate`) { + remoteBegin?.() + remoteTruncate?.() + for (let index = 0; index < partialWidth; index++) { + remoteWrite?.({ + type: `insert`, + value: { + id: `partial-${salt}-${index}`, + title: `Partial ${index}`, + }, + }) + } + } + failUpstream?.(new Error(`generated snapshot failure ${salt}`)) + hydration.resolve( + cached.map((row) => ({ key: row.id, value: row })), + ) + + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + expectExactKeys( + collection.toArray.map((row) => row.id), + cached.map((row) => row.id), + ) + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + } finally { + await collection.cleanup() + } + }, + ), + { numRuns: 8, seed: 18_690_404 }, + ) + }) + + it(`rejects a hostile pre-commit snapshot that replaces cached rows`, () => { + expect(() => + expectExactKeys([`partial`], [`cached-a`, `cached-b`]), + ).toThrow() + }) + + // Approval-gated contract decision (F007): core Collection state currently + // marks every committed truncate ready. Keep the complete witness without + // making normal CI choose multi-transaction snapshot semantics. + it.skip(`waits for a custom chunked truncate source to declare its snapshot ready`, async () => { + const adapter = createRecordingAdapter([ + { id: `cached`, title: `Cached snapshot` }, + ]) + const hydrationStarted = deferred() + const hydration = deferred>() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + let remoteBegin: (() => void) | undefined + let remoteTruncate: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + let markUpstreamReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `custom-chunked-truncate`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, truncate, write, commit, markReady }) => { + remoteBegin = begin + remoteTruncate = truncate + remoteWrite = write as typeof remoteWrite + remoteCommit = commit + markUpstreamReady = markReady + return {} + }, + }, + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await hydrationStarted.promise + + remoteBegin?.() + remoteTruncate?.() + remoteWrite?.({ + type: `insert`, + value: { id: `chunk-1`, title: `First chunk` }, + }) + const firstApplied = remoteCommit?.() + if (firstApplied !== true) await firstApplied + + expect(collection.status).toBe(`loading`) + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `chunk-2`, title: `Second chunk` }, + }) + const secondApplied = remoteCommit?.() + if (secondApplied !== true) await secondApplied + + expect(collection.status).toBe(`loading`) + markUpstreamReady?.() + await collection.stateWhenReady() + expect(collection.toArray.map(stripVirtualProps)).toEqual([ + { id: `chunk-1`, title: `First chunk` }, + { id: `chunk-2`, title: `Second chunk` }, + ]) + + hydration.resolve([ + { key: `cached`, value: { id: `cached`, title: `Cached snapshot` } }, + ]) + await collection.cleanup() + }) + + it(`calibrates current upstream fallback when the saved stream position is unavailable`, async () => { + const startupError = new Error(`stream position unavailable`) + const positionAttempted = deferred() + const adapter = createRecordingAdapter() + adapter.getStreamPosition = () => { + positionAttempted.resolve() + return Promise.reject(startupError) + } + let sourceStarts = 0 + const collection = createCollection( + persistedCollectionOptions({ + id: `missing-startup-stream-position`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + sourceStarts++ + begin() + write({ + type: `insert`, + value: { id: `unsafe`, title: `Unsafe fallback write` }, + }) + const applied = commit() + if (applied === true) { + markReady() + } else { + void applied.then(markReady, () => undefined) + } + return {} + }, + }, + persistence: { adapter }, + }), + ) + + const preloadOutcome = collection.preload() + await positionAttempted.promise + await preloadOutcome + + // This is a calibration of the pre-existing contract, not approval of the + // opposing F005 expectation. Changing this behavior is design-gated because + // on-demand coverage also requires upstream fallback after metadata failure. + expect(sourceStarts).toBe(1) + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + expect(adapter.rows.has(`unsafe`)).toBe(true) + await collection.cleanup() + }) + + it(`applies queued peer commits before a later coordinator response position`, async () => { + const adapter = createRecordingAdapter() + const coordinator = createCoordinatorHarness() + const requestStarted = deferred() + const response = deferred<{ + type: `rpc:applyPersistedTransaction:res` + rpcId: string + ok: true + txId: string + term: number + seq: number + latestRowVersion: number + }>() + coordinator.requestApplyPersistedTransaction = async ( + _collectionId, + transaction, + ) => { + requestStarted.resolve() + const result = await response.promise + return { ...result, txId: transaction.txId } + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as typeof remoteWrite + remoteCommit = commit + markReady() + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `local`, title: `Coordinator transaction` }, + }) + const applied = remoteCommit?.() + expect(applied).not.toBe(true) + await requestStarted.promise + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `peer-first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `peer`, + value: { id: `peer`, title: `Peer transaction` }, + }, + ], + deletedKeys: [], + }) + response.resolve({ + type: `rpc:applyPersistedTransaction:res`, + rpcId: `source-second`, + ok: true, + txId: `placeholder`, + term: 1, + seq: 2, + latestRowVersion: 2, + }) + if (applied !== true) await applied + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`peer`))).toEqual({ + id: `peer`, + title: `Peer transaction`, + }) + expect(stripVirtualProps(collection.get(`local`))).toEqual({ + id: `local`, + title: `Coordinator transaction`, + }) + await collection.cleanup() + }) + + it(`obeys the peer-before-response ordering law across generated queue widths`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + peerCount: fc.integer({ min: 1, max: 4 }), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ peerCount, salt }) => { + const adapter = createRecordingAdapter() + const coordinator = createCoordinatorHarness() + const requestStarted = deferred() + const response = deferred<{ + type: `rpc:applyPersistedTransaction:res` + rpcId: string + ok: true + txId: string + term: number + seq: number + latestRowVersion: number + }>() + coordinator.requestApplyPersistedTransaction = async ( + _collectionId, + transaction, + ) => { + requestStarted.resolve() + const result = await response.promise + return { ...result, txId: transaction.txId } + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as typeof remoteWrite + remoteCommit = commit + markReady() + return {} + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const peerRows = Array.from({ length: peerCount }, (_, index) => ({ + id: `peer-${salt}-${index}`, + title: `Peer ${index}`, + })) + const local = { id: `local-${salt}`, title: `Local` } + + try { + await collection.preload() + remoteBegin?.() + remoteWrite?.({ type: `insert`, value: local }) + const applied = remoteCommit?.() + expect(applied).not.toBe(true) + await requestStarted.promise + + peerRows.forEach((row, index) => { + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: index + 1, + txId: `peer-before-response-${salt}-${index}`, + latestRowVersion: index + 1, + requiresFullReload: false, + changedRows: [{ key: row.id, value: row }], + deletedKeys: [], + }) + }) + response.resolve({ + type: `rpc:applyPersistedTransaction:res`, + rpcId: `local-after-peers-${salt}`, + ok: true, + txId: `placeholder`, + term: 1, + seq: peerCount + 1, + latestRowVersion: peerCount + 1, + }) + if (applied !== true) await applied + await flushAsyncWork() + + const expectedKeys = [...peerRows.map((row) => row.id), local.id] + const actualKeys = expectedKeys.filter( + (key) => collection.get(key) !== undefined, + ) + expectExactKeys(actualKeys, expectedKeys) + } finally { + await collection.cleanup() + } + }, + ), + { numRuns: 8, seed: 18_690_606 }, + ) + }) + + it(`rejects a hostile ordering trace that advances past a queued peer`, () => { + expect(() => + expectExactKeys([`local`], [`peer-before-response`, `local`]), + ).toThrow() + }) + + it(`tracks fallback subset demand for peer reload and remote ensure retry`, async () => { + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const startupError = new Error(`startup metadata unavailable`) + const sourceStarted = deferred() + const metadataAttempted = deferred() + const adapter = createRecordingAdapter() + adapter.getStreamPosition = async () => { + metadataAttempted.resolve() + throw startupError + } + let localLoads = 0 + adapter.loadSubset = async (collectionId, options, ctx) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: ctx?.requiredIndexSignatures ?? [], + }) + localLoads++ + return [] + } + const coordinator = createCoordinatorHarness() + let ensureCalls = 0 + coordinator.requestEnsureRemoteSubset = async () => { + ensureCalls++ + if (ensureCalls === 1) throw new Error(`leader unavailable`) + } + let upstreamLoads = 0 + let upstreamBegin: (() => void) | undefined + let upstreamWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let upstreamCommit: (() => SyncAppliedReceipt) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + sourceStarted.resolve() + upstreamBegin = begin + upstreamWrite = write as typeof upstreamWrite + upstreamCommit = commit + markReady() + return { + loadSubset: async () => { + upstreamLoads++ + upstreamBegin?.() + upstreamWrite?.({ + type: `insert`, + value: { id: `remote`, title: `Remote subset` }, + }) + const applied = upstreamCommit?.() + if (applied !== true) await applied + }, + } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const subsetOptions: LoadSubsetOptions = { limit: 7, offset: 2 } + + try { + collection.startSyncImmediate() + await Promise.all([sourceStarted.promise, metadataAttempted.promise]) + vi.useFakeTimers() + await collection._sync.loadSubset(subsetOptions) + await vi.advanceTimersByTimeAsync(0) + + expect({ localLoads, upstreamLoads, ensureCalls }).toEqual({ + localLoads: 1, + upstreamLoads: 1, + ensureCalls: 1, + }) + + await vi.advanceTimersByTimeAsync(50) + expect(ensureCalls).toBe(2) + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `peer-reload`, + latestRowVersion: 2, + requiresFullReload: true, + }) + await vi.advanceTimersByTimeAsync(0) + + expect(localLoads).toBe(2) + expect(adapter.loadSubsetCalls.at(-1)?.options).toBe(subsetOptions) + } finally { + await collection.cleanup() + warning.mockRestore() + vi.useRealTimers() + } + }) + it(`signals readiness once when eager hydration and upstream readiness race`, async () => { const hydrationStarted = deferred() const upstreamStarted = deferred() @@ -3272,6 +4010,164 @@ describe(`persistedCollectionOptions`, () => { }, ) + it(`does not reload an on-demand collection after its final subset unloads`, async () => { + const adapter = createRecordingAdapter([ + { id: `1`, title: `Owned while subscribed` }, + ]) + const coordinator = createCoordinatorHarness() + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const options: LoadSubsetOptions = { limit: 1 } + + collection.startSyncImmediate() + await collection._sync.loadSubset(options) + collection._sync.unloadSubset(options) + const callsAfterUnload = adapter.loadSubsetCalls.length + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `peer-after-final-unload`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await flushAsyncWork() + await flushAsyncWork() + + expect(adapter.loadSubsetCalls).toHaveLength(callsAfterUnload) + await collection.cleanup() + }) + + it(`obeys the demand-ownership law across generated failure and unload histories`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + history: fc.constantFrom<`fallback-active` | `released`>( + `fallback-active`, + `released`, + ), + limit: fc.integer({ min: 1, max: 4 }), + offset: fc.integer({ min: 0, max: 2 }), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ history, limit, offset, salt }) => { + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const adapter = createRecordingAdapter() + const metadataAttempted = deferred() + if (history === `fallback-active`) { + adapter.getStreamPosition = async () => { + metadataAttempted.resolve() + throw new Error(`generated startup failure ${salt}`) + } + } else { + metadataAttempted.resolve() + } + const originalLoadSubset = adapter.loadSubset.bind(adapter) + let localLoads = 0 + adapter.loadSubset = async (collectionId, options, ctx) => { + localLoads++ + return originalLoadSubset(collectionId, options, ctx) + } + const coordinator = createCoordinatorHarness() + let ensureCalls = 0 + coordinator.requestEnsureRemoteSubset = async () => { + ensureCalls++ + if (history === `fallback-active` && ensureCalls === 1) { + throw new Error(`generated leader failure ${salt}`) + } + } + const sourceStarted = deferred() + let upstreamLoads = 0 + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + sourceStarted.resolve() + return { + loadSubset: () => { + upstreamLoads++ + return true + }, + } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const options: LoadSubsetOptions = { limit, offset } + + try { + collection.startSyncImmediate() + await Promise.all([ + sourceStarted.promise, + metadataAttempted.promise, + ]) + vi.useFakeTimers() + await collection._sync.loadSubset(options) + await vi.advanceTimersByTimeAsync(0) + + expect(localLoads).toBe(1) + expect(upstreamLoads).toBe(1) + + if (history === `fallback-active`) { + expect(ensureCalls).toBe(1) + await vi.advanceTimersByTimeAsync(50) + expect(ensureCalls).toBe(2) + } else { + collection._sync.unloadSubset(options) + } + + const callsBeforeReload = localLoads + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `generated-demand-${salt}`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await vi.advanceTimersByTimeAsync(0) + + if (history === `fallback-active`) { + expect(localLoads).toBe(callsBeforeReload + 1) + expect(adapter.loadSubsetCalls.at(-1)?.options).toBe(options) + } else { + expect(localLoads).toBe(callsBeforeReload) + } + } finally { + await collection.cleanup() + vi.useRealTimers() + warning.mockRestore() + } + }, + ), + { numRuns: 8, seed: 18_690_812 }, + ) + }) + + it(`rejects hostile demand traces that fabricate or replace ownership`, () => { + const owned = { limit: 1 } + expect(() => expect({}).toBe(owned)).toThrow() + expect(() => expect(2).toBe(1)).toThrow() + }) + it(`does not retain refresh history as permanent subset demand`, async () => { const adapter = createRecordingAdapter([{ id: `1`, title: `Before` }]) const coordinator = createCoordinatorHarness() diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index de10c93e6c..b0d77d47f0 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' import { afterEach, describe, expect, it } from 'vitest' +import fc from 'fast-check' import { IR } from '@tanstack/db' import { SQLiteCorePersistenceAdapter, createPersistedTableName } from '../src' import { harnessScope } from './contracts/harness-scope' @@ -341,6 +342,160 @@ export function runSQLiteCoreAdapterContractSuite( expect(tombstoneRows[0]?.row_version).toBe(3) }) + it(`returns the canonical application for a stable transaction id at a new leader position`, async () => { + const { adapter, driver } = registerContractHarness() + const collectionId = `stable-transaction-identity` + const first = await adapter.applyCommittedTx(collectionId, { + txId: `logical-transaction`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `1`, + value: { + id: `1`, + title: `Applied once`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: 1, + }, + }, + ], + }) + const retry = await adapter.applyCommittedTx(collectionId, { + txId: `logical-transaction`, + term: 2, + seq: 2, + rowVersion: 2, + mutations: [ + { + type: `insert`, + key: `2`, + value: { + id: `2`, + title: `Must not be applied`, + createdAt: `2026-01-02T00:00:00.000Z`, + score: 2, + }, + }, + ], + }) + + expect(first).toEqual({ + applied: true, + term: 1, + seq: 1, + rowVersion: 1, + }) + expect(retry).toEqual({ + applied: false, + term: 1, + seq: 1, + rowVersion: 1, + }) + expect(await adapter.loadSubset(collectionId, {})).toEqual([ + { + key: `1`, + value: { + id: `1`, + title: `Applied once`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: 1, + }, + }, + ]) + const txRows = await driver.query<{ count: number }>( + `SELECT COUNT(*) AS count FROM applied_tx WHERE collection_id = ?`, + [collectionId], + ) + expect(txRows[0]?.count).toBe(1) + }) + + it(`obeys stable transaction identity across generated leader positions`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + retryTerm: fc.integer({ min: 2, max: 6 }), + retrySeq: fc.integer({ min: 1, max: 6 }), + retryRowVersion: fc.integer({ min: 2, max: 12 }), + salt: fc.integer({ min: 0, max: 1_000_000 }), + }), + async ({ retryTerm, retrySeq, retryRowVersion, salt }) => { + const { adapter } = registerContractHarness() + const collectionId = `stable-id-law-${salt}` + const txId = `logical-${salt}` + const first = await adapter.applyCommittedTx(collectionId, { + txId, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `first`, + value: { + id: `first`, + title: `Canonical`, + createdAt: `2026-01-01T00:00:00.000Z`, + score: salt, + }, + }, + ], + }) + const retry = await adapter.applyCommittedTx(collectionId, { + txId, + term: retryTerm, + seq: retrySeq, + rowVersion: retryRowVersion, + mutations: [ + { + type: `insert`, + key: `duplicate`, + value: { + id: `duplicate`, + title: `Duplicate`, + createdAt: `2026-01-02T00:00:00.000Z`, + score: salt + 1, + }, + }, + ], + }) + + expect(first).toEqual({ + applied: true, + term: 1, + seq: 1, + rowVersion: 1, + }) + expect(retry).toEqual({ + applied: false, + term: 1, + seq: 1, + rowVersion: 1, + }) + expect( + (await adapter.loadSubset(collectionId, {})).map( + (row) => row.key, + ), + ).toEqual([`first`]) + }, + ), + { numRuns: 6, seed: 18_690_909 }, + ) + }) + + it(`rejects a hostile stable-id trace that reports the retry position`, () => { + expect(() => + expect({ applied: false, term: 2, seq: 2, rowVersion: 2 }).toEqual({ + applied: false, + term: 1, + seq: 1, + rowVersion: 1, + }), + ).toThrow() + }) + it(`rolls back partially applied mutations when transaction fails`, async () => { const { adapter, driver } = registerContractHarness() const collectionId = `atomicity` diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index a4c6bb7fe8..4040eca791 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -654,30 +654,49 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin } } - // Assign stream position - state.latestSeq++ - state.latestRowVersion++ + const tx = await this.withWriterLock(async () => { + const currentState = this.collections.get(collectionId) + if (!currentState?.isLeader) { + throw new Error(`not the leader for ${collectionId}`) + } - const term = state.latestTerm - const seq = state.latestSeq - const rowVersion = state.latestRowVersion + const proposedTx = { + txId: safeRandomUUID(), + term: currentState.latestTerm, + seq: currentState.latestSeq + 1, + rowVersion: currentState.latestRowVersion + 1, + mutations: request.mutations.map((m) => ({ + type: m.type, + key: m.key, + value: m.value, + })), + } - // Build and apply the persisted transaction - const tx = { - txId: safeRandomUUID(), - term, - seq, - rowVersion, - mutations: request.mutations.map((m) => ({ - type: m.type, - key: m.key, - value: m.value, - })), - } + const application = await this.requireAdapter().applyCommittedTx( + collectionId, + proposedTx, + ) + const appliedTx = application + ? { + ...proposedTx, + term: application.term, + seq: application.seq, + rowVersion: application.rowVersion, + } + : proposedTx + currentState.latestTerm = Math.max( + currentState.latestTerm, + appliedTx.term, + ) + currentState.latestSeq = Math.max(currentState.latestSeq, appliedTx.seq) + currentState.latestRowVersion = Math.max( + currentState.latestRowVersion, + appliedTx.rowVersion, + ) + return appliedTx + }) - await this.withWriterLock(() => - this.requireAdapter().applyCommittedTx(collectionId, tx), - ) + const { term, seq, rowVersion } = tx // Track envelope for dedup this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) @@ -784,9 +803,15 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin const lockName = `tsdb:writer:${this.dbName}` for (let attempt = 0; attempt <= WRITER_LOCK_MAX_RETRIES; attempt++) { + const lockAttempt = { enteredCallback: false } try { - return await navigator.locks.request(lockName, async () => fn()) + return await navigator.locks.request(lockName, async () => { + lockAttempt.enteredCallback = true + return fn() + }) } catch (error) { + if (lockAttempt.enteredCallback) throw error + if (error instanceof DOMException && error.name === `AbortError`) { throw error } diff --git a/packages/electron-db-sqlite-persistence/src/main.ts b/packages/electron-db-sqlite-persistence/src/main.ts index e412ebd4fe..fe78433ec4 100644 --- a/packages/electron-db-sqlite-persistence/src/main.ts +++ b/packages/electron-db-sqlite-persistence/src/main.ts @@ -156,13 +156,16 @@ async function executeRequestAgainstAdapter( } case `applyCommittedTx`: { - await adapter.applyCommittedTx(request.collectionId, request.payload.tx) + const result = await adapter.applyCommittedTx( + request.collectionId, + request.payload.tx, + ) return { v: ELECTRON_PERSISTENCE_PROTOCOL_VERSION, requestId: request.requestId, method: request.method, ok: true, - result: null, + result: result || null, } } diff --git a/packages/electron-db-sqlite-persistence/src/protocol.ts b/packages/electron-db-sqlite-persistence/src/protocol.ts index 7dcdc8dec5..a305ce6d5d 100644 --- a/packages/electron-db-sqlite-persistence/src/protocol.ts +++ b/packages/electron-db-sqlite-persistence/src/protocol.ts @@ -3,6 +3,7 @@ import type { PersistedCollectionMode, PersistedIndexSpec, PersistedTx, + PersistedTxApplicationResult, SQLitePullSinceResult, } from '@tanstack/db-sqlite-persistence-core' @@ -62,7 +63,7 @@ export type ElectronPersistenceResultMap = { value: ElectronPersistedRow metadata?: unknown }> - applyCommittedTx: null + applyCommittedTx: PersistedTxApplicationResult | null ensureIndex: null markIndexRemoved: null pullSince: SQLitePullSinceResult diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index 2ac7203000..1c04d8adad 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -13,6 +13,7 @@ import type { PersistedCollectionPersistence, PersistedIndexSpec, PersistedTx, + PersistedTxApplicationResult, SQLitePullSinceResult, } from '@tanstack/db-sqlite-persistence-core' import type { @@ -203,8 +204,8 @@ function createResolvedRendererAdapter( applyCommittedTx: async ( collectionId: string, tx: PersistedTx, string | number>, - ): Promise => { - await executeRequest( + ): Promise => { + const result = await executeRequest( `applyCommittedTx`, collectionId, { @@ -212,6 +213,7 @@ function createResolvedRendererAdapter( }, resolution, ) + return result ?? undefined }, loadCollectionMetadata: async ( collectionId: string, diff --git a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts new file mode 100644 index 0000000000..4465559660 --- /dev/null +++ b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts @@ -0,0 +1,196 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import fc from 'fast-check' +import { ElectronCollectionCoordinator } from '../src/electron-coordinator' +import type { PersistenceAdapter } from '../../db-sqlite-persistence-core/src' + +/** + * Protected-work failure law. + * + * Source: a writer lock may retry admission, but work invoked after admission + * has exactly one durable attempt and a failure consumes no stream position. + * Generated histories cross synchronous and asynchronous callback rejection with + * one to three mutations. The real Electron coordinator is observed at adapter + * calls, error identity, and the next successful public response. The hostile + * trace rejects both replay and a consumed sequence; browser parity is owned by + * the corresponding BrowserCollectionCoordinator law. + */ + +class MockBroadcastChannel { + onmessage: ((event: MessageEvent) => void) | null = null + + postMessage(): void {} + close(): void {} +} + +const locks = { + request: ( + name: string, + optionsOrCallback: + | { signal?: AbortSignal } + | ((lock: { name: string }) => Promise | T), + maybeCallback?: (lock: { name: string }) => Promise | T, + ): Promise => { + const options = + typeof optionsOrCallback === `function` ? undefined : optionsOrCallback + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + if (options?.signal?.aborted) { + return Promise.reject( + new DOMException(`Lock request aborted`, `AbortError`), + ) + } + return Promise.resolve(callback({ name })) + }, +} + +function createAdapter(): PersistenceAdapter { + return { + loadSubset: async () => [], + applyCommittedTx: async () => {}, + ensureIndex: async () => {}, + getStreamPosition: async () => ({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }), + } +} + +describe(`ElectronCollectionCoordinator`, () => { + beforeEach(() => { + vi.stubGlobal(`BroadcastChannel`, MockBroadcastChannel) + vi.stubGlobal(`navigator`, { locks }) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it(`does not retry an adapter application failure under the writer lock`, async () => { + const failure = new Error(`durable application failed after entry`) + const adapter = createAdapter() + let applyCalls = 0 + adapter.applyCommittedTx = async () => { + applyCalls++ + if (applyCalls === 1) throw failure + } + const coordinator = new ElectronCollectionCoordinator({ + dbName: `writer-failure-law`, + adapter, + }) + coordinator.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(coordinator.isLeader(`todos`)).toBe(true)) + + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, [ + { + mutationId: `one-logical-attempt`, + type: `insert`, + key: `1`, + value: { id: `1`, title: `Apply once` }, + }, + ]) + .then( + (response) => response, + (error: unknown) => error, + ) + + expect(applyCalls).toBe(1) + expect(outcome).toBe(failure) + + const recovery = await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `later-success`, + type: `insert`, + key: `2`, + value: { id: `2`, title: `Later success` }, + }, + ]) + expect(recovery).toMatchObject({ ok: true, seq: 1, latestRowVersion: 1 }) + expect(applyCalls).toBe(2) + coordinator.dispose() + }) + + it(`obeys protected-work failure across generated callback boundaries`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + boundary: fc.constantFrom<`sync` | `async`>(`sync`, `async`), + mutationCount: fc.integer({ min: 1, max: 3 }), + salt: fc.integer({ min: 0, max: 10_000 }), + }), + async ({ boundary, mutationCount, salt }) => { + const failure = new Error(`generated durable failure ${salt}`) + const adapter = createAdapter() + let applyCalls = 0 + adapter.applyCommittedTx = () => { + applyCalls++ + if (applyCalls !== 1) return Promise.resolve() + if (boundary === `sync`) throw failure + return Promise.resolve().then(() => { + throw failure + }) + } + const coordinator = new ElectronCollectionCoordinator({ + dbName: `generated-writer-failure-${salt}`, + adapter, + }) + coordinator.subscribe(`todos`, () => {}) + + try { + await vi.waitFor(() => + expect(coordinator.isLeader(`todos`)).toBe(true), + ) + const mutations = Array.from( + { length: mutationCount }, + (_, index) => ({ + mutationId: `failed-${salt}-${index}`, + type: `insert` as const, + key: `${index}`, + value: { id: `${index}`, title: `Generated ${index}` }, + }), + ) + const outcome = await coordinator + .requestApplyLocalMutations(`todos`, mutations) + .then( + (response) => response, + (error: unknown) => error, + ) + + expect(outcome).toBe(failure) + expect(applyCalls).toBe(1) + await expect( + coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `recovery-${salt}`, + type: `insert`, + key: `recovery`, + value: { id: `recovery`, title: `Recovery` }, + }, + ]), + ).resolves.toMatchObject({ + ok: true, + seq: 1, + latestRowVersion: 1, + }) + expect(applyCalls).toBe(2) + } finally { + coordinator.dispose() + } + }, + ), + { numRuns: 6, seed: 18_690_313 }, + ) + }) + + it(`rejects a hostile protected-work trace that retries or consumes a position`, () => { + expect(() => + expect({ calls: 2, recoverySeq: 2 }).toEqual({ + calls: 1, + recoverySeq: 1, + }), + ).toThrow() + }) +}) diff --git a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts index d1c330fb6f..aeb059d426 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts @@ -181,22 +181,56 @@ describe(`electron sqlite persistence bridge`, () => { timeoutMs: electronRuntimeBridgeTimeoutMs, }) - await rendererPersistence.adapter.applyCommittedTx(`todos`, { - txId: `tx-1`, + const firstApplication = await rendererPersistence.adapter.applyCommittedTx( + `todos`, + { + txId: `tx-1`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `1`, + value: { + id: `1`, + title: `From renderer`, + score: 10, + }, + }, + ], + }, + ) + const retriedApplication = + await rendererPersistence.adapter.applyCommittedTx(`todos`, { + txId: `tx-1`, + term: 2, + seq: 2, + rowVersion: 2, + mutations: [ + { + type: `insert`, + key: `duplicate`, + value: { + id: `duplicate`, + title: `Must not cross the bridge as a second application`, + score: 11, + }, + }, + ], + }) + + expect(firstApplication).toEqual({ + applied: true, + term: 1, + seq: 1, + rowVersion: 1, + }) + expect(retriedApplication).toEqual({ + applied: false, term: 1, seq: 1, rowVersion: 1, - mutations: [ - { - type: `insert`, - key: `1`, - value: { - id: `1`, - title: `From renderer`, - score: 10, - }, - }, - ], }) const rows = await rendererPersistence.adapter.loadSubset(`todos`, {}) From 27437bc1db806c3c2ee2c58df92c42747885e953 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 15:11:15 +0100 Subject: [PATCH 07/10] test(electric): harden recovery oracle replay --- .../tests/electric-recovery-oracle.test.ts | 285 +++++++++++++++--- 1 file changed, 241 insertions(+), 44 deletions(-) diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 1ab3f1e68d..b8ca253812 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -1,5 +1,5 @@ import { isDeepStrictEqual } from 'node:util' -import { fc, test as fcTest } from '@fast-check/vitest' +import { fc } from '@fast-check/vitest' import { beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '@tanstack/db' import { ShapeStream } from '@electric-sql/client' @@ -39,17 +39,158 @@ import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' * Observation cuts include coordinator metadata publication, `up-to-date`, * restart, resume metadata, and exact public and durable rows. * - * Replay matrices by exact Vitest title; replay the generated property with the - * fast-check seed and path printed on failure. The trace fault control rejects a - * missing intermediate publication, while exact row checks reject stale cached - * rows with fresh metadata and incomplete replacement. This is a fixed matrix - * plus a bounded generated property over a mocked ShapeStream, not a live - * Electric service or PowerSync authority. + * Normal runs retain one fixed campaign and add one seedless campaign over the + * same property. Set both TANSTACK_DB_ELECTRIC_RECOVERY_ORACLE_SEED and + * TANSTACK_DB_ELECTRIC_RECOVERY_ORACLE_PATH to run only that replay. The named + * trace fault control rejects the plausible wrong answer that omits an observed + * stale publication. This is a fixed matrix plus a bounded generated property + * over a mocked ShapeStream, not a live Electric service or PowerSync authority. */ type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } +type RecoveryCommand = { + id: number + name: string + deleted: boolean + fullReload: boolean + splitCommitControl: boolean +} +type RecoveryHistory = { + startupOrder: `hydrate-before-ready` | `ready-before-hydrate` + commands: Array +} +type RecoveryCampaign = { + name: `fixed` | `random` | `replay` + seed: number | undefined + path: string | undefined +} + +const FIXED_RECOVERY_SEED = 1_869_1659 +const RECOVERY_SEED_ENV = `TANSTACK_DB_ELECTRIC_RECOVERY_ORACLE_SEED` +const RECOVERY_PATH_ENV = `TANSTACK_DB_ELECTRIC_RECOVERY_ORACLE_PATH` +const RECOVERY_RUNS_ENV = `TANSTACK_DB_ELECTRIC_RECOVERY_ORACLE_RUNS` + +function readRecoveryCampaigns( + environment: Record, +): { campaigns: Array; numRuns: number } { + const seedText = environment[RECOVERY_SEED_ENV] + const path = environment[RECOVERY_PATH_ENV] + const runsText = environment[RECOVERY_RUNS_ENV] ?? `20` + const numRuns = Number(runsText) + if ( + runsText.trim() === `` || + !Number.isSafeInteger(numRuns) || + numRuns <= 0 + ) { + throw new Error(`${RECOVERY_RUNS_ENV} must be a positive integer`) + } + if (seedText === undefined && path === undefined) { + return { + campaigns: [ + { name: `fixed`, seed: FIXED_RECOVERY_SEED, path: undefined }, + { name: `random`, seed: undefined, path: undefined }, + ], + numRuns, + } + } + if (seedText === undefined) { + throw new Error(`${RECOVERY_PATH_ENV} requires ${RECOVERY_SEED_ENV}`) + } + if (path === undefined) { + throw new Error(`${RECOVERY_SEED_ENV} requires ${RECOVERY_PATH_ENV}`) + } + const seed = Number(seedText) + if (seedText.trim() === `` || !Number.isSafeInteger(seed)) { + throw new Error(`${RECOVERY_SEED_ENV} must be an integer`) + } + if (path.trim() === `` || !/^\d+(?::\d+)*$/.test(path)) { + throw new Error( + `${RECOVERY_PATH_ENV} must contain colon-separated nonnegative integers`, + ) + } + return { + campaigns: [{ name: `replay`, seed, path }], + numRuns, + } +} + +/** + * The pinned examples reconstruct the retained insert/delete/full-reload + * witness and both startup orders. Removing startupOrder loses the independent + * ready/hydrate boundary; removing delete or fullReload loses membership or + * peer-reload transitions; removing splitCommitControl loses legal callback + * partitioning. IDs 2..4 permit same-key and disjoint histories, names include + * empty/bounded strings, and 1..8 commands include the one-step marginal case. + * A change batch without up-to-date/subset-end is intentionally excluded: it is + * open protocol state, not a completed Electric publication. + */ +const recoveryHistoryArbitrary = fc.record({ + startupOrder: fc.constantFrom( + `hydrate-before-ready` as const, + `ready-before-hydrate` as const, + ), + commands: fc.array( + fc.record({ + id: fc.integer({ min: 2, max: 4 }), + name: fc.string({ maxLength: 8 }), + deleted: fc.boolean(), + fullReload: fc.boolean(), + splitCommitControl: fc.boolean(), + }), + { minLength: 1, maxLength: 8 }, + ), +}) + +const recoveryExamples: Array<[RecoveryHistory]> = [ + [ + { + startupOrder: `hydrate-before-ready`, + commands: [ + { + id: 2, + name: `external`, + deleted: false, + fullReload: false, + splitCommitControl: false, + }, + { + id: 2, + name: `removed`, + deleted: true, + fullReload: true, + splitCommitControl: true, + }, + ], + }, + ], + [ + { + startupOrder: `ready-before-hydrate`, + commands: [ + { + id: 3, + name: `ready-first`, + deleted: false, + fullReload: true, + splitCommitControl: false, + }, + ], + }, + ], +] + +const recoveryConfig = readRecoveryCampaigns(process.env) + +function recoveryCampaignParameters(campaign: RecoveryCampaign) { + return { + numRuns: recoveryConfig.numRuns, + examples: recoveryExamples, + ...(campaign.seed === undefined ? {} : { seed: campaign.seed }), + ...(campaign.path === undefined ? {} : { path: campaign.path }), + } +} function expectWholeRecoveryTrace( entries: Array, @@ -235,7 +376,39 @@ describe(`persisted Electric recovery laws`, () => { vi.clearAllMocks() }) - it(`keeps repaired intermediate publications in the persisted recovery record`, async () => { + it(`selects only a validated seed-and-path replay when requested`, () => { + expect( + readRecoveryCampaigns({ + [RECOVERY_SEED_ENV]: `42`, + [RECOVERY_PATH_ENV]: `0:3:1`, + [RECOVERY_RUNS_ENV]: `7`, + }), + ).toEqual({ + campaigns: [{ name: `replay`, seed: 42, path: `0:3:1` }], + numRuns: 7, + }) + }) + + it.each([ + [{ [RECOVERY_PATH_ENV]: `0` }, `requires ${RECOVERY_SEED_ENV}`], + [{ [RECOVERY_SEED_ENV]: `42` }, `requires ${RECOVERY_PATH_ENV}`], + [ + { [RECOVERY_SEED_ENV]: `nope`, [RECOVERY_PATH_ENV]: `0` }, + `must be an integer`, + ], + [ + { [RECOVERY_SEED_ENV]: `42`, [RECOVERY_PATH_ENV]: `0:-1` }, + `colon-separated nonnegative integers`, + ], + [{ [RECOVERY_RUNS_ENV]: `0` }, `must be a positive integer`], + ])( + `rejects an invalid recovery replay configuration`, + (environment, text) => { + expect(() => readRecoveryCampaigns(environment)).toThrow(text) + }, + ) + + it(`rejects the wrong whole-trace answer that omits a stale intermediate publication`, async () => { const f = fixture(`eager`) try { f.start() @@ -256,9 +429,8 @@ describe(`persisted Electric recovery laws`, () => { kind === `event` && rows[0]?.name === `wrong`, ), ).toBe(true) - expect(() => - expectWholeRecoveryTrace(entries, [[oldRow], correct]), - ).toThrow() + const wrongAnswer = [[oldRow], correct] + expect(() => expectWholeRecoveryTrace(entries, wrongAnswer)).toThrow() expectWholeRecoveryTrace(entries, [ [oldRow], [{ ...oldRow, name: `wrong` }], @@ -373,32 +545,9 @@ describe(`persisted Electric recovery laws`, () => { }, ) - fcTest.prop( - [ - fc.array( - fc.record({ - id: fc.integer({ min: 2, max: 4 }), - name: fc.string({ maxLength: 8 }), - deleted: fc.boolean(), - fullReload: fc.boolean(), - }), - { minLength: 1, maxLength: 8 }, - ), - ], - { - numRuns: 20, - examples: [ - [ - [ - { id: 2, name: `external`, deleted: false, fullReload: false }, - { id: 2, name: `removed`, deleted: true, fullReload: true }, - ], - ], - ], - }, - )( - `independent persistence publications and stream deltas agree with complete-row state`, - async (commands) => { + const independentPublicationProperty = fc.asyncProperty( + recoveryHistoryArbitrary, + async ({ startupOrder, commands }) => { subscribers.length = 0 const peer = externalPublisher() const f = fixture(`on-demand`, peer.coordinator) @@ -410,8 +559,25 @@ describe(`persisted Electric recovery laws`, () => { await vi.waitFor(() => expect(subscribers).toHaveLength(1), { interval: 1, }) - await f.collection._sync.loadSubset({}) - subscribers[0]!([upToDate]) + + // These two legal startup orders ablate the only readiness boundary in + // this history. On-demand hydration alone is not upstream readiness. + if (startupOrder === `hydrate-before-ready`) { + await f.collection._sync.loadSubset({}) + expect(f.collection.status).toBe(`loading`) + subscribers[0]!([upToDate]) + } else { + subscribers[0]!([upToDate]) + await vi.waitFor(() => expect(f.collection.status).toBe(`ready`), { + interval: 1, + }) + await f.collection._sync.loadSubset({}) + } + await vi.waitFor(() => expect(f.collection.status).toBe(`ready`), { + interval: 1, + }) + expect(f.publicRows()).toEqual(expectedRows()) + for (const command of commands) { const before = expectedRows() const cut = f.exposures.length @@ -447,18 +613,39 @@ describe(`persisted Electric recovery laws`, () => { { interval: 1 }, ) expect(f.publicRows()).toEqual(expectedRows()) + expect(f.collection.status).toBe(`ready`) f.record(`peer revision ${revision} settled`) const afterPeer = expectedRows() expectWholeRecoveryTrace(f.exposures.slice(cut), [before, afterPeer]) const streamCut = f.exposures.length f.record(`before stream revision ${revision}`) - subscribers[0]!([ - change(`update`, { id: row.id, name: `stream` }), - upToDate, - ]) + const streamUpdate = change(`update`, { + id: row.id, + name: `stream`, + }) + if (command.splitCommitControl) { + subscribers[0]!([streamUpdate]) + f.record(`after uncommitted stream data ${revision}`) + expect(f.publicRows()).toEqual(afterPeer) + expect(f.durableRows()).toEqual(afterPeer) + expect(f.collection.status).toBe(`ready`) + subscribers[0]!([upToDate]) + } else { + subscribers[0]!([streamUpdate, upToDate]) + } if (!command.deleted) expected.set(row.id, { ...row, name: `stream` }) f.record(`after stream revision ${revision}`) - expect(f.publicRows()).toEqual(expectedRows()) + expect( + f.publicRows(), + JSON.stringify({ + command, + publicRows: f.publicRows(), + durableRows: f.durableRows(), + expectedRows: expectedRows(), + status: f.collection.status, + }), + ).toEqual(expectedRows()) + expect(f.collection.status).toBe(`ready`) await vi.waitFor( () => expect(f.durableRows()).toEqual(expectedRows()), { interval: 1 }, @@ -475,6 +662,16 @@ describe(`persisted Electric recovery laws`, () => { }, ) + it.each(recoveryConfig.campaigns)( + `independent persistence publications and stream deltas agree with complete-row state ($name campaign)`, + async (campaign) => { + await fc.assert( + independentPublicationProperty, + recoveryCampaignParameters(campaign), + ) + }, + ) + it.each(scenarios)( `$syncMode invalid resume replaces omitted cached rows: empty=$empty, hydration=$hydration callback`, async ({ syncMode, empty, hydration }) => { From 2bf9d49a8f43be5db80d41f42757e944cec17a29 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 16:52:53 +0100 Subject: [PATCH 08/10] test(persistence): preserve reentrant readiness guard --- .../tests/persisted.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index fe550b1817..a5d639972e 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1811,6 +1811,42 @@ describe(`persistedCollectionOptions`, () => { await collection.cleanup() }) + it(`signals ready once under synchronous upstream-ready reentry`, async () => { + let signalUpstreamReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `on-demand-reentrant-readiness`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + signalUpstreamReady = markReady + return {} + }, + }, + persistence: { adapter: createRecordingAdapter() }, + }), + ) + const lifecycle = collection._lifecycle as unknown as { + markReady: () => void + } + const originalMarkReady = lifecycle.markReady.bind(lifecycle) + let underlyingReadySignals = 0 + lifecycle.markReady = () => { + underlyingReadySignals++ + signalUpstreamReady?.() + originalMarkReady() + } + + collection.startSyncImmediate() + await vi.waitFor(() => expect(signalUpstreamReady).toBeTypeOf(`function`)) + signalUpstreamReady!() + + expect(underlyingReadySignals).toBe(1) + expect(collection.status).toBe(`ready`) + await collection.cleanup() + }) + it(`preserves upstream ready-error-ready transitions for on-demand sync`, async () => { const upstreamError = new Error(`rebuild failed`) const upstreamStarted = deferred() From 685ecb637d4eeb08d6c234a8a45df9ea884ef4dd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 17:12:57 +0100 Subject: [PATCH 09/10] fix(browser-sqlite): reject non-finite peer positions --- .../src/browser-coordinator.ts | 6 +- .../tests/browser-coordinator.test.ts | 67 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 5fa5ccb838..903beadae6 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -821,7 +821,6 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (!state?.isLeader) throw new LostLeadershipError() const adapter = this.requireAdapter() - if (!this.isLeader(collectionId)) throw new LostLeadershipError() const proposedTx: PersistedTx = { ...transaction, term: state.latestTerm, @@ -1002,7 +1001,10 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina record.type !== `tx:committed` || typeof record.term !== `number` || typeof record.seq !== `number` || - typeof record.latestRowVersion !== `number` + typeof record.latestRowVersion !== `number` || + !Number.isFinite(record.term) || + !Number.isFinite(record.seq) || + !Number.isFinite(record.latestRowVersion) ) { return } diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 6a8487ae4f..6b1baed6bd 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -429,6 +429,73 @@ describe(`BrowserCollectionCoordinator`, () => { expect(received.length).toBe(0) coord.dispose() }) + + it.each([Number.NaN, Number.POSITIVE_INFINITY])( + `ignores a peer transaction with a non-finite position: %s`, + async (nonFinitePosition) => { + const adapter = createStubAdapter() + let appliedTx: PersistedTx | undefined + adapter.applyCommittedTx = (collectionId, tx) => { + appliedTx = tx + adapter.appliedTxs.push({ collectionId, txId: tx.txId }) + return Promise.resolve() + } + const leader = createCoordinator(adapter) + const peer = createCoordinator() + leader.subscribe(`todos`, () => {}) + await flush(50) + peer.subscribe(`todos`, () => {}) + + try { + peer.publish(`todos`, { + v: 1, + dbName: `test-db`, + collectionId: `todos`, + senderId: peer.getNodeId(), + ts: Date.now(), + payload: { + type: `tx:committed`, + term: nonFinitePosition, + seq: nonFinitePosition, + txId: `hostile-peer-position`, + latestRowVersion: nonFinitePosition, + requiresFullReload: true, + }, + }) + await flush() + + const response = await leader.requestApplyPersistedTransaction( + `todos`, + { + txId: `valid-after-hostile-peer`, + mutations: [], + }, + ) + if (!response.ok) { + throw new Error(`valid transaction was rejected: ${response.error}`) + } + + expect({ + applied: appliedTx && { + term: appliedTx.term, + seq: appliedTx.seq, + rowVersion: appliedTx.rowVersion, + }, + response: { + term: response.term, + seq: response.seq, + latestRowVersion: response.latestRowVersion, + }, + }).toEqual({ + applied: { term: 1, seq: 1, rowVersion: 1 }, + response: { term: 1, seq: 1, latestRowVersion: 1 }, + }) + } finally { + peer.dispose() + leader.dispose() + } + }, + ) }) describe(`RPC - applyLocalMutations`, () => { From c24f642c44028c437d729cebd98382263d9bab01 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 17:50:00 +0100 Subject: [PATCH 10/10] fix(sqlite-persistence): close coordination races --- .../src/browser-coordinator.ts | 102 ++++-- .../tests/browser-coordinator.test.ts | 318 ++++++++++++++++++ .../src/persisted.ts | 40 ++- .../src/sqlite-core-adapter.ts | 4 + .../tests/persisted.test.ts | 155 +++++++++ .../tests/sqlite-core-adapter.test.ts | 62 ++++ .../src/electron-coordinator.ts | 68 ++-- .../tests/electron-coordinator.test.ts | 94 ++++++ 8 files changed, 786 insertions(+), 57 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts index 903beadae6..e8b0f9dfef 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-coordinator.ts @@ -83,6 +83,7 @@ type PendingRPC = { type CollectionState = { isLeader: boolean + isPositionReady: boolean lockAbortController: AbortController | null heartbeatTimer: ReturnType | null latestTerm: number @@ -137,7 +138,16 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private readonly channel: BroadcastChannel private readonly collections = new Map() private readonly pendingRPCs = new Map() - private readonly appliedEnvelopeIds = new Map() + private readonly appliedEnvelopeResults = new Map< + string, + { + timestamp: number + term: number + seq: number + latestRowVersion: number + acceptedMutationIds: Array + } + >() private readonly appliedPersistedTransactions = new Map< string, { @@ -154,6 +164,17 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina return this.disposed } + private hasRecoveredLeadership( + state: CollectionState, + abortController: AbortController, + ): boolean { + return ( + state.isLeader && + state.isPositionReady && + state.lockAbortController === abortController + ) + } + private requireAdapter(): AdapterWithPullSince { if (!this.adapter) { throw new Error( @@ -374,6 +395,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (!state) { state = { isLeader: false, + isPositionReady: false, lockAbortController: null, heartbeatTimer: null, latestTerm: 0, @@ -405,17 +427,34 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina if (this.isDisposed()) return try { - // Restore stream position from DB before claiming leadership - const adapter = this.requireAdapter() - if (adapter.getStreamPosition) { - const pos = await adapter.getStreamPosition(collectionId) - state.latestTerm = pos.latestTerm - state.latestSeq = pos.latestSeq - state.latestRowVersion = pos.latestRowVersion - } - - state.latestTerm++ state.isLeader = true + state.isPositionReady = false + + // Serialize position recovery with outgoing leader writes so a new + // leader cannot cache a position from before their final commit. + await this.withWriterLock(async () => { + if ( + this.isDisposed() || + abortController.signal.aborted || + state.lockAbortController !== abortController + ) { + return + } + + const adapter = this.requireAdapter() + if (adapter.getStreamPosition) { + const pos = await adapter.getStreamPosition(collectionId) + state.latestTerm = pos.latestTerm + state.latestSeq = pos.latestSeq + state.latestRowVersion = pos.latestRowVersion + } + + state.latestTerm++ + state.isPositionReady = true + }) + if (!this.hasRecoveredLeadership(state, abortController)) { + return + } this.emitHeartbeat(collectionId, state) state.heartbeatTimer = setInterval(() => { @@ -436,6 +475,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina }) } finally { state.isLeader = false + state.isPositionReady = false if (state.heartbeatTimer) { clearInterval(state.heartbeatTimer) state.heartbeatTimer = null @@ -469,6 +509,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina state.heartbeatTimer = null } state.isLeader = false + state.isPositionReady = false } private emitHeartbeat(collectionId: string, state: CollectionState): void { @@ -699,19 +740,23 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina ): Promise { try { return await this.withWriterLock(async () => { - if (this.appliedEnvelopeIds.has(request.envelopeId)) { + const envelopeKey = JSON.stringify([collectionId, request.envelopeId]) + const prior = this.appliedEnvelopeResults.get(envelopeKey) + if (prior) { return { type: `rpc:applyLocalMutations:res` as const, rpcId: request.rpcId, - ok: false as const, - code: `CONFLICT` as const, - error: `envelope ${request.envelopeId} already applied`, + ok: true as const, + term: prior.term, + seq: prior.seq, + latestRowVersion: prior.latestRowVersion, + acceptedMutationIds: prior.acceptedMutationIds, } } const appliedTransaction = await this.applyPositionlessTransactionWithWriterLock(collectionId, { - txId: safeRandomUUID(), + txId: request.envelopeId, mutations: request.mutations.map((mutation) => ({ type: mutation.type, key: mutation.key, @@ -719,8 +764,17 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina })), }) const tx = appliedTransaction.tx + const acceptedMutationIds = request.mutations.map( + (mutation) => mutation.mutationId, + ) - this.appliedEnvelopeIds.set(request.envelopeId, Date.now()) + this.appliedEnvelopeResults.set(envelopeKey, { + timestamp: Date.now(), + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + acceptedMutationIds, + }) this.pruneAppliedEnvelopeIds() if (appliedTransaction.applied) { this.publishCommittedTransaction(collectionId, tx) @@ -733,9 +787,7 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina term: tx.term, seq: tx.seq, latestRowVersion: tx.rowVersion, - acceptedMutationIds: request.mutations.map( - (mutation) => mutation.mutationId, - ), + acceptedMutationIds, } }) } catch (error) { @@ -818,7 +870,9 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina transaction: PositionlessPersistedTx, ): Promise<{ tx: PersistedTx; applied: boolean }> { const state = this.collections.get(collectionId) - if (!state?.isLeader) throw new LostLeadershipError() + if (!state?.isLeader || !state.isPositionReady) { + throw new LostLeadershipError() + } const adapter = this.requireAdapter() const proposedTx: PersistedTx = { @@ -1038,9 +1092,9 @@ export class BrowserCollectionCoordinator implements PersistedCollectionCoordina private pruneAppliedEnvelopeIds(): void { // Keep envelopes for 60 seconds for dedup const cutoff = Date.now() - 60_000 - for (const [id, ts] of this.appliedEnvelopeIds) { - if (ts < cutoff) { - this.appliedEnvelopeIds.delete(id) + for (const [id, result] of this.appliedEnvelopeResults) { + if (result.timestamp < cutoff) { + this.appliedEnvelopeResults.delete(id) } } for (const [txId, applied] of this.appliedPersistedTransactions) { diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index 6b1baed6bd..d215088c47 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -49,6 +49,7 @@ const channels: Map< string, Set<{ onmessage: MessageHandler | null }> > = new Map() +let shouldDropBroadcastMessage: ((data: unknown) => boolean) | null = null class MockBroadcastChannel { readonly name: string @@ -63,6 +64,8 @@ class MockBroadcastChannel { } postMessage(data: unknown): void { + if (shouldDropBroadcastMessage?.(data)) return + const peers = channels.get(this.name) if (!peers) return // Deliver to all other instances on same channel (simulating cross-tab) @@ -87,6 +90,7 @@ class MockBroadcastChannel { type LockGrantedCallback = (lock: { name: string }) => Promise const heldLocks = new Map void }>() +let onLockGranted: ((name: string) => void) | null = null const lockQueues = new Map< string, Array<{ @@ -115,6 +119,7 @@ function tryGrantNextLock(name: string): void { }) heldLocks.set(name, { release: releaseCallback }) + onLockGranted?.(name) const result = next.callback({ name }) // When the callback resolves/rejects, release the lock @@ -199,6 +204,8 @@ function installGlobals(): void { } function cleanupGlobals(): void { + shouldDropBroadcastMessage = null + onLockGranted = null channels.clear() heldLocks.clear() lockQueues.clear() @@ -890,9 +897,320 @@ describe(`BrowserCollectionCoordinator`, () => { coord.dispose() }) + + it(`returns canonical success when a local-mutation response is retried on the same leader`, async () => { + const adapter = createStubAdapter() + const applications: Array = [] + const appliedByTxId = new Map< + string, + { term: number; seq: number; rowVersion: number } + >() + adapter.applyCommittedTx = (_collectionId, tx) => { + const prior = appliedByTxId.get(tx.txId) + if (prior) return Promise.resolve({ applied: false, ...prior }) + + applications.push(tx) + const applied = { + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + } + appliedByTxId.set(tx.txId, applied) + return Promise.resolve({ applied: true, ...applied }) + } + + const leader = createCoordinator(adapter) + const requester = createCoordinator(adapter) + leader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(leader.isLeader(`todos`)).toBe(true)) + + const responseDropped = deferred() + shouldDropBroadcastMessage = (data) => { + const payload = + typeof data === `object` && data !== null && `payload` in data + ? (data as { payload?: { type?: unknown } }).payload + : undefined + if (payload?.type !== `rpc:applyLocalMutations:res`) return false + + shouldDropBroadcastMessage = null + responseDropped.resolve() + return true + } + + vi.useFakeTimers() + try { + const response = requester.requestApplyLocalMutations(`todos`, [ + { + mutationId: `same-leader-retry`, + type: `insert`, + key: `same-leader-retry`, + value: { id: `same-leader-retry`, title: `Applied once` }, + }, + ]) + await responseDropped.promise + await vi.advanceTimersByTimeAsync(10_200) + + await expect(response).resolves.toMatchObject({ + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: [`same-leader-retry`], + }) + expect(applications).toHaveLength(1) + } finally { + vi.useRealTimers() + leader.dispose() + requester.dispose() + } + }) + + it(`scopes canonical local-mutation retries by collection`, async () => { + const adapter = createStubAdapter() + const leader = createCoordinator(adapter) + leader.subscribe(`alpha`, () => {}) + leader.subscribe(`beta`, () => {}) + await vi.waitFor(() => { + expect(leader.isLeader(`alpha`)).toBe(true) + expect(leader.isLeader(`beta`)).toBe(true) + }) + + const handleApplyLocalMutations = ( + leader as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ ok: boolean }> + } + ).handleApplyLocalMutations.bind(leader) + + const envelopeId = `shared-envelope-id` + const alpha = await handleApplyLocalMutations(`alpha`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `alpha-rpc`, + envelopeId, + mutations: [ + { + mutationId: `alpha-mutation`, + type: `insert`, + key: `alpha`, + value: { id: `alpha` }, + }, + ], + }) + const beta = await handleApplyLocalMutations(`beta`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `beta-rpc`, + envelopeId, + mutations: [ + { + mutationId: `beta-mutation`, + type: `insert`, + key: `beta`, + value: { id: `beta` }, + }, + ], + }) + + expect({ alpha, beta, appliedTxs: adapter.appliedTxs }).toEqual({ + alpha: expect.objectContaining({ ok: true }), + beta: expect.objectContaining({ ok: true }), + appliedTxs: [ + { collectionId: `alpha`, txId: envelopeId }, + { collectionId: `beta`, txId: envelopeId }, + ], + }) + + leader.dispose() + }) + + it(`preserves local-mutation identity when a lost response crosses a leader handoff`, async () => { + const adapter = createStubAdapter() + let durablePosition = { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + } + const applications: Array = [] + const appliedByTxId = new Map< + string, + { term: number; seq: number; rowVersion: number } + >() + adapter.getStreamPosition = () => Promise.resolve(durablePosition) + adapter.applyCommittedTx = (_collectionId, tx) => { + const prior = appliedByTxId.get(tx.txId) + if (prior) return Promise.resolve({ applied: false, ...prior }) + + applications.push(tx) + const applied = { + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + } + appliedByTxId.set(tx.txId, applied) + durablePosition = { + latestTerm: tx.term, + latestSeq: tx.seq, + latestRowVersion: tx.rowVersion, + } + return Promise.resolve({ applied: true, ...applied }) + } + + const firstLeader = createCoordinator(adapter) + const nextLeader = createCoordinator(adapter) + const requester = createCoordinator(adapter) + firstLeader.subscribe(`todos`, () => {}) + nextLeader.subscribe(`todos`, () => {}) + requester.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(firstLeader.isLeader(`todos`)).toBe(true)) + + const responseDropped = deferred() + shouldDropBroadcastMessage = (data) => { + const payload = + typeof data === `object` && data !== null && `payload` in data + ? (data as { payload?: { type?: unknown } }).payload + : undefined + if (payload?.type !== `rpc:applyLocalMutations:res`) return false + + shouldDropBroadcastMessage = null + firstLeader.dispose() + responseDropped.resolve() + return true + } + + vi.useFakeTimers() + try { + const response = requester.requestApplyLocalMutations(`todos`, [ + { + mutationId: `handoff-local-mutation`, + type: `insert`, + key: `handoff-local-mutation`, + value: { id: `handoff-local-mutation`, title: `Applied once` }, + }, + ]) + await responseDropped.promise + + for (let attempt = 0; attempt < 20; attempt++) { + if (nextLeader.isLeader(`todos`)) break + await Promise.resolve() + } + expect(nextLeader.isLeader(`todos`)).toBe(true) + + await vi.advanceTimersByTimeAsync(10_200) + await expect(response).resolves.toMatchObject({ + ok: true, + acceptedMutationIds: [`handoff-local-mutation`], + }) + expect(applications).toHaveLength(1) + } finally { + vi.useRealTimers() + firstLeader.dispose() + nextLeader.dispose() + requester.dispose() + } + }) }) describe(`persisted transaction retry and handoff`, () => { + it(`serializes successor position recovery with an outgoing leader commit`, async () => { + const adapter = createStubAdapter() + const applyStarted = deferred() + const releaseApply = deferred() + let durablePosition = { + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + } + const positionReads: Array = [] + const appliedPositions: Array = [] + adapter.getStreamPosition = () => { + positionReads.push({ ...durablePosition }) + return Promise.resolve({ ...durablePosition }) + } + adapter.applyCommittedTx = async (_collectionId, tx) => { + if (tx.txId === `outgoing-leader-commit`) { + applyStarted.resolve() + await releaseApply.promise + } + durablePosition = { + latestTerm: tx.term, + latestSeq: tx.seq, + latestRowVersion: tx.rowVersion, + } + appliedPositions.push({ ...durablePosition }) + return { + applied: true, + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + } + } + + const firstLeader = createCoordinator(adapter) + const nextLeader = createCoordinator(adapter) + firstLeader.subscribe(`todos`, () => {}) + nextLeader.subscribe(`todos`, () => {}) + await vi.waitFor(() => expect(firstLeader.isLeader(`todos`)).toBe(true)) + + const successorLeadershipLockGranted = deferred() + onLockGranted = (name) => { + if (name === `tsdb:leader:test-db:todos`) { + successorLeadershipLockGranted.resolve() + } + } + const outgoingCommit = firstLeader.requestApplyPersistedTransaction( + `todos`, + { txId: `outgoing-leader-commit`, mutations: [] }, + ) + + try { + await applyStarted.promise + firstLeader.dispose() + await successorLeadershipLockGranted.promise + await Promise.resolve() + await Promise.resolve() + + expect(positionReads).toEqual([ + { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 }, + ]) + + releaseApply.resolve() + await expect(outgoingCommit).resolves.toMatchObject({ ok: true }) + await vi.waitFor(() => expect(nextLeader.isLeader(`todos`)).toBe(true)) + + await expect( + nextLeader.requestApplyPersistedTransaction(`todos`, { + txId: `successor-commit`, + mutations: [], + }), + ).resolves.toMatchObject({ ok: true, seq: 2, latestRowVersion: 2 }) + expect(positionReads).toEqual([ + { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 }, + { latestTerm: 1, latestSeq: 1, latestRowVersion: 1 }, + ]) + expect(appliedPositions).toEqual([ + { latestTerm: 1, latestSeq: 1, latestRowVersion: 1 }, + { latestTerm: 2, latestSeq: 2, latestRowVersion: 2 }, + ]) + } finally { + releaseApply.resolve() + await outgoingCommit.catch(() => undefined) + firstLeader.dispose() + nextLeader.dispose() + } + }) + it(`retries a transient NOT_LEADER response through the next leader`, async () => { const adapter = createStubAdapter() const indexStarted = deferred() diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 62149fe6f9..64e3b5eea6 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1738,7 +1738,15 @@ class PersistedCollectionRuntime< const tx = this.createPersistedTxFromOperations(transaction, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + const application = await this.persistence.adapter.applyCommittedTx( + this.collectionId, + tx, + ) + if (application?.applied === false) { + throw new Error( + `persistence position collision: transaction ${tx.term}:${tx.seq} was not applied; canonical position is ${application.term}:${application.seq}`, + ) + } this.publishTxCommittedEvent( this.createTxCommittedPayload({ term: tx.term, @@ -1950,7 +1958,15 @@ class PersistedCollectionRuntime< // SingleProcessCoordinator). Apply directly and broadcast. const streamPosition = this.nextLocalStreamPosition() const tx = this.createPersistedTxFromMutations(mutations, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + const application = await this.persistence.adapter.applyCommittedTx( + this.collectionId, + tx, + ) + if (application?.applied === false) { + throw new Error( + `persistence position collision: transaction ${tx.term}:${tx.seq} was not applied; canonical position is ${application.term}:${application.seq}`, + ) + } this.publishTxCommittedEvent( this.createTxCommittedPayload({ @@ -2978,14 +2994,18 @@ function createWrappedSyncConfig< } } - if ( - openTransaction.queuedBecauseHydrating && - (!runtime.isHydratingNow() || - (openTransaction.truncate && runtime.supersedeHydration())) - ) { - openTransaction.queuedBecauseHydrating = false - openTransaction.supersededHydration = true - forwardBufferedTransaction() + if (openTransaction.queuedBecauseHydrating) { + if (!runtime.isHydratingNow()) { + openTransaction.queuedBecauseHydrating = false + forwardBufferedTransaction() + } else if ( + openTransaction.truncate && + runtime.supersedeHydration() + ) { + openTransaction.queuedBecauseHydrating = false + openTransaction.supersededHydration = true + forwardBufferedTransaction() + } } if (openTransaction.queuedBecauseHydrating) { diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index ce4de4a066..6c85fee2c3 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -2150,6 +2150,10 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { throw error } } + await this.driver.exec( + `CREATE INDEX IF NOT EXISTS applied_tx_collection_tx_id_idx + ON applied_tx (collection_id, tx_id)`, + ) await this.driver.exec( `CREATE TABLE IF NOT EXISTS collection_version ( collection_id TEXT PRIMARY KEY, diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index a5d639972e..3fbae3f461 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -374,6 +374,36 @@ describe(`persistedCollectionOptions`, () => { expect(collection.utils.getLeadershipState?.().isLeader).toBe(true) }) + it(`rejects a local fallback mutation when SQLite reports a position collision`, async () => { + const adapter = createRecordingAdapter() + adapter.applyCommittedTx = (_collectionId, tx) => + Promise.resolve({ + applied: false, + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + const collection = createCollection( + persistedCollectionOptions({ + id: `local-fallback-position-collision`, + getKey: (item) => item.id, + persistence: { adapter }, + }), + ) + + try { + const tx = collection.insert({ + id: `not-durable`, + title: `Must not be acknowledged`, + }) + + await expect(tx.isPersisted.promise).rejects.toThrow(/position collision/) + expect(collection.get(`not-durable`)).toBeUndefined() + } finally { + await collection.cleanup() + } + }) + it(`supports acceptMutations for manual transactions`, async () => { const adapter = createRecordingAdapter() const collection = createCollection( @@ -1150,6 +1180,55 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`rejects a wrapped sync receipt when SQLite reports a position collision`, async () => { + const adapter = createRecordingAdapter() + adapter.applyCommittedTx = (_collectionId, tx) => + Promise.resolve({ + applied: false, + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-fallback-position-collision`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + + try { + await collection.stateWhenReady() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `not-durable`, title: `Must not be acknowledged` }, + }) + + await expect(Promise.resolve(remoteCommit?.())).rejects.toThrow( + /position collision/, + ) + } finally { + await collection.cleanup() + } + }) + it(`handles a dropped sync receipt when persistence fails`, async () => { const adapter = createRecordingAdapter() const persistenceError = new Error(`durable write failed`) @@ -1569,6 +1648,82 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`keeps on-demand readiness upstream-gated when a non-truncate transaction spans hydration`, async () => { + const adapter = createRecordingAdapter() + const hydrationStarted = deferred() + const hydration = deferred>() + adapter.loadSubset = () => { + hydrationStarted.resolve() + return hydration.promise + } + const upstreamStarted = deferred() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => SyncAppliedReceipt) | undefined + let markUpstreamReady: (() => void) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `on-demand-commit-after-hydration`, + syncMode: `on-demand`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markUpstreamReady = markReady + upstreamStarted.resolve() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + + try { + collection.startSyncImmediate() + await upstreamStarted.promise + + const subsetLoad = collection._sync.loadSubset({}) + await hydrationStarted.promise + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `late-commit`, title: `Committed after hydration` }, + }) + + hydration.resolve([]) + await subsetLoad + expect(collection.status).toBe(`loading`) + + const applied = remoteCommit?.() + if (applied !== true) await applied + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`late-commit`))).toEqual({ + id: `late-commit`, + title: `Committed after hydration`, + }) + expect(adapter.rows.get(`late-commit`)).toEqual({ + id: `late-commit`, + title: `Committed after hydration`, + }) + expect(collection.status).toBe(`loading`) + + markUpstreamReady?.() + await collection.stateWhenReady() + expect(collection.status).toBe(`ready`) + } finally { + hydration.resolve([]) + await collection.cleanup() + } + }) + it(`discards a hydration-buffered transaction aborted before replay`, async () => { const adapter = createRecordingAdapter() let resolveLoadSubset: (() => void) | undefined diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index b0d77d47f0..8e0c245015 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -98,6 +98,9 @@ class SqliteCliDriver implements SQLiteDriver { if (!trimmedOutput) { return [] } + if (/^EXPLAIN\s+QUERY\s+PLAN\b/i.test(renderedSql.trim())) { + return [{ detail: trimmedOutput }] as Array + } return JSON.parse(trimmedOutput) as Array } @@ -412,6 +415,65 @@ export function runSQLiteCoreAdapterContractSuite( expect(txRows[0]?.count).toBe(1) }) + it(`migrates a composite transaction-id lookup index for unbounded histories`, async () => { + const { adapter, driver } = registerContractHarness({ + appliedTxPruneMaxRows: 0, + appliedTxPruneMaxAgeSeconds: 0, + }) + const collectionId = `transaction-id-query-plan` + + // Simulate a database created before replay columns and the transaction-id + // lookup index existed. Initialization must migrate this table in place. + await driver.exec( + `CREATE TABLE applied_tx ( + collection_id TEXT NOT NULL, + term INTEGER NOT NULL, + seq INTEGER NOT NULL, + tx_id TEXT NOT NULL, + row_version INTEGER NOT NULL, + applied_at INTEGER NOT NULL, + PRIMARY KEY (collection_id, term, seq) + )`, + ) + + for (let seq = 1; seq <= 64; seq++) { + await adapter.applyCommittedTx(collectionId, { + txId: `history-${seq}`, + term: 1, + seq, + rowVersion: seq, + mutations: [], + }) + } + + const retained = await driver.query<{ count: number }>( + `SELECT COUNT(*) AS count + FROM applied_tx + WHERE collection_id = ?`, + [collectionId], + ) + expect(retained[0]?.count).toBe(64) + + const indexes = await driver.query<{ name: string }>( + `PRAGMA index_list('applied_tx')`, + ) + expect(indexes.map((index) => index.name)).toContain( + `applied_tx_collection_tx_id_idx`, + ) + + const queryPlan = await driver.query<{ detail: string }>( + `EXPLAIN QUERY PLAN + SELECT term, seq, row_version + FROM applied_tx + WHERE collection_id = ? AND tx_id = ? + LIMIT 1`, + [collectionId, `history-64`], + ) + expect(queryPlan.map((row) => row.detail).join(`\n`)).toContain( + `applied_tx_collection_tx_id_idx (collection_id=? AND tx_id=?)`, + ) + }) + it(`obeys stable transaction identity across generated leader positions`, async () => { await fc.assert( fc.asyncProperty( diff --git a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts index 4040eca791..5eb8d52154 100644 --- a/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts +++ b/packages/electron-db-sqlite-persistence/src/electron-coordinator.ts @@ -660,7 +660,7 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin throw new Error(`not the leader for ${collectionId}`) } - const proposedTx = { + let proposedTx = { txId: safeRandomUUID(), term: currentState.latestTerm, seq: currentState.latestSeq + 1, @@ -672,28 +672,50 @@ export class ElectronCollectionCoordinator implements PersistedCollectionCoordin })), } - const application = await this.requireAdapter().applyCommittedTx( - collectionId, - proposedTx, - ) - const appliedTx = application - ? { - ...proposedTx, - term: application.term, - seq: application.seq, - rowVersion: application.rowVersion, - } - : proposedTx - currentState.latestTerm = Math.max( - currentState.latestTerm, - appliedTx.term, - ) - currentState.latestSeq = Math.max(currentState.latestSeq, appliedTx.seq) - currentState.latestRowVersion = Math.max( - currentState.latestRowVersion, - appliedTx.rowVersion, - ) - return appliedTx + for (let attempt = 0; attempt < 2; attempt++) { + const application = await this.requireAdapter().applyCommittedTx( + collectionId, + proposedTx, + ) + const canonicalTx = application + ? { + ...proposedTx, + term: application.term, + seq: application.seq, + rowVersion: application.rowVersion, + } + : proposedTx + currentState.latestTerm = Math.max( + currentState.latestTerm, + canonicalTx.term, + ) + currentState.latestSeq = Math.max( + currentState.latestSeq, + canonicalTx.seq, + ) + currentState.latestRowVersion = Math.max( + currentState.latestRowVersion, + canonicalTx.rowVersion, + ) + + if (!application || application.applied) { + return canonicalTx + } + if (attempt === 1) { + throw new Error( + `persistence position collision: transaction ${proposedTx.term}:${proposedTx.seq} was not applied after retry`, + ) + } + + proposedTx = { + ...proposedTx, + term: currentState.latestTerm, + seq: currentState.latestSeq + 1, + rowVersion: currentState.latestRowVersion + 1, + } + } + + throw new Error(`persistence position collision retry was not completed`) }) const { term, seq, rowVersion } = tx diff --git a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts index 4465559660..7030b971c0 100644 --- a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts +++ b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts @@ -113,6 +113,100 @@ describe(`ElectronCollectionCoordinator`, () => { coordinator.dispose() }) + it(`retries one position collision and acknowledges only the durable retry`, async () => { + const adapter = createAdapter() + const attemptedPositions: Array<{ + term: number + seq: number + rowVersion: number + }> = [] + adapter.applyCommittedTx = (_collectionId, tx) => { + attemptedPositions.push({ + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + if (attemptedPositions.length === 1) { + return Promise.resolve({ + applied: false, + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + } + return Promise.resolve({ + applied: true, + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + } + const coordinator = new ElectronCollectionCoordinator({ + dbName: `position-collision-law`, + adapter, + }) + const committed: Array = [] + coordinator.subscribe(`todos`, (envelope) => committed.push(envelope)) + await vi.waitFor(() => expect(coordinator.isLeader(`todos`)).toBe(true)) + + const response = await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `collision-retry`, + type: `insert`, + key: `1`, + value: { id: `1`, title: `Apply after collision` }, + }, + ]) + + expect(attemptedPositions).toEqual([ + { term: 1, seq: 1, rowVersion: 1 }, + { term: 1, seq: 2, rowVersion: 2 }, + ]) + expect(response).toMatchObject({ + ok: true, + term: 1, + seq: 2, + latestRowVersion: 2, + }) + expect(committed).toHaveLength(1) + coordinator.dispose() + }) + + it(`rejects a second position collision without publishing success`, async () => { + const adapter = createAdapter() + let applyCalls = 0 + adapter.applyCommittedTx = (_collectionId, tx) => { + applyCalls++ + return Promise.resolve({ + applied: false, + term: tx.term, + seq: tx.seq, + rowVersion: tx.rowVersion, + }) + } + const coordinator = new ElectronCollectionCoordinator({ + dbName: `persistent-position-collision-law`, + adapter, + }) + const committed: Array = [] + coordinator.subscribe(`todos`, (envelope) => committed.push(envelope)) + await vi.waitFor(() => expect(coordinator.isLeader(`todos`)).toBe(true)) + + await expect( + coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `persistent-collision`, + type: `insert`, + key: `1`, + value: { id: `1`, title: `Never applied` }, + }, + ]), + ).rejects.toThrow(/position collision/) + expect(applyCalls).toBe(2) + expect(committed).toEqual([]) + coordinator.dispose() + }) + it(`obeys protected-work failure across generated callback boundaries`, async () => { await fc.assert( fc.asyncProperty(