From 65feff5d0f23061c8a71f887d17dd17895053c65 Mon Sep 17 00:00:00 2001 From: "James E. Kemp" Date: Mon, 26 Oct 2026 00:27:31 -0700 Subject: [PATCH 1/5] feat(turso): self-heal a stale WAL index on open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A turso store whose `-tshm` outlived its `-wal` cannot be opened: either "I/O error: short read on WAL frame at offset N" or "Corrupt database: Invalid page type: 0". The main `.db` stays intact in both cases, so the store is recoverable by discarding the sidecars — but nothing in the plugin does that, so the whole memory subsystem stays down until someone intervenes by hand (issue #435). On either signature, verify the main database on a COPY (opening the live file with another engine is itself a corruption source), move `-shm`, `-tshm` and `-wal` into `/recovery/-/` — moved, never deleted — and retry the open once. Unhealthy copy or failed retry leaves the store untouched and re-raises the original error. One attempt per path per process; no loop. Also warn once per path when a foreign SQLite `-shm` appears beside a Turso `-tshm`: measured, read-only access is harmless, but a foreign writer diverges silently and loses its own next write. Config: `tursoAutoHeal` (default true, only fires when the store is already unopenable) and `tursoMultiprocessWal` (default true; false makes foreign access fail loudly with "database is locked" instead of diverging). --- opencode-mem.example.jsonc | 11 ++ src/config.ts | 27 +++ src/services/turso/connection-manager.ts | 84 +++++++- src/services/turso/wal-index-recovery.ts | 110 +++++++++++ tests/turso-wal-index-recovery.test.ts | 236 +++++++++++++++++++++++ 5 files changed, 464 insertions(+), 4 deletions(-) create mode 100644 src/services/turso/wal-index-recovery.ts create mode 100644 tests/turso-wal-index-recovery.test.ts diff --git a/opencode-mem.example.jsonc b/opencode-mem.example.jsonc index 8a1a9a3b..ebdb3bd8 100644 --- a/opencode-mem.example.jsonc +++ b/opencode-mem.example.jsonc @@ -346,6 +346,17 @@ // Pinned versions like opencode-mem@2.26.0 are never auto-updated. "autoUpdate": true, + // Recover a store whose Turso WAL index outlived its WAL (a stale -tshm/-shm + // beside a 0-byte -wal). The guard copies the main .db, verifies it with + // PRAGMA integrity_check, moves -wal/-shm/-tshm to /recovery/, + // then retries the open once. Files are moved, never deleted. + "tursoAutoHeal": true, + + // Turso experimental multiprocess_wal (Unix). Set false for single-owner mode: + // a second process then fails loudly with "database is locked" instead of + // writing into a divergent WAL. See issue #435. + "tursoMultiprocessWal": true, + // ============================================ // User Profile System // ============================================ diff --git a/src/config.ts b/src/config.ts index bbb32d85..261af5cc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -165,6 +165,18 @@ interface OpenCodeMemConfig { | "aegis128x4"; /** Automatically clear OpenCode's cached plugin install when a newer npm latest is available. */ autoUpdate?: boolean; + /** + * Self-heal a store whose Turso WAL index outlived its WAL (`-tshm`/`-shm` + * beside a 0-byte `-wal`): verify a copy of the main `.db`, move the sidecars + * to `/recovery/`, then retry once. Default true. + */ + tursoAutoHeal?: boolean; + /** + * Open with Turso's experimental `multiprocess_wal` (Unix). Set false for + * single-owner mode, so a foreign process fails loudly with "database is + * locked" instead of writing into a divergent WAL. Default true. + */ + tursoMultiprocessWal?: boolean; compaction?: { enabled?: boolean; memoryLimit?: number; @@ -302,6 +314,8 @@ const DEFAULTS: Required< databaseEncryptionEnabled: false, databaseEncryptionCipher: "aes256gcm", autoUpdate: true, + tursoAutoHeal: true, + tursoMultiprocessWal: true, memory: { defaultScope: "project", }, @@ -681,6 +695,17 @@ const CONFIG_TEMPLATE = `{ // Pinned versions like opencode-mem@2.26.0 are never auto-updated. "autoUpdate": true, + // Recover a store whose Turso WAL index outlived its WAL (a stale -tshm/-shm + // beside a 0-byte -wal). The guard copies the main .db, verifies it with + // PRAGMA integrity_check, moves -wal/-shm/-tshm to /recovery/, + // then retries the open once. Files are moved, never deleted. + "tursoAutoHeal": true, + + // Turso experimental multiprocess_wal (Unix). Set false for single-owner mode: + // a second process then fails loudly with "database is locked" instead of + // writing into a divergent WAL. See issue #435. + "tursoMultiprocessWal": true, + // ============================================ // User Profile System // ============================================ @@ -1197,6 +1222,8 @@ function buildConfig(fileConfig: OpenCodeMemConfig) { databaseEncryptionCipher: fileConfig.databaseEncryptionCipher ?? DEFAULTS.databaseEncryptionCipher, autoUpdate: fileConfig.autoUpdate ?? DEFAULTS.autoUpdate, + tursoAutoHeal: fileConfig.tursoAutoHeal ?? DEFAULTS.tursoAutoHeal, + tursoMultiprocessWal: fileConfig.tursoMultiprocessWal ?? DEFAULTS.tursoMultiprocessWal, memory: { defaultScope: fileConfig.memory?.defaultScope ?? DEFAULTS.memory.defaultScope, }, diff --git a/src/services/turso/connection-manager.ts b/src/services/turso/connection-manager.ts index 21909bb4..cf1f48df 100644 --- a/src/services/turso/connection-manager.ts +++ b/src/services/turso/connection-manager.ts @@ -7,6 +7,7 @@ import { log } from "../logger.js"; import { collectReleasedSqliteHandles } from "./sqlite-handle-release.js"; import { TursoDb } from "./turso-db.js"; import { resolveOrCreateDatabaseEncryptionKey } from "./encryption-key.js"; +import { isWalIndexCorruptionError, quarantineWalSidecars } from "./wal-index-recovery.js"; export type ConnectFactory = (path: string, opts?: DatabaseOpts) => Promise; @@ -23,11 +24,16 @@ export function supportsTursoMultiprocessWal( return platform !== "win32"; } -/** Experimental flags required on every Turso open for the current platform. */ +/** + * Experimental flags required on every Turso open for the current platform. + * `multiprocessWal: false` opts into single-owner mode (Unix), so a foreign + * process gets a loud "database is locked" instead of silent divergence. + */ export function tursoExperimentalFeatures( - platform: NodeJS.Platform = process.platform + platform: NodeJS.Platform = process.platform, + multiprocessWal: boolean = CONFIG.tursoMultiprocessWal ): Array<(typeof TURSO_BASE_EXPERIMENTAL_FEATURES)[number] | "multiprocess_wal"> { - if (supportsTursoMultiprocessWal(platform)) { + if (multiprocessWal && supportsTursoMultiprocessWal(platform)) { return [...TURSO_BASE_EXPERIMENTAL_FEATURES, "multiprocess_wal"]; } return [...TURSO_BASE_EXPERIMENTAL_FEATURES]; @@ -94,9 +100,78 @@ export class TursoConnectionManager { private readonly pending = new Map>(); private readonly closingConnections = new Map>(); private closingPromise: Promise | null = null; + /** One WAL-index recovery attempt per path per process — never loop. */ + private readonly healedPaths = new Set(); + private readonly warnedForeignShmPaths = new Set(); constructor(private readonly connectFactory: ConnectFactory = connect) {} + /** + * Connect, and on a stale-WAL-index failure quarantine the sidecars once and + * retry once. Only the two known signatures heal (see #435); any other open + * error propagates unchanged. + */ + private async connectWithWalIndexRecovery(dbPath: string, opts: DatabaseOpts): Promise { + try { + return await this.connectFactory(dbPath, opts); + } catch (error) { + if (!CONFIG.tursoAutoHeal || !isWalIndexCorruptionError(error)) { + throw error; + } + if (this.healedPaths.has(dbPath)) { + throw error; + } + this.healedPaths.add(dbPath); + + const outcome = await quarantineWalSidecars(dbPath, this.connectFactory, opts); + if (outcome.status !== "quarantined") { + log("Turso open failed and the store was left untouched (no recovery attempted)", { + path: dbPath, + reason: outcome.reason, + hint: "Restore this database from a backup; the sidecar files were not moved.", + error: String(error), + }); + throw error; + } + + log("Moved stale WAL sidecars aside after a Turso open failure; retrying once", { + path: dbPath, + quarantineDir: outcome.quarantineDir, + moved: outcome.moved, + }); + + try { + return await this.connectFactory(dbPath, opts); + } catch (retryError) { + log("Turso open still failing after WAL sidecar recovery", { + path: dbPath, + quarantineDir: outcome.quarantineDir, + error: String(retryError), + }); + throw error; + } + } + } + + /** + * A foreign sqlite process that wrote this store leaves `-shm` next to a + * Turso `-tshm`. Read-only access is harmless; a foreign writer diverges + * silently and loses its own next write, so warn once per path. Never delete + * `-shm` here — it may belong to a live reader. + */ + private warnOnForeignSqliteShm(dbPath: string): void { + if (this.warnedForeignShmPaths.has(dbPath)) return; + if (!existsSync(`${dbPath}-shm`) || !existsSync(`${dbPath}-tshm`)) return; + this.warnedForeignShmPaths.add(dbPath); + log("Foreign SQLite process touched this store (-shm beside -tshm)", { + path: dbPath, + hint: + "Read-only access is harmless, but a foreign writer silently loses data — " + + "see issue #435. Close other tools on this store, or set tursoMultiprocessWal=false " + + "for single-owner mode.", + }); + } + async getConnection(dbPath: string): Promise { if (this.closingPromise) { await this.closingPromise; @@ -127,9 +202,10 @@ export class TursoConnectionManager { const opts = buildConnectOptions(encryption); let database: Database | null = null; try { - database = await this.connectFactory(dbPath, opts); + database = await this.connectWithWalIndexRecovery(dbPath, opts); const db = new TursoDb(database); await db.execute("PRAGMA foreign_keys = ON"); + this.warnOnForeignSqliteShm(dbPath); this.connections.set(dbPath, db); return db; } catch (error) { diff --git a/src/services/turso/wal-index-recovery.ts b/src/services/turso/wal-index-recovery.ts new file mode 100644 index 00000000..454a3408 --- /dev/null +++ b/src/services/turso/wal-index-recovery.ts @@ -0,0 +1,110 @@ +import type { Database } from "@tursodatabase/database"; +import type { DatabaseOpts } from "@tursodatabase/database-common"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, renameSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, extname, join } from "node:path"; +import { CONFIG } from "../../config.js"; +import { log } from "../logger.js"; +import type { ConnectFactory } from "./connection-manager.js"; + +/** + * Turso keeps its multiprocess WAL index in `-tshm`; stock sqlite uses + * `-shm`. A `-tshm` (or `-shm`) that outlives the WAL it indexes makes the + * next open fail with one of these two errors (see issue #435): + * - "I/O error: short read on WAL frame at offset N: expected 4096 bytes, got 0" + * - "Corrupt database: Invalid page type: 0" + * The main `.db` itself stays intact, so the store is recoverable by discarding + * the sidecars. + */ +const WAL_INDEX_CORRUPTION_PATTERNS = [ + /short read on WAL frame at offset \d+/i, + /invalid page type:\s*0\b/i, +]; + +export const WAL_SIDECAR_SUFFIXES = ["-shm", "-tshm", "-wal"] as const; + +/** True for the two "stale WAL index" open failures that self-heal can repair. */ +export function isWalIndexCorruptionError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return WAL_INDEX_CORRUPTION_PATTERNS.some((pattern) => pattern.test(message)); +} + +export type WalIndexRecoveryResult = + | { status: "quarantined"; quarantineDir: string; moved: string[] } + | { status: "skipped"; reason: string }; + +function quarantineDirFor(dbPath: string): string { + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const stem = basename(dbPath, extname(dbPath)); + return join(CONFIG.storagePath, "recovery", `${stem}-${stamp}`); +} + +/** + * Verify a copy of the main database file. The live file is never opened with a + * different engine — that is what produces the "Invalid page type: 0" class. + */ +async function isCopiedDatabaseHealthy( + dbPath: string, + connectFactory: ConnectFactory, + opts: DatabaseOpts +): Promise { + const tempDir = mkdtempSync(join(tmpdir(), "opencode-mem-wal-check-")); + try { + const copyPath = join(tempDir, basename(dbPath)); + copyFileSync(dbPath, copyPath); + let database: Database | null = null; + try { + database = await connectFactory(copyPath, opts); + const rows: unknown = await database.all("PRAGMA integrity_check"); + const first = Array.isArray(rows) ? (rows[0] as Record | undefined) : null; + const value = first ? Object.values(first)[0] : undefined; + return String(value ?? "").toLowerCase() === "ok"; + } finally { + if (database) { + try { + await database.close(); + } catch { + // ignore close errors on a throwaway copy + } + } + } + } catch (error) { + log("Turso WAL index recovery: could not verify a copy of the database", { + path: dbPath, + error: String(error), + }); + return false; + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +/** + * Move the WAL sidecars of an unopenable store into + * `/recovery/-/` so the next open starts clean. + * Sidecars are moved, never deleted, and only after a copy of the main `.db` + * passes `PRAGMA integrity_check`. + */ +export async function quarantineWalSidecars( + dbPath: string, + connectFactory: ConnectFactory, + opts: DatabaseOpts +): Promise { + if (!existsSync(dbPath)) { + return { status: "skipped", reason: "main database file is missing" }; + } + if (!(await isCopiedDatabaseHealthy(dbPath, connectFactory, opts))) { + return { status: "skipped", reason: "integrity_check failed on a copy of the main database" }; + } + + const quarantineDir = quarantineDirFor(dbPath); + mkdirSync(quarantineDir, { recursive: true }); + const moved: string[] = []; + for (const suffix of WAL_SIDECAR_SUFFIXES) { + const sidecar = `${dbPath}${suffix}`; + if (!existsSync(sidecar)) continue; + renameSync(sidecar, join(quarantineDir, basename(sidecar))); + moved.push(basename(sidecar)); + } + return { status: "quarantined", quarantineDir, moved }; +} diff --git a/tests/turso-wal-index-recovery.test.ts b/tests/turso-wal-index-recovery.test.ts new file mode 100644 index 00000000..ef654c3f --- /dev/null +++ b/tests/turso-wal-index-recovery.test.ts @@ -0,0 +1,236 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { existsSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { cleanupTursoTestDirectory } from "./turso-test-utils.js"; + +const SHORT_READ = "I/O error: short read on WAL frame at offset 4096: expected 4096 bytes, got 0"; +const INVALID_PAGE = "Corrupt database: Invalid page type: 0"; + +function stubDatabase(integrityRows: Array>) { + return { + async exec() {}, + async run() { + return { changes: 0, lastInsertRowid: 0 }; + }, + async get(sql: string) { + if (sql === "PRAGMA integrity_check") return integrityRows[0]; + if (sql === "PRAGMA foreign_keys") return { foreign_keys: 1 }; + return undefined; + }, + async all(sql: string) { + if (sql === "PRAGMA integrity_check") return integrityRows; + if (sql === "PRAGMA foreign_keys") return [{ foreign_keys: 1 }]; + return []; + }, + async batch() { + return []; + }, + transactionAsync() { + return { + immediate: async () => undefined, + deferred: async () => undefined, + }; + }, + async close() {}, + } as any; +} + +describe("turso WAL index recovery", () => { + let baseDir: string; + + afterEach(async () => { + const { CONFIG } = await import("../src/config.js"); + CONFIG.tursoAutoHeal = true; + CONFIG.tursoMultiprocessWal = true; + await cleanupTursoTestDirectory(baseDir); + }); + + async function prepare(prefix: string) { + baseDir = mkdtempSync(join(tmpdir(), prefix)); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.databaseEncryptionEnabled = false; + const dbPath = join(baseDir, "memories.db"); + writeFileSync(dbPath, "main"); + writeFileSync(`${dbPath}-wal`, ""); + writeFileSync(`${dbPath}-shm`, "shm"); + writeFileSync(`${dbPath}-tshm`, "tshm"); + return dbPath; + } + + it("quarantines the sidecars and retries the open exactly once", async () => { + const dbPath = await prepare("turso-heal-ok-"); + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + if (path === dbPath && calls.length === 1) { + throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await manager.getConnection(dbPath); + } finally { + await manager.closeAll(); + } + + expect(calls.filter((path) => path === dbPath)).toHaveLength(2); + expect(calls[0]).toBe(dbPath); + // The verification copy lives in a temp dir, never at the live path. + expect(calls[1]).not.toBe(dbPath); + expect(calls[1]).toContain("memories.db"); + + const recoveryRoot = join(baseDir, "recovery"); + const quarantined = readdirSync(recoveryRoot); + expect(quarantined).toHaveLength(1); + const quarantineDir = join(recoveryRoot, quarantined[0] as string); + expect(readdirSync(quarantineDir).sort()).toEqual([ + "memories.db-shm", + "memories.db-tshm", + "memories.db-wal", + ]); + expect(existsSync(`${dbPath}-wal`)).toBe(false); + expect(existsSync(`${dbPath}-shm`)).toBe(false); + expect(existsSync(`${dbPath}-tshm`)).toBe(false); + expect(existsSync(dbPath)).toBe(true); + }); + + it("heals the invalid-page-type failure too", async () => { + const dbPath = await prepare("turso-heal-page-"); + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + if (path === dbPath && calls.length === 1) { + throw new Error(INVALID_PAGE); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await manager.getConnection(dbPath); + } finally { + await manager.closeAll(); + } + + expect(readdirSync(join(baseDir, "recovery"))).toHaveLength(1); + expect(existsSync(`${dbPath}-tshm`)).toBe(false); + }); + + it("moves nothing when the copy fails integrity_check", async () => { + const dbPath = await prepare("turso-heal-bad-"); + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + if (path === dbPath) { + throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "*** in database main *** page 2 is never used" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/short read on WAL frame/); + } finally { + await manager.closeAll(); + } + + expect(calls.filter((path) => path === dbPath)).toHaveLength(1); + expect(existsSync(join(baseDir, "recovery"))).toBe(false); + expect(existsSync(`${dbPath}-wal`)).toBe(true); + expect(existsSync(`${dbPath}-shm`)).toBe(true); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + }); + + it("surfaces the original error and does not loop when the retry fails", async () => { + const dbPath = await prepare("turso-heal-retry-"); + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + if (path === dbPath) { + throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/short read on WAL frame/); + } finally { + await manager.closeAll(); + } + + // One copy verification plus the single retry — no second heal cycle. + expect(calls.filter((path) => path === dbPath)).toHaveLength(2); + expect(readdirSync(join(baseDir, "recovery"))).toHaveLength(1); + }); + + it("ignores open errors outside the two known signatures", async () => { + const dbPath = await prepare("turso-heal-other-"); + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + throw new Error("disk quota exceeded"); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/disk quota exceeded/); + } finally { + await manager.closeAll(); + } + + expect(calls).toHaveLength(1); + expect(existsSync(join(baseDir, "recovery"))).toBe(false); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + }); + + it("does not heal when tursoAutoHeal is false", async () => { + const dbPath = await prepare("turso-heal-off-"); + const { CONFIG } = await import("../src/config.js"); + CONFIG.tursoAutoHeal = false; + + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + throw new Error(SHORT_READ); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/short read on WAL frame/); + } finally { + await manager.closeAll(); + } + + expect(calls).toHaveLength(1); + expect(existsSync(join(baseDir, "recovery"))).toBe(false); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + }); + + it("drops multiprocess_wal when tursoMultiprocessWal is false", async () => { + const { CONFIG } = await import("../src/config.js"); + const { buildConnectOptions, tursoExperimentalFeatures } = + await import("../src/services/turso/connection-manager.js"); + try { + CONFIG.tursoMultiprocessWal = false; + expect(tursoExperimentalFeatures("linux")).toEqual(["encryption"]); + expect(buildConnectOptions()).toEqual({ experimental: ["encryption"] }); + } finally { + CONFIG.tursoMultiprocessWal = true; + } + + expect(tursoExperimentalFeatures("linux")).toEqual(["encryption", "multiprocess_wal"]); + expect(tursoExperimentalFeatures("linux", false)).toEqual(["encryption"]); + expect(tursoExperimentalFeatures("linux", true)).toEqual(["encryption", "multiprocess_wal"]); + }); +}); From 344e01f3a9cf9e0adec6e62e9686c2bd29eb16ae Mon Sep 17 00:00:00 2001 From: "James E. Kemp" Date: Mon, 26 Oct 2026 00:29:06 -0700 Subject: [PATCH 2/5] docs(readme): document tursoMultiprocessWal and tursoAutoHeal Both knobs change observable behaviour on shared storage: single-owner mode turns silent divergence from a foreign writer into a loud 'database is locked', and auto-heal is what recovers a stale WAL index on open. English README only; translations are maintained separately. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 26e4498c..d72e3841 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to On macOS and Linux, multiple OpenCode sessions can share the same `storagePath` via Turso’s experimental `multiprocess_wal` (every process must use the same mode — restart all sessions after upgrading). On Windows the engine rejects that flag, so only one OpenCode session can own the memory databases at a time. +`tursoMultiprocessWal: false` opens in single-owner mode instead: other processes then fail with `database is locked` rather than sharing the store. That is the safer choice if non-plugin tools touch the same files, because a *writing* foreign SQLite process makes Turso’s WAL index stale and its own next write is silently lost. `tursoAutoHeal` (default `true`) moves a store’s `-shm`/`-tshm`/`-wal` sidecars into `storagePath/recovery/-/` and retries once when an open fails with a stale WAL index, after verifying a copy of the main database. + On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format: - Each shard is backed up as `.db.legacy.bak` before rewrite From e68518e49c08fe5ecd930fed81724d302838cc47 Mon Sep 17 00:00:00 2001 From: "James E. Kemp" Date: Mon, 26 Oct 2026 00:29:31 -0700 Subject: [PATCH 3/5] style(readme): use prettier emphasis form --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d72e3841..e8f463c2 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to On macOS and Linux, multiple OpenCode sessions can share the same `storagePath` via Turso’s experimental `multiprocess_wal` (every process must use the same mode — restart all sessions after upgrading). On Windows the engine rejects that flag, so only one OpenCode session can own the memory databases at a time. -`tursoMultiprocessWal: false` opens in single-owner mode instead: other processes then fail with `database is locked` rather than sharing the store. That is the safer choice if non-plugin tools touch the same files, because a *writing* foreign SQLite process makes Turso’s WAL index stale and its own next write is silently lost. `tursoAutoHeal` (default `true`) moves a store’s `-shm`/`-tshm`/`-wal` sidecars into `storagePath/recovery/-/` and retries once when an open fails with a stale WAL index, after verifying a copy of the main database. +`tursoMultiprocessWal: false` opens in single-owner mode instead: other processes then fail with `database is locked` rather than sharing the store. That is the safer choice if non-plugin tools touch the same files, because a _writing_ foreign SQLite process makes Turso’s WAL index stale and its own next write is silently lost. `tursoAutoHeal` (default `true`) moves a store’s `-shm`/`-tshm`/`-wal` sidecars into `storagePath/recovery/-/` and retries once when an open fails with a stale WAL index, after verifying a copy of the main database. On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format: From 10495179ce1b0e047550b717144f4ff0d3017122 Mon Sep 17 00:00:00 2001 From: "James E. Kemp" Date: Mon, 26 Oct 2026 08:57:02 -0700 Subject: [PATCH 4/5] fix(turso): preserve non-empty WALs and serialise recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on #441. Data safety: recovery now quarantines only the index sidecars (-shm, -tshm). A -wal is removed solely when it is absent or 0 bytes; a non-empty WAL is left in place, because committed transactions can live there while the main .db still passes integrity_check. Deleting it would silently roll the store back to an older snapshot. Measured on this engine: after discarding -tshm and keeping a 16,512-byte WAL, reopen returns both committed rows — the index is rebuilt from the surviving WAL. A torn WAL is the documented "retry fails, original error surfaces, WAL untouched" case. Cross-process: healing is serialised with /recovery/.lock, created O_EXCL. A second process that cannot take the lock leaves the store alone instead of moving files a live owner is writing to; a lock older than five minutes is treated as a crashed healer and taken over. Under the lock the open is re-attempted before anything moves, so a peer that healed first leaves nothing to do. Error reporting: lock acquisition, verification, copy, quarantine and cleanup failures are caught and logged with the database path, the quarantine state and the original error, then the original open error is rethrown — a filesystem problem can no longer mask the corruption or skip its diagnostics. Lock release runs in finally and cannot throw past the original error. --- README.md | 2 +- src/config.ts | 17 +- src/services/turso/connection-manager.ts | 134 +++++++++-- src/services/turso/wal-index-recovery.ts | 242 ++++++++++++++++--- tests/turso-wal-index-recovery.test.ts | 294 ++++++++++++++++++++--- 5 files changed, 603 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index e8f463c2..c848975b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to On macOS and Linux, multiple OpenCode sessions can share the same `storagePath` via Turso’s experimental `multiprocess_wal` (every process must use the same mode — restart all sessions after upgrading). On Windows the engine rejects that flag, so only one OpenCode session can own the memory databases at a time. -`tursoMultiprocessWal: false` opens in single-owner mode instead: other processes then fail with `database is locked` rather than sharing the store. That is the safer choice if non-plugin tools touch the same files, because a _writing_ foreign SQLite process makes Turso’s WAL index stale and its own next write is silently lost. `tursoAutoHeal` (default `true`) moves a store’s `-shm`/`-tshm`/`-wal` sidecars into `storagePath/recovery/-/` and retries once when an open fails with a stale WAL index, after verifying a copy of the main database. +`tursoMultiprocessWal: false` opens in single-owner mode instead: other processes then fail with `database is locked` rather than sharing the store. That is the safer choice if non-plugin tools touch the same files, because a _writing_ foreign SQLite process makes Turso’s WAL index stale and its own next write is silently lost. `tursoAutoHeal` (default `true`) moves a store’s `-shm`/`-tshm` index sidecars into `storagePath/recovery/-/` and retries once when an open fails with a stale WAL index, after verifying a copy of the main database. A non-empty `-wal` is never moved, so committed but uncheckpointed transactions survive; only an absent or 0-byte `-wal` is removed. Concurrent sessions serialise healing through `storagePath/recovery/.lock`. On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format: diff --git a/src/config.ts b/src/config.ts index 261af5cc..b52e217c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -166,9 +166,12 @@ interface OpenCodeMemConfig { /** Automatically clear OpenCode's cached plugin install when a newer npm latest is available. */ autoUpdate?: boolean; /** - * Self-heal a store whose Turso WAL index outlived its WAL (`-tshm`/`-shm` - * beside a 0-byte `-wal`): verify a copy of the main `.db`, move the sidecars - * to `/recovery/`, then retry once. Default true. + * Self-heal a store whose Turso WAL index is stale (a leftover `-tshm`/`-shm`, + * often after a foreign writer or beside a 0-byte `-wal`): verify a copy of + * the main `.db`, move the *index* sidecars to `/recovery/`, then + * retry once. A non-empty `-wal` is left in place so the engine can rebuild + * its index from it. Recovery is serialised across processes with + * `/recovery/.lock`. Default true. */ tursoAutoHeal?: boolean; /** @@ -695,10 +698,12 @@ const CONFIG_TEMPLATE = `{ // Pinned versions like opencode-mem@2.26.0 are never auto-updated. "autoUpdate": true, - // Recover a store whose Turso WAL index outlived its WAL (a stale -tshm/-shm + // Recover a store whose Turso WAL index is stale (a leftover -tshm/-shm, often // beside a 0-byte -wal). The guard copies the main .db, verifies it with - // PRAGMA integrity_check, moves -wal/-shm/-tshm to /recovery/, - // then retries the open once. Files are moved, never deleted. + // PRAGMA integrity_check, moves the -shm/-tshm index sidecars to + // /recovery/, and retries the open once. A non-empty -wal is + // never moved, so committed but uncheckpointed transactions are preserved. + // Recovery is serialised across processes by /recovery/.lock. "tursoAutoHeal": true, // Turso experimental multiprocess_wal (Unix). Set false for single-owner mode: diff --git a/src/services/turso/connection-manager.ts b/src/services/turso/connection-manager.ts index cf1f48df..d3ae4b88 100644 --- a/src/services/turso/connection-manager.ts +++ b/src/services/turso/connection-manager.ts @@ -7,7 +7,11 @@ import { log } from "../logger.js"; import { collectReleasedSqliteHandles } from "./sqlite-handle-release.js"; import { TursoDb } from "./turso-db.js"; import { resolveOrCreateDatabaseEncryptionKey } from "./encryption-key.js"; -import { isWalIndexCorruptionError, quarantineWalSidecars } from "./wal-index-recovery.js"; +import { + isWalIndexCorruptionError, + quarantineWalSidecars, + acquireRecoveryLock, +} from "./wal-index-recovery.js"; export type ConnectFactory = (path: string, opts?: DatabaseOpts) => Promise; @@ -107,9 +111,15 @@ export class TursoConnectionManager { constructor(private readonly connectFactory: ConnectFactory = connect) {} /** - * Connect, and on a stale-WAL-index failure quarantine the sidecars once and - * retry once. Only the two known signatures heal (see #435); any other open + * Connect, and on a stale-WAL-index failure quarantine the index sidecars once + * and retry once. Only the two known signatures heal (see #435); any other open * error propagates unchanged. + * + * Recovery is serialised across processes (they share `storagePath` through + * `multiprocess_wal`) with `/recovery/.lock`. Under that + * lock the open is re-attempted before any file is moved, so a peer that + * healed first leaves us nothing to do. A non-empty WAL is never moved — see + * `wal-index-recovery.ts` for the data-safety contract. */ private async connectWithWalIndexRecovery(dbPath: string, opts: DatabaseOpts): Promise { try { @@ -121,34 +131,120 @@ export class TursoConnectionManager { if (this.healedPaths.has(dbPath)) { throw error; } - this.healedPaths.add(dbPath); - const outcome = await quarantineWalSidecars(dbPath, this.connectFactory, opts); - if (outcome.status !== "quarantined") { - log("Turso open failed and the store was left untouched (no recovery attempted)", { + let lock: ReturnType; + try { + lock = acquireRecoveryLock(dbPath); + } catch (lockError) { + log("Turso WAL index recovery skipped: could not take the recovery lock", { path: dbPath, - reason: outcome.reason, - hint: "Restore this database from a backup; the sidecar files were not moved.", + error: String(lockError), + originalError: String(error), + }); + throw error; + } + if (!lock) { + log("Turso WAL index recovery skipped: another process owns the recovery lock", { + path: dbPath, + hint: "The owning process is healing this store; leaving its files untouched.", error: String(error), }); throw error; } + if (lock.tookOverStaleLock) { + log("Took over a stale Turso WAL index recovery lock", { + path: dbPath, + lock: lock.lockPath, + }); + } - log("Moved stale WAL sidecars aside after a Turso open failure; retrying once", { - path: dbPath, - quarantineDir: outcome.quarantineDir, - moved: outcome.moved, - }); + this.healedPaths.add(dbPath); try { - return await this.connectFactory(dbPath, opts); - } catch (retryError) { - log("Turso open still failing after WAL sidecar recovery", { + // The peer may have healed between our failed open and taking the lock. + try { + const connection = await this.connectFactory(dbPath, opts); + log("Turso open succeeded on the recovery-lock recheck; nothing was moved", { + path: dbPath, + }); + return connection; + } catch (recheckError) { + if (!isWalIndexCorruptionError(recheckError)) { + // e.g. a lock error: do not move a live owner's files. + log( + "Turso recovery-lock recheck failed with an unrelated error; store left untouched", + { + path: dbPath, + error: String(recheckError), + originalError: String(error), + } + ); + throw error; + } + } + + const outcome = await quarantineWalSidecars(dbPath, this.connectFactory, opts); + if (outcome.status === "failed") { + log("Turso WAL index recovery failed; the store was left as-is where possible", { + path: dbPath, + reason: outcome.reason, + quarantineDir: outcome.quarantineDir, + moved: outcome.moved, + error: String(error), + }); + throw error; + } + if (outcome.status === "skipped") { + log("Turso open failed and the store was left untouched (no recovery attempted)", { + path: dbPath, + reason: outcome.reason, + hint: "Restore this database from a backup; the sidecar files were not moved.", + error: String(error), + }); + throw error; + } + + log("Moved stale WAL index sidecars aside after a Turso open failure; retrying once", { path: dbPath, quarantineDir: outcome.quarantineDir, - error: String(retryError), + moved: outcome.moved, + removedEmptyWal: outcome.removedEmptyWal, + retainedWalBytes: outcome.retainedWalBytes, + hint: + outcome.retainedWalBytes > 0 + ? "A non-empty -wal was kept; the engine rebuilds its index from it." + : undefined, }); - throw error; + + try { + return await this.connectFactory(dbPath, opts); + } catch (retryError) { + log("Turso open still failing after WAL index recovery", { + path: dbPath, + quarantineDir: outcome.quarantineDir, + error: String(retryError), + walPreserved: outcome.retainedWalBytes > 0, + hint: + outcome.retainedWalBytes > 0 + ? "The non-empty -wal was not moved, so no committed data was discarded." + : "The -wal was empty or absent, so no committed data was discarded.", + }); + throw error; + } + } finally { + try { + lock.release(); + } catch (releaseError) { + try { + log("Failed to release the Turso WAL index recovery lock", { + path: dbPath, + lock: lock.lockPath, + error: String(releaseError), + }); + } catch { + // A logging failure must never replace the original open error. + } + } } } } diff --git a/src/services/turso/wal-index-recovery.ts b/src/services/turso/wal-index-recovery.ts index 454a3408..d85989e2 100644 --- a/src/services/turso/wal-index-recovery.ts +++ b/src/services/turso/wal-index-recovery.ts @@ -1,8 +1,19 @@ import type { Database } from "@tursodatabase/database"; import type { DatabaseOpts } from "@tursodatabase/database-common"; -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, renameSync, rmSync } from "node:fs"; +import { + closeSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + openSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { basename, extname, join } from "node:path"; +import { basename, dirname, extname, join } from "node:path"; import { CONFIG } from "../../config.js"; import { log } from "../logger.js"; import type { ConnectFactory } from "./connection-manager.js"; @@ -13,15 +24,41 @@ import type { ConnectFactory } from "./connection-manager.js"; * next open fail with one of these two errors (see issue #435): * - "I/O error: short read on WAL frame at offset N: expected 4096 bytes, got 0" * - "Corrupt database: Invalid page type: 0" - * The main `.db` itself stays intact, so the store is recoverable by discarding - * the sidecars. + * + * Recovery contract — what this module does, and what it does not promise: + * - Only the *index* sidecars `-shm` and `-tshm` are moved aside. A `-wal` is + * removed only when it is absent or 0 bytes. A non-empty `-wal` is left in + * place: committed transactions can live only in the WAL, and a copy of the + * main `.db` alone can still pass `PRAGMA integrity_check`. Discarding such + * a WAL would silently roll the store back to an older snapshot. + * - With the index gone, the engine rebuilds `-tshm` from the surviving WAL on + * the next open. That is measured, not assumed: this repo's test seeds a real + * database, leaves committed rows only in the WAL, discards `-tshm` and reads + * the rows back through the real engine (Linux x64, + * `@tursodatabase/database` 0.8.3). It is still engine behaviour rather than + * an API this repository controls. A *torn* WAL (partial final frame) is not + * covered: then the retry fails, the original open error is surfaced, and the + * WAL is left untouched, so no committed data is lost. + * - A copy of the main `.db` (never the live file) must pass + * `PRAGMA integrity_check` first. That is a sanity gate only — it says + * nothing about the WAL's consistency. + * - Recovery is serialised across processes with + * `/recovery/.lock`. A process that cannot take the + * lock leaves the store alone rather than racing the owner. + * - A failure while quarantining is reported, never substituted for the + * original open error. */ const WAL_INDEX_CORRUPTION_PATTERNS = [ /short read on WAL frame at offset \d+/i, /invalid page type:\s*0\b/i, ]; -export const WAL_SIDECAR_SUFFIXES = ["-shm", "-tshm", "-wal"] as const; +/** Index sidecars that are moved aside. The WAL itself is handled separately. */ +export const WAL_INDEX_SIDECAR_SUFFIXES = ["-shm", "-tshm"] as const; +export const WAL_FILE_SUFFIX = "-wal"; + +/** A lock older than this is assumed to belong to a crashed healer. */ +export const RECOVERY_LOCK_STALE_MS = 5 * 60 * 1000; /** True for the two "stale WAL index" open failures that self-heal can repair. */ export function isWalIndexCorruptionError(error: unknown): boolean { @@ -30,26 +67,127 @@ export function isWalIndexCorruptionError(error: unknown): boolean { } export type WalIndexRecoveryResult = - | { status: "quarantined"; quarantineDir: string; moved: string[] } - | { status: "skipped"; reason: string }; + | { + status: "quarantined"; + quarantineDir: string | null; + moved: string[]; + removedEmptyWal: boolean; + retainedWalBytes: number; + } + | { status: "skipped"; reason: string } + | { status: "failed"; reason: string; quarantineDir: string | null; moved: string[] }; + +export interface RecoveryLock { + lockPath: string; + /** True when the previous owner's lock looked abandoned and was replaced. */ + tookOverStaleLock: boolean; + /** Idempotent, never throws. */ + release(): void; +} + +function recoveryRoot(): string { + return join(CONFIG.storagePath, "recovery"); +} + +export function recoveryLockPathFor(dbPath: string): string { + const stem = basename(dbPath, extname(dbPath)); + return join(recoveryRoot(), `${stem}.lock`); +} function quarantineDirFor(dbPath: string): string { const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const stem = basename(dbPath, extname(dbPath)); - return join(CONFIG.storagePath, "recovery", `${stem}-${stamp}`); + return join(recoveryRoot(), `${stem}-${stamp}`); +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isStaleLock(lockPath: string): boolean { + try { + return Date.now() - statSync(lockPath).mtimeMs > RECOVERY_LOCK_STALE_MS; + } catch { + // Vanished between EEXIST and stat: retry the atomic create instead. + return true; + } +} + +/** + * Take the per-database recovery lock, or return null when a live process owns + * it. The lock file is created with `O_EXCL`, so two healers can never both win. + * A lock whose mtime is older than `RECOVERY_LOCK_STALE_MS` is treated as a + * crashed healer's and taken over (reported via `tookOverStaleLock`). + */ +export function acquireRecoveryLock(dbPath: string): RecoveryLock | null { + const lockPath = recoveryLockPathFor(dbPath); + mkdirSync(dirname(lockPath), { recursive: true }); + let tookOverStaleLock = false; + + for (let attempt = 0; attempt < 2; attempt++) { + try { + const fd = openSync(lockPath, "wx"); + try { + writeFileSync(fd, `${process.pid} ${new Date().toISOString()}\n`); + } catch { + // Owner metadata is advisory only; the lock is still held. + } + let released = false; + return { + lockPath, + tookOverStaleLock, + release() { + if (released) return; + released = true; + try { + closeSync(fd); + } catch { + // ignore: fd may already be gone + } + try { + rmSync(lockPath, { force: true }); + } catch { + // ignore: a stale lock file is recoverable, a thrown error is not + } + }, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + if (attempt > 0 || !isStaleLock(lockPath)) { + return null; + } + tookOverStaleLock = true; + // Atomic rename, not unlink: only one process can move the stale inode, + // so two healers cannot both delete-then-create and end up sharing it. + const stalePath = `${lockPath}.${process.pid}.stale`; + try { + renameSync(lockPath, stalePath); + rmSync(stalePath, { force: true }); + } catch { + return null; + } + } + } + return null; } /** * Verify a copy of the main database file. The live file is never opened with a * different engine — that is what produces the "Invalid page type: 0" class. + * + * Any failure here (temp dir, copy, close) is contained: it reports "unhealthy" + * rather than throwing, so the caller can keep rethrowing the original error. */ async function isCopiedDatabaseHealthy( dbPath: string, connectFactory: ConnectFactory, opts: DatabaseOpts ): Promise { - const tempDir = mkdtempSync(join(tmpdir(), "opencode-mem-wal-check-")); + let tempDir: string | null = null; try { + tempDir = mkdtempSync(join(tmpdir(), "opencode-mem-wal-check-")); const copyPath = join(tempDir, basename(dbPath)); copyFileSync(dbPath, copyPath); let database: Database | null = null; @@ -75,36 +213,82 @@ async function isCopiedDatabaseHealthy( }); return false; } finally { - rmSync(tempDir, { recursive: true, force: true }); + if (tempDir) { + try { + rmSync(tempDir, { recursive: true, force: true }); + } catch { + // Temp-dir leftovers are harmless; never mask the caller's error. + } + } } } /** - * Move the WAL sidecars of an unopenable store into - * `/recovery/-/` so the next open starts clean. - * Sidecars are moved, never deleted, and only after a copy of the main `.db` - * passes `PRAGMA integrity_check`. + * Move the WAL *index* sidecars of an unopenable store into + * `/recovery/-/` so the next open rebuilds them. + * A non-empty `-wal` is deliberately left in place (see the module doc comment); + * only an absent or 0-byte `-wal` is deleted. */ export async function quarantineWalSidecars( dbPath: string, connectFactory: ConnectFactory, opts: DatabaseOpts ): Promise { - if (!existsSync(dbPath)) { - return { status: "skipped", reason: "main database file is missing" }; - } - if (!(await isCopiedDatabaseHealthy(dbPath, connectFactory, opts))) { - return { status: "skipped", reason: "integrity_check failed on a copy of the main database" }; - } - - const quarantineDir = quarantineDirFor(dbPath); - mkdirSync(quarantineDir, { recursive: true }); + let quarantineDir: string | null = null; const moved: string[] = []; - for (const suffix of WAL_SIDECAR_SUFFIXES) { - const sidecar = `${dbPath}${suffix}`; - if (!existsSync(sidecar)) continue; - renameSync(sidecar, join(quarantineDir, basename(sidecar))); - moved.push(basename(sidecar)); + + try { + if (!existsSync(dbPath)) { + return { status: "skipped", reason: "main database file is missing" }; + } + if (!(await isCopiedDatabaseHealthy(dbPath, connectFactory, opts))) { + return { status: "skipped", reason: "integrity_check failed on a copy of the main database" }; + } + + const walPath = `${dbPath}${WAL_FILE_SUFFIX}`; + const walExists = existsSync(walPath); + const walBytes = walExists ? statSync(walPath).size : 0; + const indexSidecars = WAL_INDEX_SIDECAR_SUFFIXES.map((suffix) => `${dbPath}${suffix}`).filter( + (sidecar) => existsSync(sidecar) + ); + const emptyingWal = walExists && walBytes === 0; + + if (indexSidecars.length === 0 && !emptyingWal) { + // A non-empty WAL is never touched, so there is nothing left to recover. + return { status: "skipped", reason: "no WAL index sidecars to quarantine" }; + } + + if (indexSidecars.length > 0) { + quarantineDir = quarantineDirFor(dbPath); + mkdirSync(quarantineDir, { recursive: true }); + } + + for (const sidecar of indexSidecars) { + // quarantineDir is non-null whenever indexSidecars is non-empty. + renameSync(sidecar, join(quarantineDir as string, basename(sidecar))); + moved.push(basename(sidecar)); + } + + let removedEmptyWal = false; + if (emptyingWal) { + // Empty WAL: no frames, nothing to lose. + rmSync(walPath, { force: true }); + removedEmptyWal = true; + } + + return { + status: "quarantined", + quarantineDir, + moved, + removedEmptyWal, + retainedWalBytes: walBytes, + }; + } catch (error) { + return { + status: "failed", + reason: describeError(error), + quarantineDir, + moved, + }; } - return { status: "quarantined", quarantineDir, moved }; } diff --git a/tests/turso-wal-index-recovery.test.ts b/tests/turso-wal-index-recovery.test.ts index ef654c3f..459f4404 100644 --- a/tests/turso-wal-index-recovery.test.ts +++ b/tests/turso-wal-index-recovery.test.ts @@ -1,7 +1,17 @@ -import { afterEach, describe, expect, it } from "bun:test"; -import { existsSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs"; +import { afterEach, describe, expect, it, setSystemTime } from "bun:test"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; +import type { ConnectFactory } from "../src/services/turso/connection-manager.js"; import { cleanupTursoTestDirectory } from "./turso-test-utils.js"; const SHORT_READ = "I/O error: short read on WAL frame at offset 4096: expected 4096 bytes, got 0"; @@ -36,36 +46,50 @@ function stubDatabase(integrityRows: Array>) { } as any; } +function recoveryEntries(baseDir: string): string[] { + const root = join(baseDir, "recovery"); + return existsSync(root) ? readdirSync(root).sort() : []; +} + +function quarantineDirs(baseDir: string): string[] { + return recoveryEntries(baseDir).filter((entry) => !entry.endsWith(".lock")); +} + describe("turso WAL index recovery", () => { let baseDir: string; afterEach(async () => { + setSystemTime(); + delete process.env.OPENCODE_MEM_LOG_FILE; const { CONFIG } = await import("../src/config.js"); CONFIG.tursoAutoHeal = true; CONFIG.tursoMultiprocessWal = true; await cleanupTursoTestDirectory(baseDir); }); - async function prepare(prefix: string) { + async function prepare(prefix: string, options: { wal?: string } = {}) { baseDir = mkdtempSync(join(tmpdir(), prefix)); const { CONFIG } = await import("../src/config.js"); CONFIG.storagePath = baseDir; CONFIG.databaseEncryptionEnabled = false; const dbPath = join(baseDir, "memories.db"); writeFileSync(dbPath, "main"); - writeFileSync(`${dbPath}-wal`, ""); + writeFileSync(`${dbPath}-wal`, options.wal ?? ""); writeFileSync(`${dbPath}-shm`, "shm"); writeFileSync(`${dbPath}-tshm`, "tshm"); return dbPath; } - it("quarantines the sidecars and retries the open exactly once", async () => { + it("quarantines only the index sidecars (0-byte WAL) and retries the open exactly once", async () => { const dbPath = await prepare("turso-heal-ok-"); const calls: string[] = []; + let liveOpens = 0; const connectFactory = async (path: string) => { calls.push(path); - if (path === dbPath && calls.length === 1) { - throw new Error(SHORT_READ); + if (path === dbPath) { + liveOpens += 1; + // Fail the open and the lock recheck, then succeed after the quarantine. + if (liveOpens <= 2) throw new Error(SHORT_READ); } return stubDatabase([{ integrity_check: "ok" }]); }; @@ -78,34 +102,96 @@ describe("turso WAL index recovery", () => { await manager.closeAll(); } - expect(calls.filter((path) => path === dbPath)).toHaveLength(2); - expect(calls[0]).toBe(dbPath); - // The verification copy lives in a temp dir, never at the live path. - expect(calls[1]).not.toBe(dbPath); - expect(calls[1]).toContain("memories.db"); - - const recoveryRoot = join(baseDir, "recovery"); - const quarantined = readdirSync(recoveryRoot); - expect(quarantined).toHaveLength(1); - const quarantineDir = join(recoveryRoot, quarantined[0] as string); - expect(readdirSync(quarantineDir).sort()).toEqual([ + // failed open → lock recheck → copy verification → single retry + expect(calls.filter((path) => path === dbPath)).toHaveLength(3); + expect(calls).toHaveLength(4); + expect(calls[2]).toContain("memories.db"); + expect(calls[2]).not.toBe(dbPath); + const [quarantined] = quarantineDirs(baseDir); + expect(quarantined).toBeDefined(); + expect(readdirSync(join(baseDir, "recovery", quarantined as string)).sort()).toEqual([ "memories.db-shm", "memories.db-tshm", - "memories.db-wal", ]); expect(existsSync(`${dbPath}-wal`)).toBe(false); expect(existsSync(`${dbPath}-shm`)).toBe(false); expect(existsSync(`${dbPath}-tshm`)).toBe(false); expect(existsSync(dbPath)).toBe(true); + expect(recoveryEntries(baseDir).filter((entry) => entry.endsWith(".lock"))).toEqual([]); }); + it.skipIf(process.platform === "win32")( + "keeps a non-empty -wal and preserves committed rows across the reopen", + async () => { + baseDir = mkdtempSync(join(tmpdir(), "turso-heal-wal-")); + const { CONFIG } = await import("../src/config.js"); + CONFIG.storagePath = baseDir; + CONFIG.databaseEncryptionEnabled = false; + + const { connect } = await import("@tursodatabase/database"); + const { buildConnectOptions, TursoConnectionManager } = + await import("../src/services/turso/connection-manager.js"); + + const dbPath = join(baseDir, "memories.db"); + const opts = buildConnectOptions(); + const seed = await connect(dbPath, opts); + await seed.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + await seed.run("INSERT INTO t (v) VALUES (?)", ["checkpointed"]); + await seed.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + await seed.run("INSERT INTO t (v) VALUES (?)", ["wal-only"]); + await seed.close(); + + const walPath = `${dbPath}-wal`; + const walInode = statSync(walPath).ino; + expect(statSync(walPath).size).toBeGreaterThan(0); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + + let liveOpens = 0; + const connectFactory: ConnectFactory = async (path, openOpts) => { + if (path === dbPath) { + liveOpens += 1; + // Fail the first attempt and the lock recheck; heal, then really reopen. + if (liveOpens <= 2) throw new Error(SHORT_READ); + } + return connect(path, openOpts); + }; + + const manager = new TursoConnectionManager(connectFactory); + let values: string[]; + try { + const db = await manager.getConnection(dbPath); + values = (await db.all<{ v: string }>("SELECT v FROM t ORDER BY id")).map((row) => row.v); + } finally { + await manager.closeAll(); + } + + // The WAL was not discarded, and its committed row survived the recovery. + expect(values).toEqual(["checkpointed", "wal-only"]); + expect(existsSync(walPath)).toBe(true); + expect(statSync(walPath).size).toBeGreaterThan(0); + // Same inode: the file was kept, not moved and recreated. + expect(statSync(walPath).ino).toBe(walInode); + + const [quarantined] = quarantineDirs(baseDir); + const quarantinedFiles = readdirSync(join(baseDir, "recovery", quarantined as string)).sort(); + expect(quarantinedFiles).toContain("memories.db-tshm"); + expect(quarantinedFiles.every((name) => /-(shm|tshm)$/.test(name))).toBe(true); + // Engine evidence, not an assumption: the old -tshm is in quarantine *and* + // a fresh, non-empty index exists beside the WAL again, rebuilt on reopen. + // The seeded WAL has no torn trailing frame; a torn WAL is the + // "retry fails, original error surfaces, WAL untouched" case. + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + expect(statSync(`${dbPath}-tshm`).size).toBeGreaterThan(0); + } + ); + it("heals the invalid-page-type failure too", async () => { const dbPath = await prepare("turso-heal-page-"); - const calls: string[] = []; + let liveOpens = 0; const connectFactory = async (path: string) => { - calls.push(path); - if (path === dbPath && calls.length === 1) { - throw new Error(INVALID_PAGE); + if (path === dbPath) { + liveOpens += 1; + if (liveOpens <= 2) throw new Error(INVALID_PAGE); } return stubDatabase([{ integrity_check: "ok" }]); }; @@ -118,7 +204,7 @@ describe("turso WAL index recovery", () => { await manager.closeAll(); } - expect(readdirSync(join(baseDir, "recovery"))).toHaveLength(1); + expect(quarantineDirs(baseDir)).toHaveLength(1); expect(existsSync(`${dbPath}-tshm`)).toBe(false); }); @@ -141,8 +227,9 @@ describe("turso WAL index recovery", () => { await manager.closeAll(); } - expect(calls.filter((path) => path === dbPath)).toHaveLength(1); - expect(existsSync(join(baseDir, "recovery"))).toBe(false); + // failed open + lock recheck, then the verification copy — no retry. + expect(calls.filter((path) => path === dbPath)).toHaveLength(2); + expect(recoveryEntries(baseDir)).toEqual([]); expect(existsSync(`${dbPath}-wal`)).toBe(true); expect(existsSync(`${dbPath}-shm`)).toBe(true); expect(existsSync(`${dbPath}-tshm`)).toBe(true); @@ -167,9 +254,154 @@ describe("turso WAL index recovery", () => { await manager.closeAll(); } - // One copy verification plus the single retry — no second heal cycle. - expect(calls.filter((path) => path === dbPath)).toHaveLength(2); - expect(readdirSync(join(baseDir, "recovery"))).toHaveLength(1); + // failed open, lock recheck and the single retry — no second heal cycle. + expect(calls.filter((path) => path === dbPath)).toHaveLength(3); + expect(quarantineDirs(baseDir)).toHaveLength(1); + expect(recoveryEntries(baseDir).filter((entry) => entry.endsWith(".lock"))).toEqual([]); + }); + + it("does not recover while another process holds the recovery lock", async () => { + const dbPath = await prepare("turso-heal-lock-"); + const { acquireRecoveryLock, recoveryLockPathFor } = + await import("../src/services/turso/wal-index-recovery.js"); + const held = acquireRecoveryLock(dbPath); + expect(held).not.toBeNull(); + const lockPath = recoveryLockPathFor(dbPath); + + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + if (path === dbPath) { + throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/short read on WAL frame/); + } finally { + await manager.closeAll(); + } + + try { + // Only the initial open: no recheck, no copy, no moves. + expect(calls).toEqual([dbPath]); + expect(existsSync(lockPath)).toBe(true); + expect(quarantineDirs(baseDir)).toEqual([]); + expect(existsSync(`${dbPath}-wal`)).toBe(true); + expect(existsSync(`${dbPath}-shm`)).toBe(true); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + } finally { + held?.release(); + } + }); + + it("returns the peer-healed connection when the open succeeds under the lock", async () => { + const dbPath = await prepare("turso-heal-recheck-"); + const peerConnection = stubDatabase([{ integrity_check: "ok" }]); + const calls: string[] = []; + const connectFactory = async (path: string) => { + calls.push(path); + if (path !== dbPath) { + return stubDatabase([{ integrity_check: "ok" }]); + } + if (calls.filter((call) => call === dbPath).length === 1) { + throw new Error(SHORT_READ); + } + return peerConnection; + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + const db = await manager.getConnection(dbPath); + expect(db.getClient()).toBe(peerConnection); + } finally { + await manager.closeAll(); + } + + // failed open → recheck only; the verification copy is never opened. + expect(calls).toEqual([dbPath, dbPath]); + expect(recoveryEntries(baseDir)).toEqual([]); + expect(existsSync(`${dbPath}-wal`)).toBe(true); + expect(existsSync(`${dbPath}-shm`)).toBe(true); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + }); + + it("rethrows the original error when a quarantine rename fails mid-way", async () => { + const dbPath = await prepare("turso-heal-fs-"); + const logFile = join(baseDir, "recovery.log"); + process.env.OPENCODE_MEM_LOG_FILE = logFile; + + // Freeze the clock so the timestamped quarantine dir is predictable. + setSystemTime(new Date("2026-01-02T03:04:05.678Z")); + const quarantineDir = join(baseDir, "recovery", "memories-2026-01-02T03-04-05-678Z"); + const blocker = join(quarantineDir, "memories.db-tshm"); + mkdirSync(blocker, { recursive: true }); + writeFileSync(join(blocker, "blocker"), "x"); + + const connectFactory = async (path: string) => { + if (path === dbPath) { + throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/short read on WAL frame/); + } finally { + await manager.closeAll(); + } + + // -shm moved before the -tshm rename failed; the WAL was untouched. + expect(readdirSync(quarantineDir).sort()).toEqual(["memories.db-shm", "memories.db-tshm"]); + expect(existsSync(`${dbPath}-shm`)).toBe(false); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + expect(existsSync(`${dbPath}-wal`)).toBe(true); + expect(recoveryEntries(baseDir).filter((entry) => entry.endsWith(".lock"))).toEqual([]); + + const logged = readFileSync(logFile, "utf8"); + expect(logged).toContain("WAL index recovery failed"); + expect(logged).toContain(dbPath); + expect(logged).toContain(quarantineDir); + expect(logged).toContain("short read on WAL frame"); + }); + + it("takes over a stale recovery lock and proceeds", async () => { + const dbPath = await prepare("turso-heal-stale-"); + const { recoveryLockPathFor } = await import("../src/services/turso/wal-index-recovery.js"); + const lockPath = recoveryLockPathFor(dbPath); + mkdirSync(join(baseDir, "recovery"), { recursive: true }); + writeFileSync(lockPath, "999999 crashed healer\n"); + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); + utimesSync(lockPath, tenMinutesAgo, tenMinutesAgo); + + let liveOpens = 0; + const connectFactory = async (path: string) => { + if (path === dbPath) { + liveOpens += 1; + if (liveOpens <= 2) throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await manager.getConnection(dbPath); + } finally { + await manager.closeAll(); + } + + // initial open, lock recheck, single retry after the sidecars were moved. + expect(liveOpens).toBe(3); + expect(quarantineDirs(baseDir)).toHaveLength(1); + expect(existsSync(`${dbPath}-tshm`)).toBe(false); + expect(existsSync(lockPath)).toBe(false); }); it("ignores open errors outside the two known signatures", async () => { @@ -189,7 +421,7 @@ describe("turso WAL index recovery", () => { } expect(calls).toHaveLength(1); - expect(existsSync(join(baseDir, "recovery"))).toBe(false); + expect(recoveryEntries(baseDir)).toEqual([]); expect(existsSync(`${dbPath}-tshm`)).toBe(true); }); @@ -213,7 +445,7 @@ describe("turso WAL index recovery", () => { } expect(calls).toHaveLength(1); - expect(existsSync(join(baseDir, "recovery"))).toBe(false); + expect(recoveryEntries(baseDir)).toEqual([]); expect(existsSync(`${dbPath}-tshm`)).toBe(true); }); From edfa1d7f760ddc9f7f39a3e2e2a9acd528fa92dc Mon Sep 17 00:00:00 2001 From: EyJunge1 <149941075+EyJunge1@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:44:31 +0100 Subject: [PATCH 5/5] fix(turso): retry WAL heal after a blocked quarantine Keep healedPaths unset when quarantine moved nothing so the same process can finish recovery, retry open after a partial -tshm move, prefer quarantining -tshm first, and align the example.jsonc comment with the non-empty WAL policy. --- opencode-mem.example.jsonc | 8 ++-- src/services/turso/connection-manager.ts | 35 ++++++++++++-- src/services/turso/wal-index-recovery.ts | 8 +++- tests/turso-wal-index-recovery.test.ts | 60 ++++++++++++++++++++++-- 4 files changed, 97 insertions(+), 14 deletions(-) diff --git a/opencode-mem.example.jsonc b/opencode-mem.example.jsonc index ebdb3bd8..850c05de 100644 --- a/opencode-mem.example.jsonc +++ b/opencode-mem.example.jsonc @@ -346,10 +346,12 @@ // Pinned versions like opencode-mem@2.26.0 are never auto-updated. "autoUpdate": true, - // Recover a store whose Turso WAL index outlived its WAL (a stale -tshm/-shm + // Recover a store whose Turso WAL index is stale (a leftover -tshm/-shm, often // beside a 0-byte -wal). The guard copies the main .db, verifies it with - // PRAGMA integrity_check, moves -wal/-shm/-tshm to /recovery/, - // then retries the open once. Files are moved, never deleted. + // PRAGMA integrity_check, moves the -shm/-tshm index sidecars to + // /recovery/, and retries the open once. A non-empty -wal is + // never moved; only an absent or 0-byte -wal is removed. Concurrent sessions + // serialise healing through /recovery/.lock. "tursoAutoHeal": true, // Turso experimental multiprocess_wal (Unix). Set false for single-owner mode: diff --git a/src/services/turso/connection-manager.ts b/src/services/turso/connection-manager.ts index d3ae4b88..fd543eb1 100644 --- a/src/services/turso/connection-manager.ts +++ b/src/services/turso/connection-manager.ts @@ -104,7 +104,11 @@ export class TursoConnectionManager { private readonly pending = new Map>(); private readonly closingConnections = new Map>(); private closingPromise: Promise | null = null; - /** One WAL-index recovery attempt per path per process — never loop. */ + /** + * Paths that already had a completed recovery attempt this process (no loop). + * Set after a successful quarantine or a definitive skip — not when quarantine + * moved nothing, so a later open can still finish a blocked first attempt. + */ private readonly healedPaths = new Set(); private readonly warnedForeignShmPaths = new Set(); @@ -158,8 +162,6 @@ export class TursoConnectionManager { }); } - this.healedPaths.add(dbPath); - try { // The peer may have healed between our failed open and taking the lock. try { @@ -192,9 +194,29 @@ export class TursoConnectionManager { moved: outcome.moved, error: String(error), }); - throw error; + // A partial move (e.g. -tshm gone, -shm rename failed) may already be + // enough to open. Retry once when anything moved; leave healedPaths + // clear when nothing moved so a later attempt can finish the job. + if (outcome.moved.length === 0) { + throw error; + } + this.healedPaths.add(dbPath); + try { + return await this.connectFactory(dbPath, opts); + } catch (retryError) { + log("Turso open still failing after a partial WAL index recovery", { + path: dbPath, + quarantineDir: outcome.quarantineDir, + moved: outcome.moved, + error: String(retryError), + }); + throw error; + } } if (outcome.status === "skipped") { + // Retrying the same path will not help (missing main file / bad copy / + // nothing to move). Remember so we do not keep copying the database. + this.healedPaths.add(dbPath); log("Turso open failed and the store was left untouched (no recovery attempted)", { path: dbPath, reason: outcome.reason, @@ -204,6 +226,11 @@ export class TursoConnectionManager { throw error; } + // Quarantine completed: mark before the retry so a still-corrupt store + // cannot loop. Do not mark earlier — a mid-quarantine failure with + // nothing useful moved must remain retryable in this process. + this.healedPaths.add(dbPath); + log("Moved stale WAL index sidecars aside after a Turso open failure; retrying once", { path: dbPath, quarantineDir: outcome.quarantineDir, diff --git a/src/services/turso/wal-index-recovery.ts b/src/services/turso/wal-index-recovery.ts index d85989e2..eaf8923f 100644 --- a/src/services/turso/wal-index-recovery.ts +++ b/src/services/turso/wal-index-recovery.ts @@ -53,8 +53,12 @@ const WAL_INDEX_CORRUPTION_PATTERNS = [ /invalid page type:\s*0\b/i, ]; -/** Index sidecars that are moved aside. The WAL itself is handled separately. */ -export const WAL_INDEX_SIDECAR_SUFFIXES = ["-shm", "-tshm"] as const; +/** + * Index sidecars that are moved aside. The WAL itself is handled separately. + * `-tshm` is moved first: it is the Turso index that blocks open, so a partial + * quarantine that stops after the first rename is more likely to already heal. + */ +export const WAL_INDEX_SIDECAR_SUFFIXES = ["-tshm", "-shm"] as const; export const WAL_FILE_SUFFIX = "-wal"; /** A lock older than this is assumed to belong to a crashed healer. */ diff --git a/tests/turso-wal-index-recovery.test.ts b/tests/turso-wal-index-recovery.test.ts index 459f4404..6a360210 100644 --- a/tests/turso-wal-index-recovery.test.ts +++ b/tests/turso-wal-index-recovery.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, readdirSync, + rmSync, statSync, utimesSync, writeFileSync, @@ -338,13 +339,18 @@ describe("turso WAL index recovery", () => { // Freeze the clock so the timestamped quarantine dir is predictable. setSystemTime(new Date("2026-01-02T03:04:05.678Z")); const quarantineDir = join(baseDir, "recovery", "memories-2026-01-02T03-04-05-678Z"); - const blocker = join(quarantineDir, "memories.db-tshm"); + // Block the second sidecar (-shm). -tshm is moved first so a partial + // quarantine still removes the Turso index that blocks open. + const blocker = join(quarantineDir, "memories.db-shm"); mkdirSync(blocker, { recursive: true }); writeFileSync(join(blocker, "blocker"), "x"); + let liveOpens = 0; const connectFactory = async (path: string) => { if (path === dbPath) { - throw new Error(SHORT_READ); + liveOpens += 1; + // Fail open, lock recheck, and the post-partial retry. + if (liveOpens <= 3) throw new Error(SHORT_READ); } return stubDatabase([{ integrity_check: "ok" }]); }; @@ -357,12 +363,15 @@ describe("turso WAL index recovery", () => { await manager.closeAll(); } - // -shm moved before the -tshm rename failed; the WAL was untouched. + // -tshm moved before the -shm rename failed; the WAL was untouched. + // The blocker directory remains under the quarantine name. expect(readdirSync(quarantineDir).sort()).toEqual(["memories.db-shm", "memories.db-tshm"]); - expect(existsSync(`${dbPath}-shm`)).toBe(false); - expect(existsSync(`${dbPath}-tshm`)).toBe(true); + expect(existsSync(`${dbPath}-tshm`)).toBe(false); + expect(existsSync(`${dbPath}-shm`)).toBe(true); expect(existsSync(`${dbPath}-wal`)).toBe(true); expect(recoveryEntries(baseDir).filter((entry) => entry.endsWith(".lock"))).toEqual([]); + // open + recheck + partial-failure retry + expect(liveOpens).toBe(3); const logged = readFileSync(logFile, "utf8"); expect(logged).toContain("WAL index recovery failed"); @@ -371,6 +380,47 @@ describe("turso WAL index recovery", () => { expect(logged).toContain("short read on WAL frame"); }); + it("can finish recovery in-process after a quarantine that moved nothing", async () => { + const dbPath = await prepare("turso-heal-fs-retry-"); + setSystemTime(new Date("2026-01-02T03:04:05.678Z")); + const quarantineDir = join(baseDir, "recovery", "memories-2026-01-02T03-04-05-678Z"); + // Block the first sidecar so the first attempt moves nothing. + const blocker = join(quarantineDir, "memories.db-tshm"); + mkdirSync(blocker, { recursive: true }); + writeFileSync(join(blocker, "blocker"), "x"); + + let liveOpens = 0; + const connectFactory = async (path: string) => { + if (path === dbPath) { + liveOpens += 1; + // First attempt: open + recheck fail (no post-quarantine retry — nothing moved). + // Second attempt: open + recheck fail, then succeed after a real quarantine. + if (liveOpens <= 4) throw new Error(SHORT_READ); + } + return stubDatabase([{ integrity_check: "ok" }]); + }; + + const { TursoConnectionManager } = await import("../src/services/turso/connection-manager.js"); + const manager = new TursoConnectionManager(connectFactory); + try { + await expect(manager.getConnection(dbPath)).rejects.toThrow(/short read on WAL frame/); + expect(existsSync(`${dbPath}-tshm`)).toBe(true); + expect(existsSync(`${dbPath}-shm`)).toBe(true); + + // Clear the blocker so the next open can quarantine both sidecars. + rmSync(blocker, { recursive: true, force: true }); + + await manager.getConnection(dbPath); + } finally { + await manager.closeAll(); + } + + expect(existsSync(`${dbPath}-tshm`)).toBe(false); + expect(existsSync(`${dbPath}-shm`)).toBe(false); + expect(quarantineDirs(baseDir).length).toBeGreaterThanOrEqual(1); + expect(liveOpens).toBe(5); + }); + it("takes over a stale recovery lock and proceeds", async () => { const dbPath = await prepare("turso-heal-stale-"); const { recoveryLockPathFor } = await import("../src/services/turso/wal-index-recovery.js");