feat(turso): self-heal a stale WAL index on open

README.md

@@ -53,6 +53,8 @@ Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to
 
 On macOS and Linux, multiple OpenCode sessions can share the same `storagePath` via Turso’s experimental `multiprocess_wal` (every process must use the same mode — restart all sessions after upgrading). On Windows the engine rejects that flag, so only one OpenCode session can own the memory databases at a time.
 
+`tursoMultiprocessWal: false` opens in single-owner mode instead: other processes then fail with `database is locked` rather than sharing the store. That is the safer choice if non-plugin tools touch the same files, because a _writing_ foreign SQLite process makes Turso’s WAL index stale and its own next write is silently lost. `tursoAutoHeal` (default `true`) moves a store’s `-shm`/`-tshm` index sidecars into `storagePath/recovery/<db>-<timestamp>/` and retries once when an open fails with a stale WAL index, after verifying a copy of the main database. A non-empty `-wal` is never moved, so committed but uncheckpointed transactions survive; only an absent or 0-byte `-wal` is removed. Concurrent sessions serialise healing through `storagePath/recovery/<db>.lock`.
+
 On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format:
 
 - Each shard is backed up as `<shard>.db.legacy.bak` before rewrite

opencode-mem.example.jsonc

@@ -346,6 +346,17 @@
   // Pinned versions like opencode-mem@2.26.0 are never auto-updated.
   "autoUpdate": true,
 
+  // Recover a store whose Turso WAL index outlived its WAL (a stale -tshm/-shm
+  // beside a 0-byte -wal). The guard copies the main .db, verifies it with
+  // PRAGMA integrity_check, moves -wal/-shm/-tshm to <storagePath>/recovery/,
+  // then retries the open once. Files are moved, never deleted.
+  "tursoAutoHeal": true,
+
+  // Turso experimental multiprocess_wal (Unix). Set false for single-owner mode:
+  // a second process then fails loudly with "database is locked" instead of
+  // writing into a divergent WAL. See issue #435.
+  "tursoMultiprocessWal": true,
+
   // ============================================
   // User Profile System
   // ============================================

src/config.ts

@@ -165,6 +165,21 @@ interface OpenCodeMemConfig {
     | "aegis128x4";
   /** Automatically clear OpenCode's cached plugin install when a newer npm latest is available. */
   autoUpdate?: boolean;
+  /**
+   * Self-heal a store whose Turso WAL index is stale (a leftover `-tshm`/`-shm`,
+   * often after a foreign writer or beside a 0-byte `-wal`): verify a copy of
+   * the main `.db`, move the *index* sidecars to `<storagePath>/recovery/`, then
+   * retry once. A non-empty `-wal` is left in place so the engine can rebuild
+   * its index from it. Recovery is serialised across processes with
+   * `<storagePath>/recovery/<dbstem>.lock`. Default true.
+   */
+  tursoAutoHeal?: boolean;
+  /**
+   * Open with Turso's experimental `multiprocess_wal` (Unix). Set false for
+   * single-owner mode, so a foreign process fails loudly with "database is
+   * locked" instead of writing into a divergent WAL. Default true.
+   */
+  tursoMultiprocessWal?: boolean;
   compaction?: {
     enabled?: boolean;
     memoryLimit?: number;
@@ -302,6 +317,8 @@ const DEFAULTS: Required<
   databaseEncryptionEnabled: false,
   databaseEncryptionCipher: "aes256gcm",
   autoUpdate: true,
+  tursoAutoHeal: true,
+  tursoMultiprocessWal: true,
   memory: {
     defaultScope: "project",
   },
@@ -681,6 +698,19 @@ const CONFIG_TEMPLATE = `{
   // Pinned versions like opencode-mem@2.26.0 are never auto-updated.
   "autoUpdate": true,
 
+  // Recover a store whose Turso WAL index is stale (a leftover -tshm/-shm, often
+  // beside a 0-byte -wal). The guard copies the main .db, verifies it with
+  // PRAGMA integrity_check, moves the -shm/-tshm index sidecars to
+  // <storagePath>/recovery/, and retries the open once. A non-empty -wal is
+  // never moved, so committed but uncheckpointed transactions are preserved.
+  // Recovery is serialised across processes by <storagePath>/recovery/<db>.lock.
+  "tursoAutoHeal": true,
+
+  // Turso experimental multiprocess_wal (Unix). Set false for single-owner mode:
+  // a second process then fails loudly with "database is locked" instead of
+  // writing into a divergent WAL. See issue #435.
+  "tursoMultiprocessWal": true,
+
   // ============================================
   // User Profile System
   // ============================================
@@ -1197,6 +1227,8 @@ function buildConfig(fileConfig: OpenCodeMemConfig) {
     databaseEncryptionCipher:
       fileConfig.databaseEncryptionCipher ?? DEFAULTS.databaseEncryptionCipher,
     autoUpdate: fileConfig.autoUpdate ?? DEFAULTS.autoUpdate,
+    tursoAutoHeal: fileConfig.tursoAutoHeal ?? DEFAULTS.tursoAutoHeal,
+    tursoMultiprocessWal: fileConfig.tursoMultiprocessWal ?? DEFAULTS.tursoMultiprocessWal,
     memory: {
       defaultScope: fileConfig.memory?.defaultScope ?? DEFAULTS.memory.defaultScope,
     },

src/services/turso/connection-manager.ts

@@ -7,6 +7,11 @@ import { log } from "../logger.js";
 import { collectReleasedSqliteHandles } from "./sqlite-handle-release.js";
 import { TursoDb } from "./turso-db.js";
 import { resolveOrCreateDatabaseEncryptionKey } from "./encryption-key.js";
+import {
+  isWalIndexCorruptionError,
+  quarantineWalSidecars,
+  acquireRecoveryLock,
+} from "./wal-index-recovery.js";
 
 export type ConnectFactory = (path: string, opts?: DatabaseOpts) => Promise<Database>;
 
@@ -23,11 +28,16 @@ export function supportsTursoMultiprocessWal(
   return platform !== "win32";
 }
 
-/** Experimental flags required on every Turso open for the current platform. */
+/**
+ * Experimental flags required on every Turso open for the current platform.
+ * `multiprocessWal: false` opts into single-owner mode (Unix), so a foreign
+ * process gets a loud "database is locked" instead of silent divergence.
+ */
 export function tursoExperimentalFeatures(
-  platform: NodeJS.Platform = process.platform
+  platform: NodeJS.Platform = process.platform,
+  multiprocessWal: boolean = CONFIG.tursoMultiprocessWal
 ): Array<(typeof TURSO_BASE_EXPERIMENTAL_FEATURES)[number] | "multiprocess_wal"> {
-  if (supportsTursoMultiprocessWal(platform)) {
+  if (multiprocessWal && supportsTursoMultiprocessWal(platform)) {
     return [...TURSO_BASE_EXPERIMENTAL_FEATURES, "multiprocess_wal"];
   }
   return [...TURSO_BASE_EXPERIMENTAL_FEATURES];
@@ -94,9 +104,170 @@ export class TursoConnectionManager {
   private readonly pending = new Map<string, Promise<TursoDb>>();
   private readonly closingConnections = new Map<string, Promise<void>>();
   private closingPromise: Promise<void> | null = null;
+  /** One WAL-index recovery attempt per path per process — never loop. */
+  private readonly healedPaths = new Set<string>();
+  private readonly warnedForeignShmPaths = new Set<string>();
 
   constructor(private readonly connectFactory: ConnectFactory = connect) {}
 
+  /**
+   * Connect, and on a stale-WAL-index failure quarantine the index sidecars once
+   * and retry once. Only the two known signatures heal (see #435); any other open
+   * error propagates unchanged.
+   *
+   * Recovery is serialised across processes (they share `storagePath` through
+   * `multiprocess_wal`) with `<storagePath>/recovery/<dbstem>.lock`. Under that
+   * lock the open is re-attempted before any file is moved, so a peer that
+   * healed first leaves us nothing to do. A non-empty WAL is never moved — see
+   * `wal-index-recovery.ts` for the data-safety contract.
+   */
+  private async connectWithWalIndexRecovery(dbPath: string, opts: DatabaseOpts): Promise<Database> {
+    try {
+      return await this.connectFactory(dbPath, opts);
+    } catch (error) {
+      if (!CONFIG.tursoAutoHeal || !isWalIndexCorruptionError(error)) {
+        throw error;
+      }
+      if (this.healedPaths.has(dbPath)) {
+        throw error;
+      }
+
+      let lock: ReturnType<typeof acquireRecoveryLock>;
+      try {
+        lock = acquireRecoveryLock(dbPath);
+      } catch (lockError) {
+        log("Turso WAL index recovery skipped: could not take the recovery lock", {
+          path: dbPath,
+          error: String(lockError),
+          originalError: String(error),
+        });
+        throw error;
+      }
+      if (!lock) {
+        log("Turso WAL index recovery skipped: another process owns the recovery lock", {
+          path: dbPath,
+          hint: "The owning process is healing this store; leaving its files untouched.",
+          error: String(error),
+        });
+        throw error;
+      }
+      if (lock.tookOverStaleLock) {
+        log("Took over a stale Turso WAL index recovery lock", {
+          path: dbPath,
+          lock: lock.lockPath,
+        });
+      }
+
+      this.healedPaths.add(dbPath);
+
+      try {
+        // The peer may have healed between our failed open and taking the lock.
+        try {
+          const connection = await this.connectFactory(dbPath, opts);
+          log("Turso open succeeded on the recovery-lock recheck; nothing was moved", {
+            path: dbPath,
+          });
+          return connection;
+        } catch (recheckError) {
+          if (!isWalIndexCorruptionError(recheckError)) {
+            // e.g. a lock error: do not move a live owner's files.
+            log(
+              "Turso recovery-lock recheck failed with an unrelated error; store left untouched",
+              {
+                path: dbPath,
+                error: String(recheckError),
+                originalError: String(error),
+              }
+            );
+            throw error;
+          }
+        }
+
+        const outcome = await quarantineWalSidecars(dbPath, this.connectFactory, opts);
+        if (outcome.status === "failed") {
+          log("Turso WAL index recovery failed; the store was left as-is where possible", {
+            path: dbPath,
+            reason: outcome.reason,
+            quarantineDir: outcome.quarantineDir,
+            moved: outcome.moved,
+            error: String(error),
+          });
+          throw error;
+        }
+        if (outcome.status === "skipped") {
+          log("Turso open failed and the store was left untouched (no recovery attempted)", {
+            path: dbPath,
+            reason: outcome.reason,
+            hint: "Restore this database from a backup; the sidecar files were not moved.",
+            error: String(error),
+          });
+          throw error;
+        }
+
+        log("Moved stale WAL index sidecars aside after a Turso open failure; retrying once", {
+          path: dbPath,
+          quarantineDir: outcome.quarantineDir,
+          moved: outcome.moved,
+          removedEmptyWal: outcome.removedEmptyWal,
+          retainedWalBytes: outcome.retainedWalBytes,
+          hint:
+            outcome.retainedWalBytes > 0
+              ? "A non-empty -wal was kept; the engine rebuilds its index from it."
+              : undefined,
+        });
+
+        try {
+          return await this.connectFactory(dbPath, opts);
+        } catch (retryError) {
+          log("Turso open still failing after WAL index recovery", {
+            path: dbPath,
+            quarantineDir: outcome.quarantineDir,
+            error: String(retryError),
+            walPreserved: outcome.retainedWalBytes > 0,
+            hint:
+              outcome.retainedWalBytes > 0
+                ? "The non-empty -wal was not moved, so no committed data was discarded."
+                : "The -wal was empty or absent, so no committed data was discarded.",
+          });
+          throw error;
+        }
+      } finally {
+        try {
+          lock.release();
+        } catch (releaseError) {
+          try {
+            log("Failed to release the Turso WAL index recovery lock", {
+              path: dbPath,
+              lock: lock.lockPath,
+              error: String(releaseError),
+            });
+          } catch {
+            // A logging failure must never replace the original open error.
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * A foreign sqlite process that wrote this store leaves `-shm` next to a
+   * Turso `-tshm`. Read-only access is harmless; a foreign writer diverges
+   * silently and loses its own next write, so warn once per path. Never delete
+   * `-shm` here — it may belong to a live reader.
+   */
+  private warnOnForeignSqliteShm(dbPath: string): void {
+    if (this.warnedForeignShmPaths.has(dbPath)) return;
+    if (!existsSync(`${dbPath}-shm`) || !existsSync(`${dbPath}-tshm`)) return;
+    this.warnedForeignShmPaths.add(dbPath);
+    log("Foreign SQLite process touched this store (-shm beside -tshm)", {
+      path: dbPath,
+      hint:
+        "Read-only access is harmless, but a foreign writer silently loses data — " +
+        "see issue #435. Close other tools on this store, or set tursoMultiprocessWal=false " +
+        "for single-owner mode.",
+    });
+  }
+
   async getConnection(dbPath: string): Promise<TursoDb> {
     if (this.closingPromise) {
       await this.closingPromise;
@@ -127,9 +298,10 @@ export class TursoConnectionManager {
       const opts = buildConnectOptions(encryption);
       let database: Database | null = null;
       try {
-        database = await this.connectFactory(dbPath, opts);
+        database = await this.connectWithWalIndexRecovery(dbPath, opts);
         const db = new TursoDb(database);
         await db.execute("PRAGMA foreign_keys = ON");
+        this.warnOnForeignSqliteShm(dbPath);
         this.connections.set(dbPath, db);
         return db;
       } catch (error) {