test: Workers-pool D1/FTS5 integration tests (test-leg slice 2)

package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "dev": "wrangler dev",
     "deploy": "wrangler deploy",
-    "test": "vitest run"
+    "test": "vitest run && vitest run --config vitest.workers.config.ts"
   },
   "dependencies": {
     "@cloudflare/workers-oauth-provider": "^0.3.1",
@@ -16,9 +16,16 @@
     "zod": "^4.0.0"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.0.0",
+    "@cloudflare/vitest-pool-workers": "^0.16.18",
+    "@cloudflare/workers-types": "4.20260404.1",
     "typescript": "^5.5.0",
     "vitest": "^4.1.9",
     "wrangler": "^4.0.0"
+  },
+  "comments": {
+    "overrides": "Pin @cloudflare/workers-types across the tree to the version the worker code is written against; the vitest-pool-workers toolchain otherwise pulls a newer one whose stricter AI binding types break src/rerank.ts (types-only; runtime unaffected)."
+  },
+  "overrides": {
+    "@cloudflare/workers-types": "$@cloudflare/workers-types"
   }
 }

src/fts.test.ts

@@ -1,5 +1,36 @@
 import { describe, it, expect } from "vitest";
-import { toRankMap, reciprocalRankFusion } from "./fts.js";
+import {
+  toRankMap,
+  reciprocalRankFusion,
+  escapeFtsQuery,
+  tokenizerKindForType,
+} from "./fts.js";
+
+describe("fts: tokenizerKindForType", () => {
+  it("routes diff to the trigram (code) tokenizer and everything else to nat", () => {
+    expect(tokenizerKindForType("diff")).toBe("code");
+    expect(tokenizerKindForType("issue")).toBe("nat");
+    expect(tokenizerKindForType("release")).toBe("nat");
+    expect(tokenizerKindForType("wiki_doc")).toBe("nat");
+  });
+});
+
+describe("fts: escapeFtsQuery", () => {
+  it("quotes each whitespace-separated token", () => {
+    expect(escapeFtsQuery("hello world")).toBe('"hello" "world"');
+    expect(escapeFtsQuery("single")).toBe('"single"');
+  });
+
+  it("collapses empty / whitespace-only input to empty string", () => {
+    expect(escapeFtsQuery("")).toBe("");
+    expect(escapeFtsQuery("   \n\t ")).toBe("");
+  });
+
+  it("escapes embedded double quotes by doubling them (FTS5 syntax)", () => {
+    // token `"hi"` -> inner quotes doubled -> wrapped -> `"""hi"""`
+    expect(escapeFtsQuery('say "hi"')).toBe('"say" """hi"""');
+  });
+});
 
 describe("fts: toRankMap", () => {
   it("assigns 1-based ranks in best-first order", () => {

src/fts.workers.test.ts

@@ -0,0 +1,134 @@
+import { describe, it, expect, beforeAll } from "vitest";
+import { env, applyD1Migrations } from "cloudflare:test";
+import { upsertFtsRow, queryFts, deleteFtsRow, type FtsUpsertRow } from "./fts.js";
+
+// One shared local D1 (isolatedStorage:false — per-test snapshotting corrupts FTS5
+// external-content shadow tables). Tests therefore share a DB and do NOT clean up:
+//   - every vector_id is globally unique, so no cross-test ON CONFLICT UPDATE fires
+//     (a stray cross-test update on the FTS5 external-content table is what corrupts
+//     the vtab — SQLITE_CORRUPT_VTAB);
+//   - every query is scoped with the `repo` filter so accumulated rows from other
+//     tests are excluded by the SQL WHERE clause.
+beforeAll(async () => {
+  await applyD1Migrations(env.DB_FTS, env.TEST_MIGRATIONS);
+});
+
+function mkRow(
+  overrides: Partial<FtsUpsertRow> & Pick<FtsUpsertRow, "vectorId" | "type" | "content" | "repo">,
+): FtsUpsertRow {
+  return {
+    state: "open",
+    labels: "",
+    milestone: "",
+    assignees: "",
+    updatedAt: "2026-01-01T00:00:00Z",
+    ...overrides,
+  };
+}
+
+describe("fts D1: upsert + query (natural-language / porter)", () => {
+  it("indexes a nat row and matches it by word", async () => {
+    const repo = "t/nat-match";
+    await upsertFtsRow(
+      env.DB_FTS,
+      mkRow({ vectorId: "i:nat-match", type: "issue", repo, content: "authentication failure in the login handler" }),
+    );
+    const hits = await queryFts(env.DB_FTS, "authentication", 10, { repo });
+    expect(hits.map((h) => h.vectorId)).toEqual(["i:nat-match"]);
+    expect(hits[0].repo).toBe(repo);
+    expect(hits[0].type).toBe("issue");
+  });
+
+  it("does not match unrelated queries", async () => {
+    const repo = "t/nat-nomatch";
+    await upsertFtsRow(
+      env.DB_FTS,
+      mkRow({ vectorId: "i:nat-nomatch", type: "issue", repo, content: "authentication failure in the login handler" }),
+    );
+    expect(await queryFts(env.DB_FTS, "kubernetes", 10, { repo })).toEqual([]);
+  });
+});
+
+describe("fts D1: diff rows via trigram (the #135 surface)", () => {
+  it("indexes a diff row (code tokenizer) and matches an identifier substring", async () => {
+    const repo = "t/diff-match";
+    await upsertFtsRow(
+      env.DB_FTS,
+      mkRow({
+        vectorId: "c:diff-match",
+        type: "diff",
+        repo,
+        commitSha: "abc123",
+        filePath: "src/auth.ts",
+        content: "export function handleLoginCallback(req) { return verify(req); }",
+      }),
+    );
+    const hits = await queryFts(env.DB_FTS, "handleLogin", 10, { repo });
+    expect(hits.map((h) => h.vectorId)).toContain("c:diff-match");
+  });
+
+  it("re-upserting the same vector_id re-syncs the FTS mirror (ON CONFLICT + update trigger)", async () => {
+    // The exact regression class behind #135: repeated upsert of a diff row must
+    // re-sync the FTS5 mirror, not leave stale content searchable or duplicate rows.
+    const repo = "t/diff-dup";
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "c:diff-dup", type: "diff", repo, content: "alpha bravo charlie" }));
+    expect((await queryFts(env.DB_FTS, "bravo", 10, { repo })).map((h) => h.vectorId)).toContain("c:diff-dup");
+
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "c:diff-dup", type: "diff", repo, content: "delta echo foxtrot" }));
+    expect(await queryFts(env.DB_FTS, "bravo", 10, { repo })).toEqual([]); // stale content gone
+    expect((await queryFts(env.DB_FTS, "echo", 10, { repo })).map((h) => h.vectorId)).toContain("c:diff-dup"); // new found
+
+    const row = await env.DB_FTS.prepare("SELECT COUNT(*) AS n FROM search_docs WHERE vector_id = ?")
+      .bind("c:diff-dup")
+      .first<{ n: number }>();
+    expect(row?.n).toBe(1); // upsert, not duplicate insert
+  });
+});
+
+describe("fts D1: deleteFtsRow", () => {
+  it("removes the row and its FTS mirror (no orphan hit)", async () => {
+    const repo = "t/del";
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:del", type: "issue", repo, content: "deletable indexing entry" }));
+    expect((await queryFts(env.DB_FTS, "deletable", 10, { repo })).length).toBe(1);
+
+    await deleteFtsRow(env.DB_FTS, "i:del");
+    expect(await queryFts(env.DB_FTS, "deletable", 10, { repo })).toEqual([]);
+  });
+});
+
+describe("fts D1: structured filters", () => {
+  it("filters by repo", async () => {
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:repo-a", type: "issue", repo: "t/repo-alpha", content: "shared keyword token" }));
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:repo-b", type: "issue", repo: "t/repo-bravo", content: "shared keyword token" }));
+    const hits = await queryFts(env.DB_FTS, "keyword", 10, { repo: "t/repo-alpha" });
+    expect(hits.map((h) => h.vectorId)).toEqual(["i:repo-a"]);
+  });
+
+  it("filters by type within a repo", async () => {
+    const repo = "t/type-filter";
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:type-iss", type: "issue", repo, content: "common search term alpha" }));
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "r:type-rel", type: "release", repo, content: "common search term alpha" }));
+    const hits = await queryFts(env.DB_FTS, "common", 10, { repo, type: "release" });
+    expect(hits.map((h) => h.vectorId)).toEqual(["r:type-rel"]);
+  });
+});
+
+describe("fts D1: queryFts edge cases", () => {
+  it("returns [] for an empty / whitespace query (no MATCH)", async () => {
+    const repo = "t/empty";
+    await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:empty", type: "issue", repo, content: "something searchable" }));
+    expect(await queryFts(env.DB_FTS, "", 10, { repo })).toEqual([]);
+    expect(await queryFts(env.DB_FTS, "   ", 10, { repo })).toEqual([]);
+  });
+
+  it("respects topK", async () => {
+    const repo = "t/topk";
+    for (let i = 0; i < 5; i++) {
+      await upsertFtsRow(
+        env.DB_FTS,
+        mkRow({ vectorId: `i:topk-${i}`, type: "issue", repo, content: `repeated common token number ${i}` }),
+      );
+    }
+    expect((await queryFts(env.DB_FTS, "common", 2, { repo })).length).toBe(2);
+  });
+});

src/test-env.d.ts

@@ -0,0 +1,24 @@
+// Ambient declaration of the vitest-pool-workers `cloudflare:test` virtual module,
+// scoped to just what the *.workers.test.ts files use. Declared by hand (instead of
+// `/// <reference types="@cloudflare/vitest-pool-workers/types" />`) so the worker's
+// `tsc --noEmit` does not pull a newer @cloudflare/workers-types whose stricter AI
+// binding types conflict with existing code (src/rerank.ts). D1Database is global
+// via the worker's @cloudflare/workers-types.
+
+interface TestD1Migration {
+  name: string;
+  queries: string[];
+}
+
+declare module "cloudflare:test" {
+  interface ProvidedEnv {
+    DB_FTS: D1Database;
+    TEST_MIGRATIONS: TestD1Migration[];
+  }
+  export const env: ProvidedEnv;
+  export function applyD1Migrations(
+    db: D1Database,
+    migrations: TestD1Migration[],
+    migrationsTableName?: string,
+  ): Promise<void>;
+}

vitest.config.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from "vitest/config";
+
+// Node project: binding-independent unit tests (slice 1).
+// Workers-pool D1 integration tests live in *.workers.test.ts and run under
+// vitest.workers.config.ts (a separate workerd pool). They are excluded here so
+// the node run does not try to resolve the `cloudflare:test` virtual module.
+export default defineConfig({
+  test: {
+    include: ["src/**/*.test.ts"],
+    exclude: ["**/node_modules/**", "src/**/*.workers.test.ts"],
+  },
+});

vitest.workers.config.ts

@@ -0,0 +1,35 @@
+import { defineConfig } from "vitest/config";
+import { cloudflarePool, cloudflareTest, readD1Migrations } from "@cloudflare/vitest-pool-workers";
+
+// Workers-realistic pool: runs *.workers.test.ts inside workerd with a real local
+// D1 (miniflare-backed SQLite). The D1 migrations under ./migrations are read at
+// config time and handed to the test via the TEST_MIGRATIONS binding, then applied
+// with applyD1Migrations(). Vectorize / Workers AI are intentionally NOT bound here
+// — they have no local emulation, so this pool covers the D1 / FTS5 surface only.
+//
+// vitest 4 wiring: cloudflareTest() is a Vite plugin (provides the `cloudflare:test`
+// module + transforms); cloudflarePool() is the pool runner for `test.pool`.
+export default defineConfig(async () => {
+  const migrations = await readD1Migrations("./migrations");
+  const workersOptions = {
+    singleWorker: true,
+    // One shared local D1 for the whole file, no per-test snapshotting. The
+    // isolatedStorage snapshot/restore and bulk DELETE both corrupt FTS5
+    // external-content shadow tables (SQLITE_CORRUPT_VTAB), so instead each test
+    // scopes itself to its own repo via the queryFts repo filter.
+    isolatedStorage: false,
+    miniflare: {
+      compatibilityDate: "2025-03-26",
+      compatibilityFlags: ["nodejs_compat"],
+      d1Databases: { DB_FTS: "test-fts" },
+      bindings: { TEST_MIGRATIONS: migrations },
+    },
+  };
+  return {
+    plugins: [cloudflareTest(workersOptions)],
+    test: {
+      include: ["src/**/*.workers.test.ts"],
+      pool: cloudflarePool(workersOptions),
+    },
+  };
+});