From 103429364957651d21504a16d2623c6579f52339 Mon Sep 17 00:00:00 2001 From: Claude Lin & Lay Date: Sun, 21 Jun 2026 10:04:55 +0900 Subject: [PATCH] test: add Workers-pool D1/FTS5 integration tests (test-leg slice 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second slice of the CI test leg (#165), on the Workers-realistic harness: @cloudflare/vitest-pool-workers runs tests inside workerd against a real local D1 (miniflare). Migrations from ./migrations are applied via applyD1Migrations. Covers the fts.ts D1 surface, including the #135 regression class (diff-row FTS5 upsert re-sync): - upsertFtsRow + queryFts across both tokenizers (porter / trigram) - ON CONFLICT re-upsert re-syncs the FTS5 mirror (no stale hits, no duplicate rows) - deleteFtsRow removes the row and its FTS mirror - repo / type filters, empty-query short-circuit, topK Plus pure escapeFtsQuery / tokenizerKindForType added to the node suite. Config notes: - two vitest configs: node (vitest.config.ts, slice-1 unit tests) + workers pool (vitest.workers.config.ts); `npm test` runs both. - isolatedStorage off + globally-unique vector_ids per test: the FTS5 external-content vtab corrupts under per-test snapshots and under cross-test ON CONFLICT updates. - @cloudflare/workers-types pinned via overrides to the version the worker is written against; the toolchain otherwise bumps it and breaks src/rerank.ts typing. Workers バインディング非依存の次として、実機寄り harness で D1/FTS5（#135 の面）を 統合テストした。Vectorize/AI はローカル emulation が無いため対象外で #165 に残す。 Refs #165 --- package-lock.json | 366 +++++++++++++++++++++------------------ package.json | 11 +- src/fts.test.ts | 33 +++- src/fts.workers.test.ts | 134 ++++++++++++++ src/test-env.d.ts | 24 +++ vitest.config.ts | 12 ++ vitest.workers.config.ts | 35 ++++ 7 files changed, 448 insertions(+), 167 deletions(-) create mode 100644 src/fts.workers.test.ts create mode 100644 src/test-env.d.ts create mode 100644 vitest.config.ts create mode 100644 vitest.workers.config.ts diff --git a/package-lock.json b/package-lock.json index 1745428..4517295 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,8 @@ "zod": "^4.0.0" }, "devDependencies": { - "@cloudflare/workers-types": "^4.0.0", + "@cloudflare/vitest-pool-workers": "^0.16.18", + "@cloudflare/workers-types": "4.20260404.1", "typescript": "^5.5.0", "vitest": "^4.1.9", "wrangler": "^4.0.0" @@ -475,24 +476,24 @@ "license": "MIT" }, "node_modules/@cloudflare/kv-asset-handler": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.4.2.tgz", - "integrity": "sha512-SIOD2DxrRRwQ+jgzlXCqoEFiKOFqaPjhnNTGKXSRLvp1HiOvapLaFG2kEr9dYQTYe8rKrd9uvDUzmAITeNyaHQ==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", + "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", "dev": true, "license": "MIT OR Apache-2.0", "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" } }, "node_modules/@cloudflare/unenv-preset": { - "version": "2.16.0", - "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.0.tgz", - "integrity": "sha512-8ovsRpwzPoEqPUzoErAYVv8l3FMZNeBVQfJTvtzP4AgLSRGZISRfuChFxHWUQd3n6cnrwkuTGxT+2cGo8EsyYg==", + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", + "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", "dev": true, "license": "MIT OR Apache-2.0", "peerDependencies": { "unenv": "2.0.0-rc.24", - "workerd": "1.20260301.1 || ~1.20260302.1 || ~1.20260303.1 || ~1.20260304.1 || >1.20260305.0 <2.0.0-0" + "workerd": ">1.20260305.0 <2.0.0-0" }, "peerDependenciesMeta": { "workerd": { @@ -500,10 +501,39 @@ } } }, + "node_modules/@cloudflare/vitest-pool-workers": { + "version": "0.16.18", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-pool-workers/-/vitest-pool-workers-0.16.18.tgz", + "integrity": "sha512-TEktXyevK9lkTWouElbIcDPK3YEfV+Szqgnlq5sNk+KYZR3LiDdYDaGNmUYgiT2LiiFeGU2yzCrcgmN8mJhqWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "cjs-module-lexer": "1.2.3", + "esbuild": "0.28.1", + "miniflare": "4.20260617.1", + "wrangler": "4.103.0", + "zod": "3.25.76" + }, + "peerDependencies": { + "@vitest/runner": "^4.1.0", + "@vitest/snapshot": "^4.1.0", + "vitest": "^4.1.0" + } + }, + "node_modules/@cloudflare/vitest-pool-workers/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260401.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260401.1.tgz", - "integrity": "sha512-ZSmceM70jH6k+/62VkEcmMNzrpr4kSctkX5Lsgqv38KktfhPY/hsh75y1lRoPWS3H3kgMa4p2pUSlidZR1u2hw==", + "version": "1.20260617.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260617.1.tgz", + "integrity": "sha512-jWwmgEVVWbsHNrLSNXzwjJaH90VzRxq1cWkQFUidxyeUPnMxemeNE8I9qFAfrpzGgE11e9sKDcE3ettJW08swQ==", "cpu": [ "x64" ], @@ -518,9 +548,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260401.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260401.1.tgz", - "integrity": "sha512-7UKWF+IUZ3NXMVPsDg8Cjg0r58b+uYlfvs5Yt8bvtU+geCtW4P2MxRHmRSEo8SryckXOJjb/b8tcncgCykFu8g==", + "version": "1.20260617.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260617.1.tgz", + "integrity": "sha512-LHH7b565g9znfCUOkwbec6FG2rmRbsgCy6aJiU9KN662mNheWl5sw/iKleiFSiljPKQQP3HkjnC/NSkdgi/aSA==", "cpu": [ "arm64" ], @@ -535,9 +565,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260401.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260401.1.tgz", - "integrity": "sha512-MDWUH/0bvL/l9aauN8zEddyYOXId1OueqrUCXXENNJ95R/lSmF6OgGVuXaYhoIhxQkNiEJ/0NOlnVYj9mJq4dw==", + "version": "1.20260617.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260617.1.tgz", + "integrity": "sha512-FMnaAKXe4Cfd8TQurCVd9fs2XQVBFRCsP+Id/SRdUv89MlwYu9zXfoyx6BxM+brPTIUK38SHbo8iaxiwzLi9JQ==", "cpu": [ "x64" ], @@ -552,9 +582,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260401.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260401.1.tgz", - "integrity": "sha512-UgkzpMzVWM/bwbo3vjCTg2aoKfGcUhiEoQoDdo6RGWvbHRJyLVZ4VQCG9ZcISiztkiS2ICCoYOtPy6M/lV6Gcw==", + "version": "1.20260617.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260617.1.tgz", + "integrity": "sha512-MRoifFYcqbxxIIQy7PqO5tFY/qPFSnjXzakWl0sO93l+HLyG35jRAgOi6jfqa4kBxc7gKKtH861DcewjxUfkjA==", "cpu": [ "arm64" ], @@ -569,9 +599,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260401.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260401.1.tgz", - "integrity": "sha512-HBLzcQF5iF4Qv20tQ++pG7xs3OsCnaIbc+GAi6fmhUKZhvmzvml/jwrQzLJ+MPm0cQo41K5OO/U3T4S8tvJetQ==", + "version": "1.20260617.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260617.1.tgz", + "integrity": "sha512-rgBV9wQrv0OSKgCTTbhFUFY3sLGNANZ88aqaLvtmEn2gmbFVb1J4PDGochVUdB7NSEp4D/ghHva6/8SZmbONpw==", "cpu": [ "x64" ], @@ -653,9 +683,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", "cpu": [ "ppc64" ], @@ -669,9 +699,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", "cpu": [ "arm" ], @@ -685,9 +715,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", "cpu": [ "arm64" ], @@ -701,9 +731,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", "cpu": [ "x64" ], @@ -717,9 +747,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", "cpu": [ "arm64" ], @@ -733,9 +763,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", "cpu": [ "x64" ], @@ -749,9 +779,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", "cpu": [ "arm64" ], @@ -765,9 +795,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", "cpu": [ "x64" ], @@ -781,9 +811,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", "cpu": [ "arm" ], @@ -797,9 +827,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", "cpu": [ "arm64" ], @@ -813,9 +843,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", "cpu": [ "ia32" ], @@ -829,9 +859,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", "cpu": [ "loong64" ], @@ -845,9 +875,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", "cpu": [ "mips64el" ], @@ -861,9 +891,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", "cpu": [ "ppc64" ], @@ -877,9 +907,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", "cpu": [ "riscv64" ], @@ -893,9 +923,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", "cpu": [ "s390x" ], @@ -909,9 +939,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", "cpu": [ "x64" ], @@ -925,9 +955,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", "cpu": [ "arm64" ], @@ -941,9 +971,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", "cpu": [ "x64" ], @@ -957,9 +987,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", "cpu": [ "arm64" ], @@ -973,9 +1003,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", "cpu": [ "x64" ], @@ -989,9 +1019,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", "cpu": [ "arm64" ], @@ -1005,9 +1035,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", "cpu": [ "x64" ], @@ -1021,9 +1051,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", "cpu": [ "arm64" ], @@ -1037,9 +1067,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", "cpu": [ "ia32" ], @@ -1053,9 +1083,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", "cpu": [ "x64" ], @@ -2623,6 +2653,13 @@ "node": ">=18" } }, + "node_modules/cjs-module-lexer": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz", + "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", + "dev": true, + "license": "MIT" + }, "node_modules/cliui": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", @@ -2860,9 +2897,9 @@ } }, "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", "devOptional": true, "hasInstallScript": true, "license": "MIT", @@ -2873,32 +2910,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" } }, "node_modules/escalade": { @@ -3830,24 +3867,24 @@ } }, "node_modules/miniflare": { - "version": "4.20260401.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260401.0.tgz", - "integrity": "sha512-lngHPzZFN9sxYG/mhzvnWiBMNVAN5MsO/7g32ttJ07rymtiK/ZBalODTKb8Od+BQdlU5DOR4CjVt9NydjnUyYg==", + "version": "4.20260617.1", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260617.1.tgz", + "integrity": "sha512-Go3/gzStm99QHptsSgU+q1S+xDfLoRgwjJNY80kaTVi0ENhTyqKq+sc4xZiWBSbM7uUcJwmzm8+QFKtcYLJ9nw==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", - "sharp": "^0.34.5", - "undici": "7.24.4", - "workerd": "1.20260401.1", - "ws": "8.18.0", + "sharp": "0.34.5", + "undici": "7.28.0", + "workerd": "1.20260617.1", + "ws": "8.21.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" } }, "node_modules/minimist": { @@ -4619,9 +4656,9 @@ } }, "node_modules/undici": { - "version": "7.24.4", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.4.tgz", - "integrity": "sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", "dev": true, "license": "MIT", "engines": { @@ -4888,9 +4925,9 @@ } }, "node_modules/workerd": { - "version": "1.20260401.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260401.1.tgz", - "integrity": "sha512-mUYCd+ohaWJWF5nhDzxugWaAD/DM8Dw0ze3B7bu8JaA7S70+XQJXcvcvwE8C4qGcxSdCyqjsrFzqxKubECDwzg==", + "version": "1.20260617.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260617.1.tgz", + "integrity": "sha512-Re5pl6pdowt3ZmWUzGlOuB7jbRIIPetgKalmo4cYmucQnVhpo7/3e4MfpekbhLi2EhZZz5EY9NWRu8zFzuEZew==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", @@ -4901,41 +4938,42 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260401.1", - "@cloudflare/workerd-darwin-arm64": "1.20260401.1", - "@cloudflare/workerd-linux-64": "1.20260401.1", - "@cloudflare/workerd-linux-arm64": "1.20260401.1", - "@cloudflare/workerd-windows-64": "1.20260401.1" + "@cloudflare/workerd-darwin-64": "1.20260617.1", + "@cloudflare/workerd-darwin-arm64": "1.20260617.1", + "@cloudflare/workerd-linux-64": "1.20260617.1", + "@cloudflare/workerd-linux-arm64": "1.20260617.1", + "@cloudflare/workerd-windows-64": "1.20260617.1" } }, "node_modules/wrangler": { - "version": "4.80.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.80.0.tgz", - "integrity": "sha512-2ZKF7uPeOZy65BGk3YfvqBCPo/xH1MrAlMmH9mVP+tCNBrTUMnwOHSj1HrZHgR8LttkAqhko0fGz+I4ax1rzyQ==", + "version": "4.103.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.103.0.tgz", + "integrity": "sha512-3Lv1P5t2xcSEkSTKtG+Lz+3JFryuU7YPLkaCUj7gNe+CJsjZJLtUwqsh1x595QBxkIbCE0GAvDx2DCJUU4+oqw==", "dev": true, "license": "MIT OR Apache-2.0", "dependencies": { - "@cloudflare/kv-asset-handler": "0.4.2", - "@cloudflare/unenv-preset": "2.16.0", + "@cloudflare/kv-asset-handler": "0.5.0", + "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", - "esbuild": "0.27.3", - "miniflare": "4.20260401.0", + "esbuild": "0.28.1", + "miniflare": "4.20260617.1", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20260401.1" + "workerd": "1.20260617.1" }, "bin": { + "cf-wrangler": "bin/cf-wrangler.js", "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js" }, "engines": { - "node": ">=20.3.0" + "node": ">=22.0.0" }, "optionalDependencies": { - "fsevents": "~2.3.2" + "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^4.20260401.1" + "@cloudflare/workers-types": "^4.20260617.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { @@ -4974,9 +5012,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 36da79d..0b0a70f 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "scripts": { "dev": "wrangler dev", "deploy": "wrangler deploy", - "test": "vitest run" + "test": "vitest run && vitest run --config vitest.workers.config.ts" }, "dependencies": { "@cloudflare/workers-oauth-provider": "^0.3.1", @@ -16,9 +16,16 @@ "zod": "^4.0.0" }, "devDependencies": { - "@cloudflare/workers-types": "^4.0.0", + "@cloudflare/vitest-pool-workers": "^0.16.18", + "@cloudflare/workers-types": "4.20260404.1", "typescript": "^5.5.0", "vitest": "^4.1.9", "wrangler": "^4.0.0" + }, + "comments": { + "overrides": "Pin @cloudflare/workers-types across the tree to the version the worker code is written against; the vitest-pool-workers toolchain otherwise pulls a newer one whose stricter AI binding types break src/rerank.ts (types-only; runtime unaffected)." + }, + "overrides": { + "@cloudflare/workers-types": "$@cloudflare/workers-types" } } diff --git a/src/fts.test.ts b/src/fts.test.ts index 304ae04..c8d1036 100644 --- a/src/fts.test.ts +++ b/src/fts.test.ts @@ -1,5 +1,36 @@ import { describe, it, expect } from "vitest"; -import { toRankMap, reciprocalRankFusion } from "./fts.js"; +import { + toRankMap, + reciprocalRankFusion, + escapeFtsQuery, + tokenizerKindForType, +} from "./fts.js"; + +describe("fts: tokenizerKindForType", () => { + it("routes diff to the trigram (code) tokenizer and everything else to nat", () => { + expect(tokenizerKindForType("diff")).toBe("code"); + expect(tokenizerKindForType("issue")).toBe("nat"); + expect(tokenizerKindForType("release")).toBe("nat"); + expect(tokenizerKindForType("wiki_doc")).toBe("nat"); + }); +}); + +describe("fts: escapeFtsQuery", () => { + it("quotes each whitespace-separated token", () => { + expect(escapeFtsQuery("hello world")).toBe('"hello" "world"'); + expect(escapeFtsQuery("single")).toBe('"single"'); + }); + + it("collapses empty / whitespace-only input to empty string", () => { + expect(escapeFtsQuery("")).toBe(""); + expect(escapeFtsQuery(" \n\t ")).toBe(""); + }); + + it("escapes embedded double quotes by doubling them (FTS5 syntax)", () => { + // token `"hi"` -> inner quotes doubled -> wrapped -> `"""hi"""` + expect(escapeFtsQuery('say "hi"')).toBe('"say" """hi"""'); + }); +}); describe("fts: toRankMap", () => { it("assigns 1-based ranks in best-first order", () => { diff --git a/src/fts.workers.test.ts b/src/fts.workers.test.ts new file mode 100644 index 0000000..b5d37f4 --- /dev/null +++ b/src/fts.workers.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { env, applyD1Migrations } from "cloudflare:test"; +import { upsertFtsRow, queryFts, deleteFtsRow, type FtsUpsertRow } from "./fts.js"; + +// One shared local D1 (isolatedStorage:false — per-test snapshotting corrupts FTS5 +// external-content shadow tables). Tests therefore share a DB and do NOT clean up: +// - every vector_id is globally unique, so no cross-test ON CONFLICT UPDATE fires +// (a stray cross-test update on the FTS5 external-content table is what corrupts +// the vtab — SQLITE_CORRUPT_VTAB); +// - every query is scoped with the `repo` filter so accumulated rows from other +// tests are excluded by the SQL WHERE clause. +beforeAll(async () => { + await applyD1Migrations(env.DB_FTS, env.TEST_MIGRATIONS); +}); + +function mkRow( + overrides: Partial & Pick, +): FtsUpsertRow { + return { + state: "open", + labels: "", + milestone: "", + assignees: "", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, + }; +} + +describe("fts D1: upsert + query (natural-language / porter)", () => { + it("indexes a nat row and matches it by word", async () => { + const repo = "t/nat-match"; + await upsertFtsRow( + env.DB_FTS, + mkRow({ vectorId: "i:nat-match", type: "issue", repo, content: "authentication failure in the login handler" }), + ); + const hits = await queryFts(env.DB_FTS, "authentication", 10, { repo }); + expect(hits.map((h) => h.vectorId)).toEqual(["i:nat-match"]); + expect(hits[0].repo).toBe(repo); + expect(hits[0].type).toBe("issue"); + }); + + it("does not match unrelated queries", async () => { + const repo = "t/nat-nomatch"; + await upsertFtsRow( + env.DB_FTS, + mkRow({ vectorId: "i:nat-nomatch", type: "issue", repo, content: "authentication failure in the login handler" }), + ); + expect(await queryFts(env.DB_FTS, "kubernetes", 10, { repo })).toEqual([]); + }); +}); + +describe("fts D1: diff rows via trigram (the #135 surface)", () => { + it("indexes a diff row (code tokenizer) and matches an identifier substring", async () => { + const repo = "t/diff-match"; + await upsertFtsRow( + env.DB_FTS, + mkRow({ + vectorId: "c:diff-match", + type: "diff", + repo, + commitSha: "abc123", + filePath: "src/auth.ts", + content: "export function handleLoginCallback(req) { return verify(req); }", + }), + ); + const hits = await queryFts(env.DB_FTS, "handleLogin", 10, { repo }); + expect(hits.map((h) => h.vectorId)).toContain("c:diff-match"); + }); + + it("re-upserting the same vector_id re-syncs the FTS mirror (ON CONFLICT + update trigger)", async () => { + // The exact regression class behind #135: repeated upsert of a diff row must + // re-sync the FTS5 mirror, not leave stale content searchable or duplicate rows. + const repo = "t/diff-dup"; + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "c:diff-dup", type: "diff", repo, content: "alpha bravo charlie" })); + expect((await queryFts(env.DB_FTS, "bravo", 10, { repo })).map((h) => h.vectorId)).toContain("c:diff-dup"); + + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "c:diff-dup", type: "diff", repo, content: "delta echo foxtrot" })); + expect(await queryFts(env.DB_FTS, "bravo", 10, { repo })).toEqual([]); // stale content gone + expect((await queryFts(env.DB_FTS, "echo", 10, { repo })).map((h) => h.vectorId)).toContain("c:diff-dup"); // new found + + const row = await env.DB_FTS.prepare("SELECT COUNT(*) AS n FROM search_docs WHERE vector_id = ?") + .bind("c:diff-dup") + .first<{ n: number }>(); + expect(row?.n).toBe(1); // upsert, not duplicate insert + }); +}); + +describe("fts D1: deleteFtsRow", () => { + it("removes the row and its FTS mirror (no orphan hit)", async () => { + const repo = "t/del"; + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:del", type: "issue", repo, content: "deletable indexing entry" })); + expect((await queryFts(env.DB_FTS, "deletable", 10, { repo })).length).toBe(1); + + await deleteFtsRow(env.DB_FTS, "i:del"); + expect(await queryFts(env.DB_FTS, "deletable", 10, { repo })).toEqual([]); + }); +}); + +describe("fts D1: structured filters", () => { + it("filters by repo", async () => { + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:repo-a", type: "issue", repo: "t/repo-alpha", content: "shared keyword token" })); + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:repo-b", type: "issue", repo: "t/repo-bravo", content: "shared keyword token" })); + const hits = await queryFts(env.DB_FTS, "keyword", 10, { repo: "t/repo-alpha" }); + expect(hits.map((h) => h.vectorId)).toEqual(["i:repo-a"]); + }); + + it("filters by type within a repo", async () => { + const repo = "t/type-filter"; + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:type-iss", type: "issue", repo, content: "common search term alpha" })); + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "r:type-rel", type: "release", repo, content: "common search term alpha" })); + const hits = await queryFts(env.DB_FTS, "common", 10, { repo, type: "release" }); + expect(hits.map((h) => h.vectorId)).toEqual(["r:type-rel"]); + }); +}); + +describe("fts D1: queryFts edge cases", () => { + it("returns [] for an empty / whitespace query (no MATCH)", async () => { + const repo = "t/empty"; + await upsertFtsRow(env.DB_FTS, mkRow({ vectorId: "i:empty", type: "issue", repo, content: "something searchable" })); + expect(await queryFts(env.DB_FTS, "", 10, { repo })).toEqual([]); + expect(await queryFts(env.DB_FTS, " ", 10, { repo })).toEqual([]); + }); + + it("respects topK", async () => { + const repo = "t/topk"; + for (let i = 0; i < 5; i++) { + await upsertFtsRow( + env.DB_FTS, + mkRow({ vectorId: `i:topk-${i}`, type: "issue", repo, content: `repeated common token number ${i}` }), + ); + } + expect((await queryFts(env.DB_FTS, "common", 2, { repo })).length).toBe(2); + }); +}); diff --git a/src/test-env.d.ts b/src/test-env.d.ts new file mode 100644 index 0000000..3b4f40b --- /dev/null +++ b/src/test-env.d.ts @@ -0,0 +1,24 @@ +// Ambient declaration of the vitest-pool-workers `cloudflare:test` virtual module, +// scoped to just what the *.workers.test.ts files use. Declared by hand (instead of +// `/// `) so the worker's +// `tsc --noEmit` does not pull a newer @cloudflare/workers-types whose stricter AI +// binding types conflict with existing code (src/rerank.ts). D1Database is global +// via the worker's @cloudflare/workers-types. + +interface TestD1Migration { + name: string; + queries: string[]; +} + +declare module "cloudflare:test" { + interface ProvidedEnv { + DB_FTS: D1Database; + TEST_MIGRATIONS: TestD1Migration[]; + } + export const env: ProvidedEnv; + export function applyD1Migrations( + db: D1Database, + migrations: TestD1Migration[], + migrationsTableName?: string, + ): Promise; +} diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..3a99c69 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +// Node project: binding-independent unit tests (slice 1). +// Workers-pool D1 integration tests live in *.workers.test.ts and run under +// vitest.workers.config.ts (a separate workerd pool). They are excluded here so +// the node run does not try to resolve the `cloudflare:test` virtual module. +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + exclude: ["**/node_modules/**", "src/**/*.workers.test.ts"], + }, +}); diff --git a/vitest.workers.config.ts b/vitest.workers.config.ts new file mode 100644 index 0000000..8e24ec7 --- /dev/null +++ b/vitest.workers.config.ts @@ -0,0 +1,35 @@ +import { defineConfig } from "vitest/config"; +import { cloudflarePool, cloudflareTest, readD1Migrations } from "@cloudflare/vitest-pool-workers"; + +// Workers-realistic pool: runs *.workers.test.ts inside workerd with a real local +// D1 (miniflare-backed SQLite). The D1 migrations under ./migrations are read at +// config time and handed to the test via the TEST_MIGRATIONS binding, then applied +// with applyD1Migrations(). Vectorize / Workers AI are intentionally NOT bound here +// — they have no local emulation, so this pool covers the D1 / FTS5 surface only. +// +// vitest 4 wiring: cloudflareTest() is a Vite plugin (provides the `cloudflare:test` +// module + transforms); cloudflarePool() is the pool runner for `test.pool`. +export default defineConfig(async () => { + const migrations = await readD1Migrations("./migrations"); + const workersOptions = { + singleWorker: true, + // One shared local D1 for the whole file, no per-test snapshotting. The + // isolatedStorage snapshot/restore and bulk DELETE both corrupt FTS5 + // external-content shadow tables (SQLITE_CORRUPT_VTAB), so instead each test + // scopes itself to its own repo via the queryFts repo filter. + isolatedStorage: false, + miniflare: { + compatibilityDate: "2025-03-26", + compatibilityFlags: ["nodejs_compat"], + d1Databases: { DB_FTS: "test-fts" }, + bindings: { TEST_MIGRATIONS: migrations }, + }, + }; + return { + plugins: [cloudflareTest(workersOptions)], + test: { + include: ["src/**/*.workers.test.ts"], + pool: cloudflarePool(workersOptions), + }, + }; +});