From 4c94b98b0fea019bfc6b519d868c85dff494b927 Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 10:54:56 +0530 Subject: [PATCH 1/7] eval: split benchmark cohorts by whether the task names the fixing file Three of twelve held-out cases contain an expected fixing path in the task text itself: mongoose as "Location: lib/document.js:2339", svelte and yargs as a GitHub permalink to the exact file and lines. Any ranker with an explicit-file-mention signal answers those by reading the task, so pooling them into one rate lets them carry the headline generalization number. Classification is derived at evaluation time from the same task text the ranker reads, never stored in dataset.json, so it cannot drift away from the case it describes. Three tiers are recorded because they are not equally strong: a repo-root-anchored full path, a multi-segment path suffix (a tsc error naming src/query/react/buildHooks.ts), and a bare basename. The first two count as named; a bare "index.ts" does not and stays in the unmentioned cohort. Datasets are unchanged and the pooled top-level rates are unchanged, so the published figures still reproduce. This only adds the cohort breakdown. Co-Authored-By: Claude Opus 5 --- benchmarks/external/results.json | 201 ++++++++++++++++++++++++-- benchmarks/heldout/results.json | 173 ++++++++++++++++++++-- scripts/evaluate-external.mjs | 36 ++++- scripts/lib/expected-path-mention.mjs | 155 ++++++++++++++++++++ 4 files changed, 536 insertions(+), 29 deletions(-) create mode 100644 scripts/lib/expected-path-mention.mjs diff --git a/benchmarks/external/results.json b/benchmarks/external/results.json index 93a8079..3bfbb56 100644 --- a/benchmarks/external/results.json +++ b/benchmarks/external/results.json @@ -61,6 +61,106 @@ ] } ], + "cohorts": { + "all": { + "cases": 16, + "top1HitRate": 0.688, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.444, + 0.858 + ], + "top3": [ + 0.806, + 1 + ], + "top5": [ + 0.806, + 1 + ] + }, + "slugs": [ + "expressjs/express", + "axios/axios", + "debug-js/debug", + "sindresorhus/ky", + "colinhacks/zod", + "pinojs/pino", + "fastify/fastify", + "chalk/chalk", + "vitest-dev/vitest", + "eslint/eslint", + "webpack/webpack", + "nodejs/undici", + "reduxjs/redux-toolkit", + "prettier/prettier", + "honojs/hono", + "sindresorhus/got" + ] + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.692, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.424, + 0.873 + ], + "top3": [ + 0.772, + 1 + ], + "top5": [ + 0.772, + 1 + ] + }, + "slugs": [ + "axios/axios", + "debug-js/debug", + "sindresorhus/ky", + "colinhacks/zod", + "pinojs/pino", + "fastify/fastify", + "chalk/chalk", + "vitest-dev/vitest", + "eslint/eslint", + "webpack/webpack", + "nodejs/undici", + "prettier/prettier", + "honojs/hono" + ] + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.667, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.208, + 0.939 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + }, + "slugs": [ + "expressjs/express", + "reduxjs/redux-toolkit", + "sindresorhus/got" + ] + } + }, "floors": { "top1": 0.3, "top3": 0.5, @@ -83,7 +183,17 @@ "topConfidence": "high", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "mentionEvidence": [ + { + "path": "lib/request.js", + "tier": "full-path", + "match": "lib/request.js", + "context": "…includes `QUERY`) **Any other relevant information**: `req.fresh` (`lib/request.js`) short-circuits to `false` for any method other than `…" + } + ] }, { "slug": "axios/axios", @@ -101,7 +211,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "debug-js/debug", @@ -119,7 +232,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "sindresorhus/ky", @@ -137,7 +253,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "colinhacks/zod", @@ -155,7 +274,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "pinojs/pino", @@ -174,7 +296,10 @@ "topConfidence": "high", "top1": false, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "fastify/fastify", @@ -192,7 +317,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "chalk/chalk", @@ -210,7 +338,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "vitest-dev/vitest", @@ -228,7 +359,10 @@ "topConfidence": "low", "top1": false, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "eslint/eslint", @@ -246,7 +380,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "webpack/webpack", @@ -264,7 +401,10 @@ "topConfidence": "medium", "top1": false, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "nodejs/undici", @@ -282,7 +422,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "reduxjs/redux-toolkit", @@ -300,7 +443,17 @@ "topConfidence": "medium", "top1": false, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": true, + "mentionTier": "path-suffix", + "mentionEvidence": [ + { + "path": "packages/toolkit/src/query/react/buildHooks.ts", + "tier": "path-suffix", + "match": "src/query/react/buildHooks.ts", + "context": "…tsgo`, and got these errors: ``` Version 7.0.0-dev.20251207.1 Error: src/query/react/buildHooks.ts(1823,13): error TS2769: No overload matc…" + } + ] }, { "slug": "prettier/prettier", @@ -318,7 +471,10 @@ "topConfidence": "low", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "honojs/hono", @@ -336,7 +492,10 @@ "topConfidence": "medium", "top1": false, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "sindresorhus/got", @@ -354,7 +513,17 @@ "topConfidence": "high", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "mentionEvidence": [ + { + "path": "source/core/index.ts", + "tier": "full-path", + "match": "source/core/index.ts", + "context": "…ub.com/sindresorhus/got/blob/e5e645a7d6deeec02933bf474727a541775772c7/source/core/index.ts#L1088-L1089 and the shouldDropBody flag https://…" + } + ] } ] } diff --git a/benchmarks/heldout/results.json b/benchmarks/heldout/results.json index 67dc525..0643ea6 100644 --- a/benchmarks/heldout/results.json +++ b/benchmarks/heldout/results.json @@ -58,6 +58,98 @@ ] } ], + "cohorts": { + "all": { + "cases": 12, + "top1HitRate": 0.583, + "top3HitRate": 0.667, + "top5HitRate": 0.75, + "intervals95": { + "top1": [ + 0.32, + 0.807 + ], + "top3": [ + 0.391, + 0.862 + ], + "top5": [ + 0.468, + 0.911 + ] + }, + "slugs": [ + "Automattic/mongoose", + "immerjs/immer", + "jestjs/jest", + "knex/knex", + "mochajs/mocha", + "react-hook-form/react-hook-form", + "socketio/socket.io", + "sveltejs/svelte", + "vitejs/vite", + "vuejs/core", + "winstonjs/winston", + "yargs/yargs" + ] + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.444, + "top3HitRate": 0.556, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.189, + 0.733 + ], + "top3": [ + 0.267, + 0.811 + ], + "top5": [ + 0.354, + 0.879 + ] + }, + "slugs": [ + "immerjs/immer", + "jestjs/jest", + "knex/knex", + "mochajs/mocha", + "react-hook-form/react-hook-form", + "socketio/socket.io", + "vitejs/vite", + "vuejs/core", + "winstonjs/winston" + ] + }, + "mentioned": { + "cases": 3, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + }, + "slugs": [ + "Automattic/mongoose", + "sveltejs/svelte", + "yargs/yargs" + ] + } + }, "floors": { "top1": 0.3, "top3": 0.5, @@ -80,7 +172,17 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "mentionEvidence": [ + { + "path": "lib/document.js", + "tier": "full-path", + "match": "lib/document.js", + "context": "…oose's own validation) **Repo:** `Automattic/mongoose` **Location:** lib/document.js:2339 (isModified -> modifiedPaths rebuild) called per-…" + } + ] }, { "slug": "immerjs/immer", @@ -98,7 +200,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "jestjs/jest", @@ -116,7 +221,10 @@ "topConfidence": "medium", "top1": false, "top3": false, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "knex/knex", @@ -134,7 +242,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "mochajs/mocha", @@ -152,7 +263,10 @@ "topConfidence": "medium", "top1": false, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "react-hook-form/react-hook-form", @@ -170,7 +284,10 @@ "topConfidence": "medium", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "socketio/socket.io", @@ -188,7 +305,10 @@ "topConfidence": "medium", "top1": false, "top3": false, - "top5Hit": false + "top5Hit": false, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "sveltejs/svelte", @@ -206,7 +326,17 @@ "topConfidence": "high", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "mentionEvidence": [ + { + "path": "packages/svelte/src/internal/client/dom/blocks/boundary.js", + "tier": "full-path", + "match": "packages/svelte/src/internal/client/dom/blocks/boundary.js", + "context": "…hub.com/sveltejs/svelte/blob/4a6a85b5f149cc96514ed3bf5e59083b9246d394/packages/svelte/src/internal/client/dom/blocks/boundary.js#L200-L210))…" + } + ] }, { "slug": "vitejs/vite", @@ -224,7 +354,10 @@ "topConfidence": "low", "top1": false, "top3": false, - "top5Hit": false + "top5Hit": false, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "vuejs/core", @@ -242,7 +375,10 @@ "topConfidence": "high", "top1": false, "top3": false, - "top5Hit": false + "top5Hit": false, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "winstonjs/winston", @@ -260,7 +396,10 @@ "topConfidence": "low", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": false, + "mentionTier": "none", + "mentionEvidence": [] }, { "slug": "yargs/yargs", @@ -278,7 +417,17 @@ "topConfidence": "high", "top1": true, "top3": true, - "top5Hit": true + "top5Hit": true, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "mentionEvidence": [ + { + "path": "lib/utils/apply-extends.ts", + "tier": "full-path", + "match": "lib/utils/apply-extends.ts", + "context": "…ability was identified in the `apply-extends` helper function within `lib/utils/apply-extends.ts`. This function is responsible for merging…" + } + ] } ] } diff --git a/scripts/evaluate-external.mjs b/scripts/evaluate-external.mjs index de4dc58..ec13d17 100644 --- a/scripts/evaluate-external.mjs +++ b/scripts/evaluate-external.mjs @@ -19,6 +19,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { fileURLToPath, pathToFileURL } from "node:url"; import { dirname, join, resolve } from "node:path"; import { materializePinnedRepository } from "./lib/external-cache.mjs"; +import { classifyExpectedPathMention, splitCohorts } from "./lib/expected-path-mention.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const { scanRepo, rankContextFiles } = await import(pathToFileURL(join(repoRoot, "packages", "core", "dist", "index.js")).href); @@ -49,6 +50,7 @@ for (const benchmark of dataset.cases) { } const ranked = rankContextFiles(repo, { issueText: benchmark.task }, 5); const paths = ranked.map((file) => file.path); + const mention = classifyExpectedPathMention(benchmark); results.push({ slug: benchmark.slug, issue: benchmark.issue, @@ -57,7 +59,12 @@ for (const benchmark of dataset.cases) { topConfidence: ranked[0]?.confidence ?? null, top1: benchmark.expected.includes(paths[0]), top3: benchmark.expected.some((path) => paths.slice(0, 3).includes(path)), - top5Hit: benchmark.expected.some((path) => paths.includes(path)) + top5Hit: benchmark.expected.some((path) => paths.includes(path)), + // Derived every run from the same task text the ranker reads, never stored in the + // dataset, so the cohort split cannot drift away from the case it describes. + mentionsExpectedPath: mention.mentionsExpectedPath, + mentionTier: mention.mentionTier, + mentionEvidence: mention.evidence }); } @@ -111,6 +118,32 @@ const calibration = ["high", "medium", "low"].map((confidence) => { }; }); +// Cases whose task text already names the fixing file are answerable by reading the task +// rather than by ranking the repository, so they measure the explicit-mention signal, not +// generalization. Scoring them in one pooled number lets a handful of them carry the +// headline rate. `unmentioned` is the number that estimates behaviour on a task that does +// not already contain its own answer, and it is the one to plan around. +function scoreCohort(cohort) { + const hitRate = (key) => + cohort.length === 0 ? null : Number((cohort.filter((result) => result[key]).length / cohort.length).toFixed(3)); + const interval = (key) => wilsonInterval(cohort.filter((result) => result[key]).length, cohort.length); + return { + cases: cohort.length, + top1HitRate: hitRate("top1"), + top3HitRate: hitRate("top3"), + top5HitRate: hitRate("top5Hit"), + intervals95: { top1: interval("top1"), top3: interval("top3"), top5: interval("top5Hit") }, + slugs: cohort.map((result) => result.slug) + }; +} + +const cohortGroups = splitCohorts(results); +const cohorts = { + all: scoreCohort(cohortGroups.all), + unmentioned: scoreCohort(cohortGroups.unmentioned), + mentioned: scoreCohort(cohortGroups.mentioned) +}; + const summary = { cases: results.length, top1HitRate: Number(rate("top1").toFixed(3)), @@ -128,6 +161,7 @@ const summary = { slugs: misleadingCases.map((result) => result.slug) }, calibration, + cohorts, floors: FLOORS, results }; diff --git a/scripts/lib/expected-path-mention.mjs b/scripts/lib/expected-path-mention.mjs new file mode 100644 index 0000000..c351099 --- /dev/null +++ b/scripts/lib/expected-path-mention.mjs @@ -0,0 +1,155 @@ +// Detects benchmark cases whose task text already names the file the fix changed. +// +// This is scored at evaluation time rather than stored in dataset.json on purpose. A +// hand-maintained flag drifts the moment a case is edited, and the flag decides which +// cohort a case is reported in — so it has to be derived from the same task text the +// ranker reads, every run. +// +// Why it matters: a task containing "Location: lib/document.js:2339", or a GitHub +// permalink to the exact fixing file, hands the answer to any ranker with an +// explicit-file-mention signal. Those cases still exercise that signal, which is a real +// feature, but they cannot be counted as evidence that FixMap locates a file it was not +// told about. They are reported as their own cohort instead. +// +// Three tiers are recorded, because they are not equally strong: +// +// full-path the expected path, anchored at the repository root — including inside a +// github.com///blob// permalink +// path-suffix a multi-segment suffix of the expected path, e.g. a tsc error reporting +// "src/query/react/buildHooks.ts" for packages/toolkit/src/.../buildHooks.ts +// basename the bare filename only +// +// full-path and path-suffix both identify the file well enough to count as named, and +// together form the `mentioned` cohort. A bare basename does not: "index.ts" or +// "socket.ts" is ordinary prose in an issue about sockets, so basename-only cases stay in +// the `unmentioned` cohort and are reported separately for audit. + +/** Characters that can continue a path token on either side of a candidate match. */ +const PATH_BOUNDARY = /[A-Za-z0-9_/-]/; + +/** + * Repository-root anchors: text that, when it immediately precedes a path, means the path + * is measured from the repo root rather than being a longer, different path. + * Matches GitHub blob/tree/blame/raw permalinks at the end of the preceding text. + */ +const REPO_ROOT_ANCHOR = /(?:\/(?:blob|tree|blame|raw)\/[^\s/]+\/|raw\.githubusercontent\.com\/[^\s/]+\/[^\s/]+\/[^\s/]+\/)$/; + +/** + * Classifies one benchmark case by whether its task text names an expected fixing path. + * + * @param {{task?: string, expected?: string[]}} benchmark + */ +export function classifyExpectedPathMention(benchmark) { + // Windows-style separators appear in pasted stack traces; compare in one orientation. + const task = String(benchmark?.task ?? "").replace(/\\/g, "/"); + const expected = Array.isArray(benchmark?.expected) ? benchmark.expected : []; + + const evidence = []; + const mentionedPaths = []; + const suffixPaths = []; + const basenamePaths = []; + + for (const expectedPath of expected) { + const normalized = expectedPath.replace(/\\/g, "/"); + const segments = normalized.split("/"); + + const fullHit = findAnchoredPath(task, normalized); + if (fullHit !== null) { + mentionedPaths.push(expectedPath); + evidence.push({ path: expectedPath, tier: "full-path", match: normalized, context: contextAround(task, fullHit) }); + continue; + } + + // Longest multi-segment suffix first: "src/query/react/buildHooks.ts" before "react/buildHooks.ts". + let suffixFound = null; + for (let start = 1; start < segments.length - 1; start += 1) { + const suffix = segments.slice(start).join("/"); + const hit = findAnchoredPath(task, suffix); + if (hit !== null) { + suffixFound = { suffix, hit }; + break; + } + } + if (suffixFound) { + suffixPaths.push(expectedPath); + evidence.push({ + path: expectedPath, + tier: "path-suffix", + match: suffixFound.suffix, + context: contextAround(task, suffixFound.hit) + }); + continue; + } + + const basename = segments[segments.length - 1]; + const baseHit = findAnchoredPath(task, basename); + if (baseHit !== null) { + basenamePaths.push(expectedPath); + evidence.push({ path: expectedPath, tier: "basename", match: basename, context: contextAround(task, baseHit) }); + } + } + + const named = mentionedPaths.length > 0 || suffixPaths.length > 0; + return { + // The cohort split. True when the task identifies an expected file by path. + mentionsExpectedPath: named, + mentionTier: mentionedPaths.length > 0 ? "full-path" : suffixPaths.length > 0 ? "path-suffix" : basenamePaths.length > 0 ? "basename" : "none", + mentionedPaths, + suffixPaths, + basenamePaths, + evidence + }; +} + +/** + * Finds `needle` in `haystack` as a repo-root-anchored path token; returns its index or null. + * + * The left boundary check stops `test/lib/request.js` from counting as a mention of + * `lib/request.js` — a genuinely different file. The repo-root anchor exempts the case + * where the `/` before the match belongs to a GitHub permalink, where the same shape means + * the opposite thing: the path *is* measured from the repository root. + */ +function findAnchoredPath(haystack, needle) { + if (!needle) { + return null; + } + let from = 0; + for (;;) { + const index = haystack.indexOf(needle, from); + if (index === -1) { + return null; + } + const before = index === 0 ? "" : haystack[index - 1]; + const after = haystack[index + needle.length] ?? ""; + const leftOk = !PATH_BOUNDARY.test(before) || REPO_ROOT_ANCHOR.test(haystack.slice(Math.max(0, index - 200), index)); + // A trailing "#L200" or ":2339" is a line reference, not a longer filename. + if (leftOk && !PATH_BOUNDARY.test(after)) { + return index; + } + from = index + 1; + } +} + +/** A short excerpt so a reviewer can audit each classification instead of trusting it. */ +function contextAround(task, index, radius = 70) { + const start = Math.max(0, index - radius); + const end = Math.min(task.length, index + radius); + return `${start > 0 ? "…" : ""}${task.slice(start, end).replace(/\s+/g, " ").trim()}${end < task.length ? "…" : ""}`; +} + +/** + * Splits scored results into reporting cohorts. + * + * `unmentioned` is the generalization number: cases where the ranker had to locate the + * file rather than read it out of the task. + * + * @template {{ mentionsExpectedPath: boolean }} T + * @param {T[]} results + */ +export function splitCohorts(results) { + return { + all: results, + unmentioned: results.filter((result) => !result.mentionsExpectedPath), + mentioned: results.filter((result) => result.mentionsExpectedPath) + }; +} From 66d91671195d975a09c68ce82ab699dfcfdab4f5 Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 11:05:23 +0530 Subject: [PATCH 2/7] eval: measure FixMap against naive retrieval on the same scanned corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FixMap published a hit rate with nothing beside it, which left a reader's actual question unanswered: is this better than what an agent already gets by searching the repository itself? scripts/evaluate-baseline.mjs scores three baselines and FixMap on ONE scanRepo() result per case, so the only difference between arms is ranking. Both keyword arms are case-insensitive and expand camelCase, which favours the baselines deliberately — a handicapped baseline proves nothing. path-extraction path-shaped tokens read out of the task text lexical-literal literal keyword search, distinct terms then occurrences bm25 BM25 (k1=1.2, b=0.75); a retrieval baseline, not a grep fixmap rankContextFiles path-extraction scores exactly 0.000 on the unmentioned cohort of both suites and 0.667 on the mentioned one, which independently confirms the cohort classifier is measuring what it claims. Arms are compared with McNemar's exact test rather than by eyeballing two Wilson intervals: the arms ran on the same cases, and the pairing carries information that independent intervals discard. Wilson is extracted to scripts/lib/wilson.mjs so a published rate and its stated precision cannot come from two different formulas. That extraction is output-neutral: --check-recorded still exits 0 on both suites. Co-Authored-By: Claude Opus 5 --- benchmarks/external/baseline-results.json | 1269 +++++++++++++++++++++ benchmarks/heldout/baseline-results.json | 1052 +++++++++++++++++ scripts/evaluate-baseline.mjs | 347 ++++++ scripts/evaluate-external.mjs | 18 +- scripts/lib/wilson.mjs | 25 + 5 files changed, 2695 insertions(+), 16 deletions(-) create mode 100644 benchmarks/external/baseline-results.json create mode 100644 benchmarks/heldout/baseline-results.json create mode 100644 scripts/evaluate-baseline.mjs create mode 100644 scripts/lib/wilson.mjs diff --git a/benchmarks/external/baseline-results.json b/benchmarks/external/baseline-results.json new file mode 100644 index 0000000..cecf2f3 --- /dev/null +++ b/benchmarks/external/baseline-results.json @@ -0,0 +1,1269 @@ +{ + "suite": "external", + "cases": 16, + "configuration": { + "topN": 5, + "corpus": "one scanRepo() result per case, shared by every arm; ranking is the only difference", + "searchField": "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", + "tokenizer": "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", + "stopwords": 158, + "caseSensitivity": "case-insensitive for both keyword arms, which favours the baselines", + "bm25": { + "k1": 1.2, + "b": 0.75 + }, + "armDescriptions": { + "path-extraction": "path-shaped tokens read out of the task text, resolved against the corpus, ranked by order of appearance", + "lexical-literal": "literal keyword search ranked by distinct query terms matched, then raw occurrence count", + "bm25": "BM25 retrieval over the same text; a retrieval baseline, not a grep", + "fixmap": "rankContextFiles from @aryam/fixmap-core" + } + }, + "arms": { + "path-extraction": { + "all": { + "cases": 16, + "top1HitRate": 0.125, + "top3HitRate": 0.125, + "top5HitRate": 0.125, + "intervals95": { + "top1": [ + 0.035, + 0.36 + ], + "top3": [ + 0.035, + 0.36 + ], + "top5": [ + 0.035, + 0.36 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0, + "top3HitRate": 0, + "top5HitRate": 0, + "intervals95": { + "top1": [ + 0, + 0.228 + ], + "top3": [ + 0, + 0.228 + ], + "top5": [ + 0, + 0.228 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.667, + "top3HitRate": 0.667, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.208, + 0.939 + ], + "top3": [ + 0.208, + 0.939 + ], + "top5": [ + 0.208, + 0.939 + ] + } + } + }, + "lexical-literal": { + "all": { + "cases": 16, + "top1HitRate": 0.125, + "top3HitRate": 0.313, + "top5HitRate": 0.313, + "intervals95": { + "top1": [ + 0.035, + 0.36 + ], + "top3": [ + 0.142, + 0.556 + ], + "top5": [ + 0.142, + 0.556 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.077, + "top3HitRate": 0.308, + "top5HitRate": 0.308, + "intervals95": { + "top1": [ + 0.014, + 0.333 + ], + "top3": [ + 0.127, + 0.576 + ], + "top5": [ + 0.127, + 0.576 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "bm25": { + "all": { + "cases": 16, + "top1HitRate": 0.188, + "top3HitRate": 0.313, + "top5HitRate": 0.375, + "intervals95": { + "top1": [ + 0.066, + 0.43 + ], + "top3": [ + 0.142, + 0.556 + ], + "top5": [ + 0.185, + 0.614 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.154, + "top3HitRate": 0.308, + "top5HitRate": 0.385, + "intervals95": { + "top1": [ + 0.043, + 0.422 + ], + "top3": [ + 0.127, + 0.576 + ], + "top5": [ + 0.177, + 0.645 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "fixmap": { + "all": { + "cases": 16, + "top1HitRate": 0.688, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.444, + 0.858 + ], + "top3": [ + 0.806, + 1 + ], + "top5": [ + 0.806, + 1 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.692, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.424, + 0.873 + ], + "top3": [ + 0.772, + 1 + ], + "top5": [ + 0.772, + 1 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.667, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.208, + 0.939 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + } + } + } + }, + "pairedVsFixmapMcnemarExact": { + "all": { + "path-extraction": { + "top1": { + "aWins": 10, + "bWins": 1, + "discordant": 11, + "pValue": 0.0117 + }, + "top3": { + "aWins": 14, + "bWins": 0, + "discordant": 14, + "pValue": 0.0001 + }, + "top5": { + "aWins": 14, + "bWins": 0, + "discordant": 14, + "pValue": 0.0001 + } + }, + "lexical-literal": { + "top1": { + "aWins": 9, + "bWins": 0, + "discordant": 9, + "pValue": 0.0039 + }, + "top3": { + "aWins": 11, + "bWins": 0, + "discordant": 11, + "pValue": 0.001 + }, + "top5": { + "aWins": 11, + "bWins": 0, + "discordant": 11, + "pValue": 0.001 + } + }, + "bm25": { + "top1": { + "aWins": 8, + "bWins": 0, + "discordant": 8, + "pValue": 0.0078 + }, + "top3": { + "aWins": 11, + "bWins": 0, + "discordant": 11, + "pValue": 0.001 + }, + "top5": { + "aWins": 10, + "bWins": 0, + "discordant": 10, + "pValue": 0.002 + } + } + }, + "unmentioned": { + "path-extraction": { + "top1": { + "aWins": 9, + "bWins": 0, + "discordant": 9, + "pValue": 0.0039 + }, + "top3": { + "aWins": 13, + "bWins": 0, + "discordant": 13, + "pValue": 0.0002 + }, + "top5": { + "aWins": 13, + "bWins": 0, + "discordant": 13, + "pValue": 0.0002 + } + }, + "lexical-literal": { + "top1": { + "aWins": 8, + "bWins": 0, + "discordant": 8, + "pValue": 0.0078 + }, + "top3": { + "aWins": 9, + "bWins": 0, + "discordant": 9, + "pValue": 0.0039 + }, + "top5": { + "aWins": 9, + "bWins": 0, + "discordant": 9, + "pValue": 0.0039 + } + }, + "bm25": { + "top1": { + "aWins": 7, + "bWins": 0, + "discordant": 7, + "pValue": 0.0156 + }, + "top3": { + "aWins": 9, + "bWins": 0, + "discordant": 9, + "pValue": 0.0039 + }, + "top5": { + "aWins": 8, + "bWins": 0, + "discordant": 8, + "pValue": 0.0078 + } + } + } + }, + "results": [ + { + "slug": "expressjs/express", + "expected": [ + "lib/request.js" + ], + "scannedFiles": 213, + "queryTermCount": 47, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "arms": { + "path-extraction": { + "top5": [ + "lib/request.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal": { + "top5": [ + "lib/request.js", + "lib/response.js", + "test/res.send.js", + "lib/application.js", + "test/app.router.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "lib/request.js", + "test/req.fresh.js", + "test/res.send.js", + "test/req.stale.js", + "lib/response.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "lib/request.js", + "lib/response.js", + "package.json", + "examples/auth/index.js", + "examples/route-middleware/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "axios/axios", + "expected": [ + "lib/core/AxiosError.js" + ], + "scannedFiles": 454, + "queryTermCount": 36, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "tests/unit/prototypePollution.test.js", + "lib/adapters/http.js", + "tests/unit/core/AxiosError.test.js", + "docs/pages/advanced/request-config.md", + "tests/unit/adapters/fetch.test.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "MIGRATION_GUIDE.md", + "docs/pages/getting-started/first-steps.md", + "tests/unit/toFormData.test.js", + "docs/pages/advanced/create-an-instance.md", + "docs/pages/advanced/testing.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/core/AxiosError.js", + "lib/adapters/http.js", + "lib/helpers/HttpStatusCode.js", + "index.d.cts", + "index.d.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "debug-js/debug", + "expected": [ + "src/node.js" + ], + "scannedFiles": 14, + "queryTermCount": 30, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "README.md", + "src/node.js", + "src/common.js", + "src/browser.js", + ".github/PULL_REQUEST_TEMPLATE.md" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "src/node.js", + "README.md", + "src/common.js", + "src/browser.js", + ".github/PULL_REQUEST_TEMPLATE.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "src/node.js", + "src/browser.js", + "src/common.js", + "test.js", + "karma.conf.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "sindresorhus/ky", + "expected": [ + "source/core/Ky.ts" + ], + "scannedFiles": 67, + "queryTermCount": 42, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "readme.md", + "source/core/Ky.ts", + "source/types/options.ts", + "source/types/hooks.ts", + "test/main.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "readme.md", + "source/core/Ky.ts", + "source/core/constants.ts", + "source/types/options.ts", + "test/http-error.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "source/core/Ky.ts", + "source/errors/HTTPError.ts", + "source/types/hooks.ts", + "source/types/options.ts", + "source/errors/SchemaValidationError.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "colinhacks/zod", + "expected": [ + "packages/zod/src/v4/core/regexes.ts" + ], + "scannedFiles": 582, + "queryTermCount": 38, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "packages/docs-v3/README_ZH.md", + "packages/zod/src/v4/mini/schemas.ts", + "packages/zod/src/v4/mini/tests/index.test.ts", + "packages/zod/src/v4/classic/from-json-schema.ts", + "packages/docs-v3/ERROR_HANDLING.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "packages/zod/src/v4/mini/tests/string.test.ts", + "packages/zod/src/v4/classic/tests/detached-methods.test.ts", + "packages/zod/src/v4/classic/tests/refine.test.ts", + "packages/zod/src/v4/classic/tests/tuple.test.ts", + "packages/zod/src/v4/mini/tests/codec.test.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "packages/zod/src/v4/core/regexes.ts", + "packages/zod/src/v4/classic/from-json-schema.ts", + "packages/zod/src/v4/core/json-schema-processors.ts", + "packages/zod/src/v4/mini/schemas.ts", + "packages/zod/src/v4/core/parse.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "pinojs/pino", + "expected": [ + "lib/transport.js", + "lib/worker.js" + ], + "scannedFiles": 200, + "queryTermCount": 28, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/transports.md", + "docs/api.md", + "pino.d.ts", + "docs/help.md", + "test/transport/core.test.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "docs/api.md", + "test/transport/core.test.js", + "docs/transports.md", + "test/types/pino-transport.tst.ts", + "docs/help.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "pino.d.ts", + "lib/multistream.js", + "lib/transport.js", + "pino.js", + "lib/levels.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "fastify/fastify", + "expected": [ + "lib/request.js" + ], + "scannedFiles": 393, + "queryTermCount": 20, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/Reference/Errors.md", + "docs/Reference/Routes.md", + "SECURITY.md", + "docs/Reference/TypeScript.md", + "docs/Guides/Migration-Guide-V5.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "SECURITY.md", + "docs/Reference/Errors.md", + ".github/workflows/links-check.yml", + "CONTRIBUTING.md", + "docs/Guides/Serverless.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/request.js", + "lib/server.js", + "fastify.js", + "integration/server.js", + "types/request.d.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "chalk/chalk", + "expected": [ + "source/vendor/supports-color/index.js" + ], + "scannedFiles": 33, + "queryTermCount": 34, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "readme.md", + "source/vendor/supports-color/index.js", + "source/vendor/supports-color/index.d.ts", + "test/chalk.js", + "source/index.d.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "source/vendor/supports-color/index.js", + "test/no-color-support.js", + "source/vendor/supports-color/index.d.ts", + "readme.md", + "source/vendor/supports-color/browser.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "source/vendor/supports-color/index.js", + "source/vendor/supports-color/browser.js", + "source/vendor/supports-color/index.d.ts", + "source/index.d.ts", + "source/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "vitest-dev/vitest", + "expected": [ + "packages/browser/src/client/tester/dialog.ts" + ], + "scannedFiles": 3050, + "queryTermCount": 46, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/guide/browser/visual-regression-testing.md", + "docs/api/vi.md", + "docs/guide/migration.md", + "docs/guide/learn/debugging-tests.md", + "docs/blog/vitest-4-1.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + ".github/ISSUE_TEMPLATE/feature_request.yml", + "docs/guide/browser/index.md", + "docs/blog/vitest-4-1.md", + "docs/guide/learn/debugging-tests.md", + "docs/guide/features.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "packages/vitest/src/integrations/env/jsdom-keys.ts", + "packages/browser/src/client/tester/dialog.ts", + "packages/vitest/src/node/printError.ts", + "packages/vitest/src/runtime/runner/suite.ts", + "packages/pretty-format/src/index.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "eslint/eslint", + "expected": [ + "lib/types/config-api.d.ts" + ], + "scannedFiles": 2297, + "queryTermCount": 34, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/src/extend/custom-rules.md", + "lib/linter/linter.js", + "docs/src/use/migrate-to-9.0.0.md", + "lib/types/index.d.ts", + "docs/src/use/configure/migration-guide.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "docs/src/use/configure/migration-guide.md", + "docs/src/extend/plugin-migration-flat-config.md", + "docs/src/use/migrate-to-9.0.0.md", + "docs/src/extend/custom-rule-tutorial.md", + "docs/src/use/configure/configuration-files.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/types/config-api.d.ts", + "package.json", + "lib/config-api.js", + "lib/types/index.d.ts", + "lib/config/config.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "webpack/webpack", + "expected": [ + "lib/DefinePlugin.js" + ], + "scannedFiles": 13454, + "queryTermCount": 38, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "AGENTS.md", + "lib/Module.js", + "CONTRIBUTING.md", + "lib/SourceMapDevToolPlugin.js", + "examples/css/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "CONTRIBUTING.md", + ".changeset/README.md", + "WORKING_GROUP.md", + ".github/workflows/release.yml", + "AGENTS.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/dependencies/ImportMetaPlugin.js", + "lib/WebpackOptionsApply.js", + "lib/DefinePlugin.js", + "lib/EnvironmentPlugin.js", + "lib/dependencies/CreateRequireParserPlugin.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "nodejs/undici", + "expected": [ + "lib/dispatcher/client-h2.js" + ], + "scannedFiles": 786, + "queryTermCount": 52, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "lib/dispatcher/client-h2.js", + "test/http2-dispatcher.js", + "docs/docs/index.md", + "README.md", + "lib/dispatcher/client-h1.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "test/http2-late-data.js", + "lib/dispatcher/client-h2.js", + "test/http2-dispatcher.js", + "lib/api/api-stream.js", + "test/fetch/issue-4058.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "lib/dispatcher/client-h2.js", + "lib/web/fetch/body.js", + "lib/web/fetch/request.js", + "lib/web/fetch/response.js", + "lib/api/api-request.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "reduxjs/redux-toolkit", + "expected": [ + "packages/toolkit/src/query/react/buildHooks.ts" + ], + "scannedFiles": 1152, + "queryTermCount": 47, + "mentionsExpectedPath": true, + "mentionTier": "path-suffix", + "arms": { + "path-extraction": { + "top5": [ + "packages/toolkit/src/query/react/buildHooks.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal": { + "top5": [ + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/core/module.ts", + "packages/toolkit/src/query/core/apiState.ts", + "packages/toolkit/src/query/tests/buildThunks.test.tsx", + "_artifacts/domain_map.yaml" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "packages/toolkit/src/query/core/apiState.ts", + "packages/toolkit/src/query/core/buildSlice.ts", + "packages/toolkit/src/query/tests/queryLifecycle.test.tsx", + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/tests/cacheLifecycle.test.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "packages/toolkit/src/query/core/apiState.ts", + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/react/buildHooks.ts", + "packages/toolkit/src/listenerMiddleware/index.ts", + "examples/query/react/authentication-with-extrareducers/public/mockServiceWorker.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "prettier/prettier", + "expected": [ + "src/language-css/print/comma-separated-value-group.js" + ], + "scannedFiles": 9319, + "queryTermCount": 61, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "website/blog/2021-05-09-2.3.0.md", + "tests/format/markdown/markdown/real-world-case.md", + "website/blog/2018-04-11-1.12.0.md", + "website/blog/2017-11-07-1.8.0.md", + "website/blog/2020-03-21-2.0.0.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "website/blog/2017-04-20-1.2.0.md", + "README.md", + "website/blog/2023-11-13-curious-ternaries.md", + "website/blog/2018-04-09-plugin-php-0.1.md", + ".vscode/settings.example.json" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "src/language-css/print/comma-separated-value-group.js", + "src/language-js/print/member-chain.js", + "scripts/build/builders/javascript-module.js", + "website/playground/Playground.jsx", + "src/language-js/print/binaryish.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "honojs/hono", + "expected": [ + "src/adapter/aws-lambda/types.ts" + ], + "scannedFiles": 482, + "queryTermCount": 35, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "runtime-tests/lambda/index.test.ts", + "src/adapter/aws-lambda/handler.ts", + "runtime-tests/lambda/stream.test.ts", + "src/adapter/aws-lambda/handler.test.ts", + "docs/MIGRATION.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "runtime-tests/lambda/stream.test.ts", + "runtime-tests/lambda/index.test.ts", + "src/adapter/aws-lambda/handler.ts", + "src/adapter/aws-lambda/handler.test.ts", + "src/adapter/aws-lambda/types.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "src/adapter/aws-lambda/handler.ts", + "src/adapter/aws-lambda/types.ts", + "src/adapter/aws-lambda/conninfo.ts", + "src/adapter/lambda-edge/handler.ts", + "src/jsx/context.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "sindresorhus/got", + "expected": [ + "source/core/index.ts" + ], + "scannedFiles": 120, + "queryTermCount": 42, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "documentation/2-options.md", + "documentation/3-streams.md", + "documentation/tips.md", + "documentation/migration-guides/request.md", + "documentation/4-pagination.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "documentation/2-options.md", + "documentation/4-pagination.md", + "source/core/response.ts", + "documentation/migration-guides/request.md", + "documentation/tips.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "source/core/index.ts", + "source/core/diagnostics-channel.ts", + "source/core/response.ts", + "source/as-promise/index.ts", + "source/core/errors.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + } + ] +} diff --git a/benchmarks/heldout/baseline-results.json b/benchmarks/heldout/baseline-results.json new file mode 100644 index 0000000..19fcfd1 --- /dev/null +++ b/benchmarks/heldout/baseline-results.json @@ -0,0 +1,1052 @@ +{ + "suite": "heldout", + "cases": 12, + "configuration": { + "topN": 5, + "corpus": "one scanRepo() result per case, shared by every arm; ranking is the only difference", + "searchField": "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", + "tokenizer": "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", + "stopwords": 158, + "caseSensitivity": "case-insensitive for both keyword arms, which favours the baselines", + "bm25": { + "k1": 1.2, + "b": 0.75 + }, + "armDescriptions": { + "path-extraction": "path-shaped tokens read out of the task text, resolved against the corpus, ranked by order of appearance", + "lexical-literal": "literal keyword search ranked by distinct query terms matched, then raw occurrence count", + "bm25": "BM25 retrieval over the same text; a retrieval baseline, not a grep", + "fixmap": "rankContextFiles from @aryam/fixmap-core" + } + }, + "arms": { + "path-extraction": { + "all": { + "cases": 12, + "top1HitRate": 0.167, + "top3HitRate": 0.167, + "top5HitRate": 0.167, + "intervals95": { + "top1": [ + 0.047, + 0.448 + ], + "top3": [ + 0.047, + 0.448 + ], + "top5": [ + 0.047, + 0.448 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0, + "top3HitRate": 0, + "top5HitRate": 0, + "intervals95": { + "top1": [ + 0, + 0.299 + ], + "top3": [ + 0, + 0.299 + ], + "top5": [ + 0, + 0.299 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.667, + "top3HitRate": 0.667, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.208, + 0.939 + ], + "top3": [ + 0.208, + 0.939 + ], + "top5": [ + 0.208, + 0.939 + ] + } + } + }, + "lexical-literal": { + "all": { + "cases": 12, + "top1HitRate": 0.083, + "top3HitRate": 0.25, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.015, + 0.354 + ], + "top3": [ + 0.089, + 0.532 + ], + "top5": [ + 0.138, + 0.609 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.111, + "top3HitRate": 0.222, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.02, + 0.435 + ], + "top3": [ + 0.063, + 0.547 + ], + "top5": [ + 0.121, + 0.646 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0, + 0.562 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "bm25": { + "all": { + "cases": 12, + "top1HitRate": 0.167, + "top3HitRate": 0.25, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.047, + 0.448 + ], + "top3": [ + 0.089, + 0.532 + ], + "top5": [ + 0.138, + 0.609 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.111, + "top3HitRate": 0.222, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.02, + 0.435 + ], + "top3": [ + 0.063, + 0.547 + ], + "top5": [ + 0.121, + 0.646 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "fixmap": { + "all": { + "cases": 12, + "top1HitRate": 0.583, + "top3HitRate": 0.667, + "top5HitRate": 0.75, + "intervals95": { + "top1": [ + 0.32, + 0.807 + ], + "top3": [ + 0.391, + 0.862 + ], + "top5": [ + 0.468, + 0.911 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.444, + "top3HitRate": 0.556, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.189, + 0.733 + ], + "top3": [ + 0.267, + 0.811 + ], + "top5": [ + 0.354, + 0.879 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + } + } + } + }, + "pairedVsFixmapMcnemarExact": { + "all": { + "path-extraction": { + "top1": { + "aWins": 5, + "bWins": 0, + "discordant": 5, + "pValue": 0.0625 + }, + "top3": { + "aWins": 6, + "bWins": 0, + "discordant": 6, + "pValue": 0.0313 + }, + "top5": { + "aWins": 7, + "bWins": 0, + "discordant": 7, + "pValue": 0.0156 + } + }, + "lexical-literal": { + "top1": { + "aWins": 6, + "bWins": 0, + "discordant": 6, + "pValue": 0.0313 + }, + "top3": { + "aWins": 5, + "bWins": 0, + "discordant": 5, + "pValue": 0.0625 + }, + "top5": { + "aWins": 5, + "bWins": 0, + "discordant": 5, + "pValue": 0.0625 + } + }, + "bm25": { + "top1": { + "aWins": 5, + "bWins": 0, + "discordant": 5, + "pValue": 0.0625 + }, + "top3": { + "aWins": 5, + "bWins": 0, + "discordant": 5, + "pValue": 0.0625 + }, + "top5": { + "aWins": 6, + "bWins": 1, + "discordant": 7, + "pValue": 0.125 + } + } + }, + "unmentioned": { + "path-extraction": { + "top1": { + "aWins": 4, + "bWins": 0, + "discordant": 4, + "pValue": 0.125 + }, + "top3": { + "aWins": 5, + "bWins": 0, + "discordant": 5, + "pValue": 0.0625 + }, + "top5": { + "aWins": 6, + "bWins": 0, + "discordant": 6, + "pValue": 0.0313 + } + }, + "lexical-literal": { + "top1": { + "aWins": 3, + "bWins": 0, + "discordant": 3, + "pValue": 0.25 + }, + "top3": { + "aWins": 3, + "bWins": 0, + "discordant": 3, + "pValue": 0.25 + }, + "top5": { + "aWins": 3, + "bWins": 0, + "discordant": 3, + "pValue": 0.25 + } + }, + "bm25": { + "top1": { + "aWins": 3, + "bWins": 0, + "discordant": 3, + "pValue": 0.25 + }, + "top3": { + "aWins": 3, + "bWins": 0, + "discordant": 3, + "pValue": 0.25 + }, + "top5": { + "aWins": 4, + "bWins": 1, + "discordant": 5, + "pValue": 0.375 + } + } + } + }, + "results": [ + { + "slug": "Automattic/mongoose", + "expected": [ + "lib/document.js" + ], + "scannedFiles": 702, + "queryTermCount": 36, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "arms": { + "path-extraction": { + "top5": [ + "lib/document.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal": { + "top5": [ + "lib/connection.js", + "docs/schematypes.md", + "lib/schemaType.js", + "types/document.d.ts", + "lib/types/subdocument.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "lib/types/subdocument.js", + "types/document.d.ts", + "docs/documents.md", + "test/types.documentarray.test.js", + "lib/helpers/symbols.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/document.js", + "lib/helpers/setDefaultsOnInsert.js", + "lib/helpers/update/modifiedPaths.js", + "lib/helpers/common.js", + "lib/helpers/update/updatedPathsByArrayFilter.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "immerjs/immer", + "expected": [ + "src/types/types-external.ts" + ], + "scannedFiles": 165, + "queryTermCount": 24, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "src/types/types-external.ts", + "src/immer.ts", + "src/plugins/arrayMethods.ts", + "src/core/immerClass.ts", + "website/docs/pitfalls.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "src/types/types-external.ts", + ".github/ISSUE_TEMPLATE/bug.md", + "src/immer.ts", + "src/core/immerClass.ts", + "__tests__/produce.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "src/types/types-external.ts", + "src/immer.ts", + "src/plugins/patches.ts", + "src/core/immerClass.ts", + "src/core/proxy.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "jestjs/jest", + "expected": [ + "packages/jest-mock/src/index.ts" + ], + "scannedFiles": 3249, + "queryTermCount": 50, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/MockFunctionAPI.md", + "website/versioned_docs/version-30.4/MockFunctionAPI.md", + "website/versioned_docs/version-29.7/MockFunctionAPI.md", + "website/versioned_docs/version-30.0/MockFunctionAPI.md", + "packages/jest-mock/src/index.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25": { + "top5": [ + "website/versioned_docs/version-29.7/MockFunctionAPI.md", + "website/versioned_docs/version-30.0/MockFunctionAPI.md", + "website/versioned_docs/version-30.4/MockFunctionAPI.md", + "docs/MockFunctionAPI.md", + "packages/expect/src/types.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "packages/jest-config/src/readConfigFileAndSetRootDir.ts", + "packages/jest-config/src/resolveConfigPath.ts", + "packages/jest-core/src/TestScheduler.ts", + "packages/jest-mock/src/index.ts", + "packages/jest-resolve/src/resolver.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + } + } + }, + { + "slug": "knex/knex", + "expected": [ + "lib/dialects/postgres/query/pg-querycompiler.js" + ], + "scannedFiles": 600, + "queryTermCount": 31, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/src/guide/schema-builder.md", + "docs/src/guide/index.md", + "test/integration2/query/select/selects.spec.js", + "test/unit/schema-builder/mysql.js", + "docs/src/guide/migrations.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "docs/src/guide/schema-builder.md", + "docs/src/guide/index.md", + "lib/dialects/postgres/query/pg-querycompiler.js", + "docs/src/faq/recipes.md", + "lib/dialects/mssql/query/mssql-querycompiler.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "lib/dialects/postgres/query/pg-querycompiler.js", + "lib/dialects/mysql/query/mysql-querycompiler.js", + "lib/dialects/mysql/schema/mysql-tablecompiler.js", + "lib/schema/compiler.js", + "lib/dialects/postgres/schema/pg-columncompiler.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "mochajs/mocha", + "expected": [ + "lib/reporters/xunit.js" + ], + "scannedFiles": 661, + "queryTermCount": 46, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + ".github/CONTRIBUTING.md", + "MAINTAINERS.md", + ".github/ISSUE_TEMPLATE/01-bug.yml", + "README.md", + "PROJECT_CHARTER.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + ".github/ISSUE_TEMPLATE/01-bug.yml", + "README.md", + ".github/CONTRIBUTING.md", + ".github/ISSUE_TEMPLATE/03-feature-request.yml", + "PROJECT_CHARTER.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/errors.js", + "lib/reporters/xunit.js", + "lib/mocha.cjs", + "lib/reporters/base.js", + "lib/reporters/html.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "react-hook-form/react-hook-form", + "expected": [ + "src/logic/createFormControl.ts" + ], + "scannedFiles": 490, + "queryTermCount": 52, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "CHANGELOG.md", + "src/types/form.ts", + "src/logic/createFormControl.ts", + "reports/api-extractor.md.api.md", + "reports/api-extractor.md" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "CHANGELOG.md", + "scripts/bench.ts", + "src/__tests__/useForm/setValue.test.tsx", + "src/types/form.ts", + "src/__tests__/useForm/subscribe.test.tsx" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "src/logic/createFormControl.ts", + "src/types/form.ts", + "src/useController.ts", + "src/__typetest__/form-state-subscribe.test-d.ts", + "src/useFormState.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "socketio/socket.io", + "expected": [ + "packages/engine.io-client/lib/socket.ts" + ], + "scannedFiles": 829, + "queryTermCount": 27, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "packages/engine.io/lib/server.ts", + "packages/socket.io-client/dist/socket.io.esm.min.js", + "packages/socket.io/client-dist/socket.io.esm.min.js", + "packages/engine.io-client/dist/engine.io.esm.min.js", + "packages/socket.io-client/CHANGELOG.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "packages/engine.io-client/README.md", + "packages/engine.io/README.md", + "packages/engine.io/lib/server.ts", + "packages/engine.io-client/lib/transports/polling-xhr.ts", + "packages/engine.io-client/lib/socket.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "packages/engine.io-client/lib/transports/polling-xhr.ts", + "packages/engine.io-client/lib/transports/websocket.ts", + "packages/engine.io-client/lib/transports/polling-xhr.node.ts", + "packages/engine.io-client/lib/transports/polling.ts", + "packages/socket.io-client/lib/url.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + } + } + }, + { + "slug": "sveltejs/svelte", + "expected": [ + "packages/svelte/src/internal/client/dom/blocks/boundary.js" + ], + "scannedFiles": 8974, + "queryTermCount": 43, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "documentation/docs/07-misc/07-v5-migration-guide.md", + "packages/svelte/src/internal/server/renderer.js", + "packages/svelte/messages/client-warnings/warnings.md", + "documentation/docs/98-reference/.generated/client-warnings.md", + "packages/svelte/src/internal/client/reactivity/batch.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "documentation/docs/05-special-elements/01-svelte-boundary.md", + "packages/svelte/src/internal/shared/errors.js", + "CODE_OF_CONDUCT.md", + "packages/svelte/tests/hydration/samples/raw-mismatch-static/_config.js", + "packages/svelte/messages/client-errors/errors.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "packages/svelte/src/internal/client/dom/blocks/boundary.js", + "packages/svelte/src/internal/client/dom/operations.js", + "packages/svelte/src/internal/client/errors.js", + "packages/svelte/src/internal/client/warnings.js", + "packages/svelte/src/internal/client/dom/hydration.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "vitejs/vite", + "expected": [ + "packages/vite/src/node/server/bundledDev.ts" + ], + "scannedFiles": 2758, + "queryTermCount": 48, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/config/shared-options.md", + "CONTRIBUTING.md", + "packages/vite/src/node/utils.ts", + "docs/guide/features.md", + "packages/vite/src/node/build.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + "CONTRIBUTING.md", + "docs/guide/static-deploy.md", + "docs/guide/index.md", + ".github/ISSUE_TEMPLATE/bug_report.yml", + "packages/create-vite/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "packages/vite/src/types/commonjs.d.ts", + ".github/ISSUE_TEMPLATE/bug_report.yml", + "packages/vite/src/node/optimizer/rolldownDepPlugin.ts", + "packages/vite/src/node/plugins/worker.ts", + "packages/vite/src/node/plugins/workerImportMetaUrl.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + } + } + }, + { + "slug": "vuejs/core", + "expected": [ + "packages/runtime-dom/src/index.ts" + ], + "scannedFiles": 699, + "queryTermCount": 31, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + ".github/contributing.md", + "packages/compiler-sfc/src/script/resolveType.ts", + "packages/runtime-core/src/component.ts", + "packages/compiler-sfc/src/compileScript.ts", + "changelogs/CHANGELOG-3.1.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/contributing.md", + "packages/compiler-sfc/README.md", + "changelogs/CHANGELOG-3.1.md", + "packages/vue-compat/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "tsconfig.json", + "packages-private/dts-built-test/tsconfig.json", + "packages-private/tsconfig.json", + "packages-private/vite-debug/tsconfig.json", + "packages-private/dts-test/ref.test-d.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + } + } + }, + { + "slug": "winstonjs/winston", + "expected": [ + "lib/winston/transports/file.js" + ], + "scannedFiles": 115, + "queryTermCount": 38, + "mentionsExpectedPath": false, + "mentionTier": "none", + "arms": { + "path-extraction": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal": { + "top5": [ + "docs/transports.md", + "README.md", + "CHANGELOG.md", + "UPGRADE-3.0.md", + "lib/winston/logger.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25": { + "top5": [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + "lib/winston/common.js", + "docs/transports.md", + "test/unit/winston/logger-legacy.test.js", + "UPGRADE-3.0.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "fixmap": { + "top5": [ + "lib/winston/transports/file.js", + "lib/winston/tail-file.js", + "lib/winston/transports/console.js", + "lib/winston/transports/http.js", + "lib/winston.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + }, + { + "slug": "yargs/yargs", + "expected": [ + "lib/utils/apply-extends.ts" + ], + "scannedFiles": 170, + "queryTermCount": 39, + "mentionsExpectedPath": true, + "mentionTier": "full-path", + "arms": { + "path-extraction": { + "top5": [ + "lib/utils/apply-extends.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal": { + "top5": [ + "docs/api.md", + "docs/advanced.md", + "lib/utils/apply-extends.ts", + "test/validation.mjs", + "lib/command.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25": { + "top5": [ + "lib/utils/apply-extends.ts", + "lib/typings/yargs-parser-types.ts", + "lib/utils/levenshtein.ts", + "docs/api.md", + "helpers/helpers.mjs" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "fixmap": { + "top5": [ + "lib/utils/apply-extends.ts", + "lib/command.ts", + "lib/completion.ts", + "lib/typings/yargs-parser-types.ts", + "lib/usage.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + } + } + } + ] +} diff --git a/scripts/evaluate-baseline.mjs b/scripts/evaluate-baseline.mjs new file mode 100644 index 0000000..cbd1eb0 --- /dev/null +++ b/scripts/evaluate-baseline.mjs @@ -0,0 +1,347 @@ +// Baseline-relative ranking evaluation. +// +// node scripts/evaluate-baseline.mjs compare arms on the regression suite +// node scripts/evaluate-baseline.mjs --suite heldout compare arms on the held-out suite +// node scripts/evaluate-baseline.mjs --record write benchmarks//baseline-results.json +// +// FixMap publishes a hit rate with nothing beside it, which leaves the question a reader +// actually has unanswered: is this better than what an agent already gets for free by +// searching the repository itself? This script answers that by scoring naive retrieval on +// exactly the same inputs. +// +// Every arm sees ONE scanRepo() result per case — the same file list, the same text +// samples, the same truncation. Only the ranking function differs, so a difference in the +// score is a difference in ranking rather than in what was read. +// +// Arms: +// +// path-extraction pulls path-shaped tokens straight out of the task text and keeps +// the ones that resolve to a real file. Ranks nothing. Exists to +// price the leak: whatever this scores, the task was carrying. +// lexical-literal a literal keyword search, the "I ran a few greps" arm. Ranks by how +// many distinct query terms appear in a file, then by raw occurrence +// count. No corpus statistics, no weighting. +// bm25 standard BM25 (k1=1.2, b=0.75) over the same text. This is a +// retrieval baseline, not a grep — it is what a competent lexical +// search engine does, and it is the harder bar of the two. +// fixmap rankContextFiles, the shipped ranker. +// +// Both keyword arms are case-insensitive and expand camelCase, which favours the +// baselines. That is deliberate: a baseline that has been handicapped proves nothing. + +import { readFile, writeFile } from "node:fs/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { dirname, join, resolve } from "node:path"; +import { materializePinnedRepository } from "./lib/external-cache.mjs"; +import { classifyExpectedPathMention, splitCohorts } from "./lib/expected-path-mention.mjs"; +import { wilsonInterval } from "./lib/wilson.mjs"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const { scanRepo, rankContextFiles } = await import( + pathToFileURL(join(repoRoot, "packages", "core", "dist", "index.js")).href +); + +const suiteIndex = process.argv.indexOf("--suite"); +const suite = suiteIndex === -1 ? "external" : process.argv[suiteIndex + 1]; +if (!["external", "heldout"].includes(suite)) { + process.stderr.write(`Unknown suite "${suite}"; expected "external" or "heldout".\n`); + process.exit(1); +} + +const suiteDir = join(repoRoot, "benchmarks", suite); +const dataset = JSON.parse(await readFile(join(suiteDir, "dataset.json"), "utf8")); + +const TOP_N = 5; + +// Ordinary issue-prose and Markdown scaffolding. Kept deliberately short: an aggressive +// list would be tuning the baseline, and the point of the baseline is that nobody tuned it. +const STOPWORDS = new Set(`a about above after again against all am an and any are as at be because been before being +below between both but by can cannot could did do does doing down during each few for from further had has have having +he her here hers him his how i if in into is it its itself just me more most my no nor not of off on once only or other +ought our out over own same she should so some such than that the their them then there these they this those through +to too under until up very was we were what when where which while who whom why with would you your +bug issue issues error errors expected actual behavior behaviour reproduce reproduction steps version versions node npm +report repo repository description example code please thanks title type severity confidence location line lines +following above below see also would should could may might must will can also using used use uses`.split(/\s+/)); + +/** Tokenizes text for retrieval: lowercase alphanumerics plus camelCase sub-tokens. */ +function tokenize(text) { + const tokens = []; + for (const raw of String(text).match(/[A-Za-z0-9_$]+/g) ?? []) { + const lower = raw.toLowerCase(); + if (lower.length >= 3) { + tokens.push(lower); + } + // "modifiedPaths" also yields "modified" and "paths" so a query written in prose can + // reach an identifier written in camelCase. + const parts = raw.split(/(?<=[a-z0-9])(?=[A-Z])|_/).filter((part) => part.length >= 3); + if (parts.length > 1) { + for (const part of parts) { + tokens.push(part.toLowerCase()); + } + } + } + return tokens; +} + +/** Distinct, non-stopword query terms from the task text. No corpus statistics used. */ +function queryTerms(task) { + return [...new Set(tokenize(task))].filter((term) => !STOPWORDS.has(term)); +} + +/** The text each arm searches: the path plus whatever the scanner sampled of the file. */ +function searchableText(file) { + return `${file.path}\n${file.textSample ?? ""}`; +} + +// ---------------------------------------------------------------------------- arms + +/** + * Pulls path-shaped tokens out of the task and keeps those resolving to a real file. + * Ranked by order of appearance. This measures how much of a score the task text is + * carrying on its own, before any ranking happens. + */ +function rankByPathExtraction(repo, task) { + const text = String(task).replace(/\\/g, "/"); + const candidates = text.match(/[A-Za-z0-9_.$-]+(?:\/[A-Za-z0-9_.$-]+)+\.[A-Za-z0-9]+/g) ?? []; + const byPath = new Map(repo.files.map((file) => [file.path, file])); + const ranked = []; + const seen = new Set(); + for (const candidate of candidates) { + // Exact repo-relative path first, then a unique suffix match — a stack trace or tsc + // error often names a path relative to a package rather than to the repository root. + let match = byPath.has(candidate) ? candidate : null; + if (!match) { + const suffixMatches = repo.files.filter((file) => file.path.endsWith(`/${candidate}`)); + if (suffixMatches.length === 1) { + match = suffixMatches[0].path; + } + } + if (match && !seen.has(match)) { + seen.add(match); + ranked.push(match); + } + } + return ranked.slice(0, TOP_N); +} + +/** + * Literal keyword search: rank by how many distinct query terms occur in the file, then + * by total occurrences. This is the "I ran a few greps and looked at what matched most" + * arm — no idf, no length normalisation, no weighting. + */ +function rankByLexicalLiteral(repo, terms) { + const scored = []; + for (const file of repo.files) { + const haystack = searchableText(file).toLowerCase(); + let distinct = 0; + let total = 0; + for (const term of terms) { + let count = 0; + let index = haystack.indexOf(term); + while (index !== -1) { + count += 1; + index = haystack.indexOf(term, index + term.length); + } + if (count > 0) { + distinct += 1; + total += count; + } + } + if (distinct > 0) { + scored.push({ path: file.path, distinct, total }); + } + } + scored.sort((a, b) => b.distinct - a.distinct || b.total - a.total || a.path.localeCompare(b.path)); + return scored.slice(0, TOP_N).map((entry) => entry.path); +} + +/** Standard BM25 over the same corpus. The harder of the two keyword baselines. */ +function rankByBm25(repo, terms, k1 = 1.2, b = 0.75) { + const documents = repo.files.map((file) => { + const counts = new Map(); + for (const token of tokenize(searchableText(file))) { + counts.set(token, (counts.get(token) ?? 0) + 1); + } + return { path: file.path, counts, length: [...counts.values()].reduce((sum, n) => sum + n, 0) }; + }); + const total = documents.length; + if (total === 0) { + return []; + } + const averageLength = documents.reduce((sum, doc) => sum + doc.length, 0) / total; + + const documentFrequency = new Map(); + for (const term of terms) { + documentFrequency.set(term, documents.reduce((count, doc) => count + (doc.counts.has(term) ? 1 : 0), 0)); + } + + const scored = documents.map((doc) => { + let score = 0; + for (const term of terms) { + const frequency = doc.counts.get(term) ?? 0; + if (frequency === 0) { + continue; + } + const df = documentFrequency.get(term) ?? 0; + const idf = Math.log(1 + (total - df + 0.5) / (df + 0.5)); + score += idf * ((frequency * (k1 + 1)) / (frequency + k1 * (1 - b + (b * doc.length) / averageLength))); + } + return { path: doc.path, score }; + }); + + return scored + .filter((entry) => entry.score > 0) + .sort((a, b2) => b2.score - a.score || a.path.localeCompare(b2.path)) + .slice(0, TOP_N) + .map((entry) => entry.path); +} + +// ---------------------------------------------------------------------------- run + +const ARMS = ["path-extraction", "lexical-literal", "bm25", "fixmap"]; +const perArmResults = Object.fromEntries(ARMS.map((arm) => [arm, []])); +const perCase = []; + +for (const benchmark of dataset.cases) { + const dir = await materializePinnedRepository(benchmark); + // One scan, shared by every arm, so the comparison isolates ranking. + const repo = await scanRepo({ repoRoot: dir }); + if (repo.files.length === 0) { + throw new Error(`Baseline evaluation could not scan any files for ${benchmark.slug} at ${benchmark.sha}.`); + } + const terms = queryTerms(benchmark.task); + const mention = classifyExpectedPathMention(benchmark); + + const ranked = { + "path-extraction": rankByPathExtraction(repo, benchmark.task), + "lexical-literal": rankByLexicalLiteral(repo, terms), + bm25: rankByBm25(repo, terms), + fixmap: rankContextFiles(repo, { issueText: benchmark.task }, TOP_N).map((file) => file.path) + }; + + const caseRow = { + slug: benchmark.slug, + expected: benchmark.expected, + scannedFiles: repo.files.length, + queryTermCount: terms.length, + mentionsExpectedPath: mention.mentionsExpectedPath, + mentionTier: mention.mentionTier, + arms: {} + }; + + for (const arm of ARMS) { + const paths = ranked[arm]; + const row = { + slug: benchmark.slug, + mentionsExpectedPath: mention.mentionsExpectedPath, + top5: paths, + top1: benchmark.expected.includes(paths[0]), + top3: benchmark.expected.some((path) => paths.slice(0, 3).includes(path)), + top5Hit: benchmark.expected.some((path) => paths.includes(path)) + }; + perArmResults[arm].push(row); + caseRow.arms[arm] = { top5: paths, top1: row.top1, top3: row.top3, top5Hit: row.top5Hit }; + } + perCase.push(caseRow); +} + +function scoreCohort(cohort) { + const hitRate = (key) => + cohort.length === 0 ? null : Number((cohort.filter((result) => result[key]).length / cohort.length).toFixed(3)); + const interval = (key) => wilsonInterval(cohort.filter((result) => result[key]).length, cohort.length); + return { + cases: cohort.length, + top1HitRate: hitRate("top1"), + top3HitRate: hitRate("top3"), + top5HitRate: hitRate("top5Hit"), + intervals95: { top1: interval("top1"), top3: interval("top3"), top5: interval("top5Hit") } + }; +} + +// Comparing two Wilson intervals is the wrong test here and it understates the evidence: +// the arms ran on the SAME cases, so the pairing carries information that two independent +// intervals throw away. McNemar's exact test looks only at cases where the two arms +// disagreed, which is where all the signal is. It stays valid at these sample sizes; what +// small n costs is power, so a significant result remains meaningful while a +// non-significant one is genuinely inconclusive rather than negative. +function mcnemarExact(rowsA, rowsB, key) { + let aWins = 0; + let bWins = 0; + for (let index = 0; index < rowsA.length; index += 1) { + if (rowsA[index][key] && !rowsB[index][key]) aWins += 1; + else if (!rowsA[index][key] && rowsB[index][key]) bWins += 1; + } + const discordant = aWins + bWins; + if (discordant === 0) { + return { aWins, bWins, discordant, pValue: null }; + } + const choose = (n, k) => { + let value = 1; + for (let i = 1; i <= k; i += 1) value = (value * (n - k + i)) / i; + return value; + }; + let tail = 0; + for (let i = 0; i <= Math.min(aWins, bWins); i += 1) { + tail += choose(discordant, i) * Math.pow(0.5, discordant); + } + return { aWins, bWins, discordant, pValue: Number(Math.min(1, 2 * tail).toFixed(4)) }; +} + +const arms = {}; +for (const arm of ARMS) { + const cohorts = splitCohorts(perArmResults[arm]); + arms[arm] = { + all: scoreCohort(cohorts.all), + unmentioned: scoreCohort(cohorts.unmentioned), + mentioned: scoreCohort(cohorts.mentioned) + }; +} + +// FixMap against each baseline, on the cohort the product claim rests on. +const pairedVsFixmap = {}; +for (const cohortName of ["all", "unmentioned"]) { + const pick = (arm) => splitCohorts(perArmResults[arm])[cohortName]; + const fixmapRows = pick("fixmap"); + pairedVsFixmap[cohortName] = Object.fromEntries( + ARMS.filter((arm) => arm !== "fixmap").map((arm) => [ + arm, + { + top1: mcnemarExact(fixmapRows, pick(arm), "top1"), + top3: mcnemarExact(fixmapRows, pick(arm), "top3"), + top5: mcnemarExact(fixmapRows, pick(arm), "top5Hit") + } + ]) + ); +} + +const summary = { + suite, + cases: dataset.cases.length, + configuration: { + topN: TOP_N, + corpus: "one scanRepo() result per case, shared by every arm; ranking is the only difference", + searchField: "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", + tokenizer: "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", + stopwords: STOPWORDS.size, + caseSensitivity: "case-insensitive for both keyword arms, which favours the baselines", + bm25: { k1: 1.2, b: 0.75 }, + armDescriptions: { + "path-extraction": "path-shaped tokens read out of the task text, resolved against the corpus, ranked by order of appearance", + "lexical-literal": "literal keyword search ranked by distinct query terms matched, then raw occurrence count", + bm25: "BM25 retrieval over the same text; a retrieval baseline, not a grep", + fixmap: "rankContextFiles from @aryam/fixmap-core" + } + }, + arms, + // aWins counts cases FixMap got and the baseline missed; bWins the reverse. + pairedVsFixmapMcnemarExact: pairedVsFixmap, + results: perCase +}; + +const rendered = `${JSON.stringify(summary, null, 2)}\n`; +process.stdout.write(rendered); + +if (process.argv.includes("--record")) { + await writeFile(join(suiteDir, "baseline-results.json"), rendered, "utf8"); +} diff --git a/scripts/evaluate-external.mjs b/scripts/evaluate-external.mjs index ec13d17..d00ad34 100644 --- a/scripts/evaluate-external.mjs +++ b/scripts/evaluate-external.mjs @@ -20,6 +20,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { dirname, join, resolve } from "node:path"; import { materializePinnedRepository } from "./lib/external-cache.mjs"; import { classifyExpectedPathMention, splitCohorts } from "./lib/expected-path-mention.mjs"; +import { wilsonInterval } from "./lib/wilson.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const { scanRepo, rankContextFiles } = await import(pathToFileURL(join(repoRoot, "packages", "core", "dist", "index.js")).href); @@ -72,22 +73,7 @@ const rate = (key) => results.filter((result) => result[key]).length / results.l // A hit rate over a dozen cases reads far more precise than it is: one case flipping // moves 9/12 by eight points. The Wilson score interval is reported next to every rate -// so a reader sees the real precision instead of inferring it from the decimals. It is -// used rather than the normal approximation because that one misbehaves near 0 and 1, -// which is exactly where a perfect result sits. -function wilsonInterval(successes, total, z = 1.96) { - if (total === 0) { - return null; - } - const proportion = successes / total; - const denominator = 1 + (z * z) / total; - const centre = proportion + (z * z) / (2 * total); - const spread = z * Math.sqrt(proportion * (1 - proportion) / total + (z * z) / (4 * total * total)); - return [ - Number(Math.max(0, (centre - spread) / denominator).toFixed(3)), - Number(Math.min(1, (centre + spread) / denominator).toFixed(3)) - ]; -} +// so a reader sees the real precision instead of inferring it from the decimals. const band = (key) => { const successes = results.filter((result) => result[key]).length; diff --git a/scripts/lib/wilson.mjs b/scripts/lib/wilson.mjs new file mode 100644 index 0000000..6659ad1 --- /dev/null +++ b/scripts/lib/wilson.mjs @@ -0,0 +1,25 @@ +// Wilson score interval, shared by every evaluation script so that a published rate and +// its stated precision can never come from two different formulas. +// +// Used rather than the normal approximation because that one misbehaves near 0 and 1, +// which is exactly where a perfect result sits. + +/** + * @param {number} successes + * @param {number} total + * @param {number} [z] 1.96 for a 95% interval + * @returns {[number, number] | null} null when there is nothing to measure + */ +export function wilsonInterval(successes, total, z = 1.96) { + if (total === 0) { + return null; + } + const proportion = successes / total; + const denominator = 1 + (z * z) / total; + const centre = proportion + (z * z) / (2 * total); + const spread = z * Math.sqrt((proportion * (1 - proportion)) / total + (z * z) / (4 * total * total)); + return [ + Number(Math.max(0, (centre - spread) / denominator).toFixed(3)), + Number(Math.min(1, (centre + spread) / denominator).toFixed(3)) + ]; +} From 57927348419969ff559ffd800e8250296b682bb0 Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 11:14:16 +0530 Subject: [PATCH 3/7] docs: publish the cohort correction and the baseline comparison Leads the README and site evidence page with the cohort that had to locate the file (held-out 44% Top-1 / 56% Top-3, n=9) instead of the pooled rate (58% / 67%, n=12), and puts a baseline number beside every FixMap number. The correction is stated as structural rather than as a measured effect size, because that is what the data supports: the regression suite barely moves under the same split and its named cases are 2/3 rather than 3/3, so with three cases per named cohort the value of a mention is not established. The baseline result is reported with the same restraint in the other direction. FixMap beats literal keyword search and BM25 on both suites and never loses a disagreeing case on the regression suite (McNemar exact, p = 0.004-0.016), but the held-out cohort cannot reach significance at all: with three disagreeing cases the smallest attainable two-sided p-value is 0.25. That is stated as a power limit rather than quietly omitted, and it is the strongest argument for growing the held-out suite. Site figures are read from the recorded JSON at build time, so the page cannot drift from the suites. Ten tests pin the classifier, including the permalink shape and the two false positives a bare substring match makes. Co-Authored-By: Claude Opus 5 --- README.md | 45 ++++++-- apps/web/app/_lib/site-data.ts | 27 +++++ apps/web/app/evidence/page.tsx | 58 ++++++++-- docs/BENCHMARKS.md | 100 ++++++++++++++++- .../core/test/expected-path-mention.test.mjs | 101 ++++++++++++++++++ 5 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 packages/core/test/expected-path-mention.test.mjs diff --git a/README.md b/README.md index a6ac3f4..a196370 100644 --- a/README.md +++ b/README.md @@ -422,15 +422,48 @@ FixMap is measured against real issues that were later fixed by a merged pull re | | Held-out — 12 repos, **never tuned against** | Regression — 16 repos, guided development | | --- | ---: | ---: | -| Fixing file ranked Top-1 | **7 / 12 — 58%**
95% CI 32–81% | 11 / 16 — 69%
95% CI 44–86% | -| Fixing file ranked Top-3 | **8 / 12 — 67%**
95% CI 39–86% | 16 / 16 — 100%
95% CI 81–100% | -| Wrong file ranked first while the right one was available | **2 / 12 — 17%** | 5 / 16 — 31% | +| Fixing file ranked Top-1 | 7 / 12 — 58%
95% CI 32–81% | 11 / 16 — 69%
95% CI 44–86% | +| Fixing file ranked Top-3 | 8 / 12 — 67%
95% CI 39–86% | 16 / 16 — 100%
95% CI 81–100% | +| Wrong file ranked first while the right one was available | 2 / 12 — 17% | 5 / 16 — 31% | -**Plan around the held-out column.** The regression suite is where the ranking heuristics were developed — a case missed, the ranker changed — so its 100% describes fit, not accuracy on your repository. +#### Some of those tasks already contained their answer -**And read the intervals, not the percentages.** At twelve cases one result flipping moves Top-3 by eight points. The honest statement is "roughly two thirds, with a wide interval", not a precise success probability. Anyone quoting these figures to two significant figures, including us, is overstating them. +Three of the twelve held-out tasks name a fixing file in the task text itself. Mongoose's says `Location: lib/document.js:2339`; svelte's and yargs' link a GitHub permalink straight to the file and line range. A ranker that reads explicit file mentions — which FixMap has — answers those by reading the task, not by searching the repository. Pooling them into one rate lets three cases carry the headline. -Two things the point estimates hide. Held-out Top-1 (58%) remains close to its Top-3 (67%) — **when FixMap finds the file at all, it usually ranks it first**, which is what actually matters to an agent that opens one file. The tuned suite's 100% Top-3 still conceals that in 31% of those cases something wrong ranks above the answer, so an agent following it opens the wrong file first. +Split by whether the task named the file, the held-out suite reads: + +| Held-out cohort | Cases | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | ---: | +| Task **did not** name the file — *plan around this one* | 9 | **44%**
95% CI 19–73% | **56%**
95% CI 27–81% | 67% | +| Task named the file | 3 | 100% | 100% | 100% | +| Pooled (what we published before) | 12 | 58% | 67% | 75% | + +The same split on the regression suite barely moves it (69% → 69% Top-1), and its three named cases are 2 / 3 rather than 3 / 3 — so being named does not guarantee a hit, and with three cases per cohort the *size* of this effect is not established. What is established is structural: a generalization headline should not be computed over tasks that contain their own answer. The cohort is now derived at evaluation time from the same task text the ranker reads, so it cannot drift. + +**Plan around the held-out, unmentioned cohort.** The regression suite is where the ranking heuristics were developed — a case missed, the ranker changed — so its 100% describes fit, not accuracy on your repository. + +**And read the intervals, not the percentages.** At nine cases one result flipping moves Top-3 by eleven points. The honest statement is "roughly half, with a wide interval", not a precise success probability. Anyone quoting these figures to two significant figures, including us, is overstating them. + +#### Is this better than just searching the repository? + +The fair question about a ranked file list is whether it beats what an agent already gets for free. So the same suites are scored against naive retrieval on **the same scanned corpus** — one repository scan per case, shared by every arm, so the only difference is ranking. Both keyword arms are case-insensitive and expand camelCase, which favours the baselines on purpose. + +Held-out, tasks that did not name the file (9 cases): + +| Arm | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | +| Path extraction — read paths out of the task | 0% | 0% | 0% | +| Literal keyword search | 11% | 22% | 33% | +| BM25 retrieval | 11% | 22% | 33% | +| **FixMap** | **44%** | **56%** | **67%** | + +FixMap wins on the regression suite too (69% vs 15% Top-1 against BM25 on its 13 unmentioned cases), and it never loses a head-to-head case against any baseline on that suite — a paired McNemar exact test puts that at p = 0.004–0.016. On the nine held-out cases the direction is identical and FixMap still never loses a disagreement, but it **cannot** reach significance there: with three disagreeing cases the smallest possible two-sided p-value is 0.25. That is a limit of the sample size, not a negative result, and it is the strongest argument for growing the held-out suite. + +Path extraction scoring exactly 0% on this cohort and 67% on the named one is the check that the cohort split is measuring what it claims. + +Reproduce it with `node scripts/evaluate-baseline.mjs --suite heldout`; every arm's ranking is recorded in [`benchmarks/heldout/baseline-results.json`](benchmarks/heldout/baseline-results.json). + +One thing the point estimates hide: held-out Top-1 stays close to Top-3 in both cohorts — **when FixMap finds the file at all, it usually ranks it first**, which is what matters to an agent that opens one file. The tuned suite's 100% Top-3 still conceals that in 31% of those cases something wrong ranks above the answer, so an agent following it opens the wrong file first. The three held-out misses are published with their real rankings in [`benchmarks/heldout/`](benchmarks/heldout), not removed or explained away. diff --git a/apps/web/app/_lib/site-data.ts b/apps/web/app/_lib/site-data.ts index f10e365..92791f4 100644 --- a/apps/web/app/_lib/site-data.ts +++ b/apps/web/app/_lib/site-data.ts @@ -1,5 +1,7 @@ import heldout from "../../../../benchmarks/heldout/results.json"; import regression from "../../../../benchmarks/external/results.json"; +import heldoutBaseline from "../../../../benchmarks/heldout/baseline-results.json"; +import regressionBaseline from "../../../../benchmarks/external/baseline-results.json"; import savings from "../../../../benchmarks/external/savings-results.json"; import cli from "../../../../packages/cli/package.json"; @@ -19,6 +21,25 @@ export const repoUrl = "https://github.com/aryamthecodebreaker/FixMap"; export const npmUrl = "https://www.npmjs.com/package/@aryam/fixmap"; export const marketplaceUrl = "https://github.com/marketplace/actions/fixmap"; +// Some benchmark tasks name the fixing file outright — a "Location: lib/document.js:2339" +// line, or a GitHub permalink to the exact range. Those are answerable by reading the task +// rather than by searching the repository, so the site reports the cohort that had to be +// located separately from the pooled rate. Read straight from the recorded results so the +// page cannot drift from the suite. +const cohortOf = (suite: typeof heldout) => ({ + unmentioned: suite.cohorts.unmentioned, + mentioned: suite.cohorts.mentioned +}); + +// The comparison a ranked list actually has to win: naive retrieval on the same corpus. +const baselineOf = (suite: typeof heldoutBaseline) => ({ + cases: suite.arms.fixmap.unmentioned.cases, + fixmap: suite.arms.fixmap.unmentioned, + bm25: suite.arms.bm25.unmentioned, + lexical: suite.arms["lexical-literal"].unmentioned, + pathExtraction: suite.arms["path-extraction"].unmentioned +}); + export const siteStats = { version: cli.version, medianSeconds: (savings.performance.medianScanAndRankMs / 1000).toFixed(2), @@ -28,6 +49,7 @@ export const siteStats = { top3: hits(heldoutResults, "top3"), top5: hits(heldoutResults, "top5Hit"), intervals95: heldout.intervals95, + cohorts: cohortOf(heldout), results: heldout.results }, regression: { @@ -36,7 +58,12 @@ export const siteStats = { top3: hits(regressionResults, "top3"), top5: hits(regressionResults, "top5Hit"), intervals95: regression.intervals95, + cohorts: cohortOf(regression), results: regression.results + }, + baselines: { + heldout: baselineOf(heldoutBaseline), + regression: baselineOf(regressionBaseline) } }; diff --git a/apps/web/app/evidence/page.tsx b/apps/web/app/evidence/page.tsx index d3b573c..1040bc5 100644 --- a/apps/web/app/evidence/page.tsx +++ b/apps/web/app/evidence/page.tsx @@ -9,11 +9,15 @@ export const metadata: Metadata = { }; const percent = (value: number, total: number) => `${Math.round((value / total) * 100)}%`; +const rate = (value: number | null) => (value === null ? "—" : `${Math.round(value * 100)}%`); + +const heldoutUnmentioned = siteStats.heldout.cohorts.unmentioned; +const heldoutMentioned = siteStats.heldout.cohorts.mentioned; +const heldoutBaselines = siteStats.baselines.heldout; // A plain-English reading of the measured rate, so the callout cannot claim "three quarters" // after a re-record moves the number to two thirds. Bands, not a fabricated precision. -function describeRate(value: number, total: number): string { - const share = value / total; +function describeRate(share: number): string { if (share >= 0.9) return "almost all"; if (share >= 0.7) return "about three quarters"; if (share >= 0.58) return "about two thirds"; @@ -38,11 +42,11 @@ export default function EvidencePage() {
-

Held-out · {siteStats.heldout.cases} repositories

- {siteStats.heldout.top3}/{siteStats.heldout.cases} +

Held-out · tasks that did not name the file · {heldoutUnmentioned.cases} repositories

+ {rate(heldoutUnmentioned.top3HitRate)}

fixes surfaced in the top 3

-

Selected after the ranker was finished. This is the result to plan around.

-
Top 1
{siteStats.heldout.top1}/{siteStats.heldout.cases} · {percent(siteStats.heldout.top1, siteStats.heldout.cases)}
Top 5
{siteStats.heldout.top5}/{siteStats.heldout.cases} · {percent(siteStats.heldout.top5, siteStats.heldout.cases)}
+

Selected after the ranker was finished, and counting only the cases where FixMap had to locate the file rather than read it out of the task. This is the result to plan around.

+
Top 1
{rate(heldoutUnmentioned.top1HitRate)}
Top 5
{rate(heldoutUnmentioned.top5HitRate)}

Regression · {siteStats.regression.cases} repositories

@@ -52,7 +56,47 @@ export default function EvidencePage() {
Top 1
{siteStats.regression.top1}/{siteStats.regression.cases} · {percent(siteStats.regression.top1, siteStats.regression.cases)}
Top 5
{siteStats.regression.top5}/{siteStats.regression.cases} · {percent(siteStats.regression.top5, siteStats.regression.cases)}
-

Read this as “{describeRate(siteStats.heldout.top3, siteStats.heldout.cases)},” not as two significant figures. With {siteStats.heldout.cases} held-out cases, one result changing moves Top-3 by roughly {Math.round(100 / siteStats.heldout.cases)} points. The 95% interval is {Math.round((siteStats.heldout.intervals95.top3[0] ?? 0) * 100)}–{Math.round((siteStats.heldout.intervals95.top3[1] ?? 0) * 100)}%.

+

Read this as “{describeRate(heldoutUnmentioned.top3HitRate ?? 0)},” not as two significant figures. With {heldoutUnmentioned.cases} held-out cases in this cohort, one result changing moves Top-3 by roughly {Math.round(100 / heldoutUnmentioned.cases)} points. The 95% interval is {Math.round((heldoutUnmentioned.intervals95.top3?.[0] ?? 0) * 100)}–{Math.round((heldoutUnmentioned.intervals95.top3?.[1] ?? 0) * 100)}%.

+
+ +
+
+

Cohorts

Some tasks already contained their answer.

+

+ {heldoutMentioned.cases} of the {siteStats.heldout.cases} held-out tasks name a fixing file + outright — one as Location: lib/document.js:2339, two as a GitHub permalink to + the exact lines. A ranker that reads explicit file mentions answers those by reading the + task. Pooling them lets {heldoutMentioned.cases} cases carry the headline, so they are + reported separately. +

+
+
+
CohortCasesTop 1Top 3
+
Task did not name the file{heldoutUnmentioned.cases}{rate(heldoutUnmentioned.top1HitRate)}{rate(heldoutUnmentioned.top3HitRate)}
+
Task named the file{heldoutMentioned.cases}{rate(heldoutMentioned.top1HitRate)}{rate(heldoutMentioned.top3HitRate)}
+
Pooled — what we published before{siteStats.heldout.cases}{percent(siteStats.heldout.top1, siteStats.heldout.cases)}{percent(siteStats.heldout.top3, siteStats.heldout.cases)}
+
+

This is a structural correction, not a measured effect size. The regression suite barely moves under the same split, and its named cases are 2 of 3 rather than 3 of 3 — being named does not guarantee a hit. With three cases per named cohort, how much a mention is worth is not established. What is established is that a generalization headline should not be computed over tasks that contain their own answer.

+
+ +
+
+

Baselines

Better than searching the repository?

+

+ A ranked list only earns its place if it beats what an agent already gets for free. Every + arm below scores on the same repository scan — same files, same text + samples — so the only difference is ranking. Both keyword arms are case-insensitive and + expand camelCase, which favours the baselines on purpose. +

+
+
+
ArmTop 1Top 3Top 5
+
Path extraction — read paths out of the task{rate(heldoutBaselines.pathExtraction.top1HitRate)}{rate(heldoutBaselines.pathExtraction.top3HitRate)}{rate(heldoutBaselines.pathExtraction.top5HitRate)}
+
Literal keyword search{rate(heldoutBaselines.lexical.top1HitRate)}{rate(heldoutBaselines.lexical.top3HitRate)}{rate(heldoutBaselines.lexical.top5HitRate)}
+
BM25 retrieval{rate(heldoutBaselines.bm25.top1HitRate)}{rate(heldoutBaselines.bm25.top3HitRate)}{rate(heldoutBaselines.bm25.top5HitRate)}
+
FixMap{rate(heldoutBaselines.fixmap.top1HitRate)}{rate(heldoutBaselines.fixmap.top3HitRate)}{rate(heldoutBaselines.fixmap.top5HitRate)}
+
+

The held-out lead is real in direction but underpowered. FixMap wins the same comparison on the regression suite at p = 0.004–0.016 by a paired McNemar exact test, and never loses a disagreeing case there. On these {heldoutBaselines.cases} held-out cases it still never loses a disagreement, but it cannot reach significance: with three disagreeing cases the smallest possible two-sided p-value is 0.25. That is a sample-size limit, not a negative result. Path extraction scoring 0% here — and 67% on the cases that named the file — is the check that the cohort split measures what it claims.

diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 1aca3cd..78e15ee 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -10,14 +10,108 @@ Ranking outputs refreshed 2026-07-31 on Node v24.13.0, Windows 11 (10.0.26200), | Quantity | Held-out (12) | Regression (16) | Evidence type | | --- | ---: | ---: | --- | -| Expected fixing file in Top-1 | 7/12 (58%) | 11/16 (69%) | Measured | -| Expected fixing file in Top-3 | 8/12 (67%) | 16/16 (100%) | Measured | -| Expected fixing file in Top-5 | 9/12 (75%) | 16/16 (100%) | Measured | +| Expected fixing file in Top-1 | 7/12 (58%) | 11/16 (69%) | Measured, **pooled — see cohorts below** | +| Expected fixing file in Top-3 | 8/12 (67%) | 16/16 (100%) | Measured, **pooled — see cohorts below** | +| Expected fixing file in Top-5 | 9/12 (75%) | 16/16 (100%) | Measured, **pooled — see cohorts below** | | Median scan + rank time | — | 1,747.7 ms | Measured, three warm runs per pinned repository | | Context proxy reduction | — | 98.56% | Estimated proxy, **not** a savings measurement | **The held-out column is the one to plan around.** The regression column describes performance on cases that shaped the ranker and will overstate what happens on a repository FixMap has never seen. +### Cohorts: tasks that already name the fixing file + +Some benchmark tasks contain a fixing path in the task text — `Location: lib/document.js:2339` in +the mongoose case, and GitHub permalinks to the exact file and line range in the svelte and yargs +cases. A ranker with an explicit-file-mention signal answers those by reading the task rather than +by searching the repository, so pooling them into a single rate lets a handful of cases carry a +generalization headline. + +Classification is derived at evaluation time by +[`scripts/lib/expected-path-mention.mjs`](../scripts/lib/expected-path-mention.mjs) from the same +task text the ranker reads — never stored in `dataset.json`, where it would drift from the case it +describes. Three tiers are recorded: a repository-root-anchored full path (including inside a +`github.com///blob//` permalink), a multi-segment path suffix such as a +`tsc` error naming `src/query/react/buildHooks.ts`, and a bare basename. The first two count as +named; a bare `index.ts` is ordinary prose in an issue and does not. + +| Suite | Cohort | Cases | Top-1 | Top-3 | Top-5 | +| --- | --- | ---: | ---: | ---: | ---: | +| Held-out | Task did not name the file | 9 | **44.4%** (95% CI 19–73%) | **55.6%** (95% CI 27–81%) | 66.7% | +| Held-out | Task named the file | 3 | 100% | 100% | 100% | +| Held-out | Pooled | 12 | 58.3% | 66.7% | 75.0% | +| Regression | Task did not name the file | 13 | 69.2% | 100% | 100% | +| Regression | Task named the file | 3 | 66.7% | 100% | 100% | +| Regression | Pooled | 16 | 68.8% | 100% | 100% | + +**Read this as a structural correction, not a measured effect size.** The regression suite barely +moves under the split, and its named cases are 2/3 rather than 3/3 — so being named does not +guarantee a hit. With three cases per named cohort, how much a mention is worth is not established. +What is established is that a generalization headline should not be computed over tasks that +contain their own answer. + +## Baseline-relative ranking + +A hit rate published on its own does not answer the question a reader actually has: is this better +than what an agent already gets by searching the repository itself? +[`scripts/evaluate-baseline.mjs`](../scripts/evaluate-baseline.mjs) scores three naive arms and +FixMap on **one `scanRepo()` result per case, shared by every arm** — the same file list, the same +text samples, the same truncation — so a difference in score is a difference in ranking. + +| Arm | What it does | +| --- | --- | +| `path-extraction` | Pulls path-shaped tokens out of the task text and keeps those resolving to a real file. Ranks nothing; it prices what the task text was carrying. | +| `lexical-literal` | Literal keyword search: distinct query terms matched, then raw occurrence count. No corpus statistics. | +| `bm25` | Standard BM25 (k1 = 1.2, b = 0.75) over the same text. A retrieval baseline, **not** a grep. | +| `fixmap` | `rankContextFiles` from `@aryam/fixmap-core`. | + +Both keyword arms are case-insensitive and expand camelCase, which favours the baselines. That is +deliberate — a handicapped baseline proves nothing. + +Held-out suite, cases whose task did not name the file (9): + +| Arm | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | +| `path-extraction` | 0.0% | 0.0% | 0.0% | +| `lexical-literal` | 11.1% | 22.2% | 33.3% | +| `bm25` | 11.1% | 22.2% | 33.3% | +| `fixmap` | **44.4%** | **55.6%** | **66.7%** | + +Regression suite, same cohort (13): + +| Arm | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | +| `path-extraction` | 0.0% | 0.0% | 0.0% | +| `lexical-literal` | 7.7% | 30.8% | 30.8% | +| `bm25` | 15.4% | 30.8% | 38.5% | +| `fixmap` | **69.2%** | **100%** | **100%** | + +`path-extraction` scoring exactly 0.0% on the unmentioned cohort of both suites, and 66.7% on the +named cohort, is the independent check that the cohort classifier measures what it claims. + +Arms are compared with **McNemar's exact test** rather than by comparing two Wilson intervals: the +arms ran on the same cases, and that pairing carries information independent intervals discard. + +| Suite | Cohort | FixMap vs | Top-1 p | Top-3 p | Top-5 p | +| --- | --- | --- | ---: | ---: | ---: | +| Regression | unmentioned (13) | `lexical-literal` | 0.0078 | 0.0039 | 0.0039 | +| Regression | unmentioned (13) | `bm25` | 0.0156 | 0.0039 | 0.0078 | +| Held-out | unmentioned (9) | `lexical-literal` | 0.25 | 0.25 | 0.25 | +| Held-out | unmentioned (9) | `bm25` | 0.25 | 0.25 | 0.375 | + +FixMap never loses a disagreeing case to any baseline on the regression suite, and loses exactly one +across the entire held-out comparison. The held-out p-values are **not** evidence of no effect: with +three disagreeing cases the smallest attainable two-sided p-value is 0.25, so that cohort is +arithmetically incapable of reaching 0.05. It is underpowered, which is the strongest available +argument for growing it. + +```bash +node scripts/evaluate-baseline.mjs --suite heldout +node scripts/evaluate-baseline.mjs --suite external +node scripts/evaluate-baseline.mjs --suite heldout --record +``` + +Per-arm rankings for every case are recorded in `benchmarks//baseline-results.json`. + The context comparison is intentionally a proxy: - **Assumed baseline:** send every scanned text-bearing file in FixMap's supported extension set—22,058,578 estimated tokens. diff --git a/packages/core/test/expected-path-mention.test.mjs b/packages/core/test/expected-path-mention.test.mjs new file mode 100644 index 0000000..efb45b8 --- /dev/null +++ b/packages/core/test/expected-path-mention.test.mjs @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import { classifyExpectedPathMention, splitCohorts } from "../../../scripts/lib/expected-path-mention.mjs"; + +// This classifier decides which cohort a benchmark case is reported in, so a silent change +// here silently changes a published rate. Each case below is drawn from a real shape seen in +// benchmarks/*/dataset.json. + +describe("expected fixing path mentions", () => { + it("detects a path stated inline with a line number", () => { + const result = classifyExpectedPathMention({ + task: "**Location:** lib/document.js:2339 (isModified -> modifiedPaths rebuild)", + expected: ["lib/document.js"] + }); + expect(result.mentionsExpectedPath).toBe(true); + expect(result.mentionTier).toBe("full-path"); + }); + + it("detects a path inside a GitHub blob permalink", () => { + // The `/` before the path belongs to the URL, not to a longer different path. Rejecting + // this shape once hid two genuine leaks that name the exact file *and* the exact lines. + const result = classifyExpectedPathMention({ + task: "see https://github.com/sveltejs/svelte/blob/4a6a85b/packages/svelte/src/boundary.js#L200-L210 for the no-op", + expected: ["packages/svelte/src/boundary.js"] + }); + expect(result.mentionsExpectedPath).toBe(true); + expect(result.mentionTier).toBe("full-path"); + }); + + it("treats a multi-segment path suffix as named", () => { + const result = classifyExpectedPathMention({ + task: "Error: src/query/react/buildHooks.ts(1823,13): error TS2345", + expected: ["packages/toolkit/src/query/react/buildHooks.ts"] + }); + expect(result.mentionsExpectedPath).toBe(true); + expect(result.mentionTier).toBe("path-suffix"); + }); + + it("does not treat a longer, different path as a mention", () => { + const result = classifyExpectedPathMention({ + task: "the failing assertion lives in test/lib/request.js", + expected: ["lib/request.js"] + }); + expect(result.mentionsExpectedPath).toBe(false); + }); + + it("does not treat a bare basename as naming the file", () => { + // "index.ts" is ordinary prose in an issue; counting it would move real cases out of the + // generalization cohort on the strength of a filename that names nothing. + const result = classifyExpectedPathMention({ + task: "the transports option array is mutated before index.ts reads it", + expected: ["packages/engine.io-client/lib/index.ts"] + }); + expect(result.mentionsExpectedPath).toBe(false); + expect(result.mentionTier).toBe("basename"); + }); + + it("does not match a longer filename that merely starts with the expected one", () => { + const result = classifyExpectedPathMention({ + task: "the regression is in lib/document.jsx", + expected: ["lib/document.js"] + }); + expect(result.mentionsExpectedPath).toBe(false); + }); + + it("reports no mention when the task never names the file", () => { + const result = classifyExpectedPathMention({ + task: "xunit reporter does not strip ansi escape sequences from failure messages", + expected: ["lib/reporters/xunit.js"] + }); + expect(result.mentionsExpectedPath).toBe(false); + expect(result.mentionTier).toBe("none"); + }); + + it("normalises Windows separators pasted from a stack trace", () => { + const result = classifyExpectedPathMention({ + task: "at Object. (lib\\winston\\transports\\file.js:120:15)", + expected: ["lib/winston/transports/file.js"] + }); + expect(result.mentionsExpectedPath).toBe(true); + }); + + it("carries evidence so a classification can be audited rather than trusted", () => { + const result = classifyExpectedPathMention({ + task: "**Location:** lib/document.js:2339 rebuilds the set", + expected: ["lib/document.js"] + }); + expect(result.evidence).toHaveLength(1); + expect(result.evidence[0].context).toContain("lib/document.js"); + }); + + it("splits cohorts so named cases never enter the generalization number", () => { + const cohorts = splitCohorts([ + { slug: "a", mentionsExpectedPath: true }, + { slug: "b", mentionsExpectedPath: false }, + { slug: "c", mentionsExpectedPath: false } + ]); + expect(cohorts.all).toHaveLength(3); + expect(cohorts.unmentioned.map((row) => row.slug)).toEqual(["b", "c"]); + expect(cohorts.mentioned.map((row) => row.slug)).toEqual(["a"]); + }); +}); From 0ca129440a7a35d2b3288a6030cbc49b5844eaec Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 11:44:35 +0530 Subject: [PATCH 4/7] eval: fix the strawman baseline, and withdraw the claim it produced The first version of this comparison pointed every baseline at the raw scan while FixMap ranked its own gated candidate set. rankContextFiles gates on isSource && !isTest and then deprioritises documentation for implementation tasks; the baselines had none of that, so they returned README.md, CONTRIBUTING.md and issue templates first. For mocha the literal arm's top five were all Markdown while the answer was lib/reporters/xunit.js. That is not a baseline, it is a handicap, and the p = 0.004-0.016 result it produced was an artifact of it. Withdrawn. Each baseline now runs under three candidate policies - raw, source (FixMap's own gate), and code (also dropping documentation) - and is compared at its STRONGEST. The conclusion reverses: held-out, unmentioned (9) top1 top3 top5 bm25:code 44.4% 55.6% 100% fixmap 44.4% 55.6% 66.7% Top-1 and Top-3 are exact ties (McNemar p = 1.0, two disagreements each way). At Top-5 the baseline wins three cases FixMap misses and FixMap wins none: bm25:code has the fixing file in its top five for 9 of 9, FixMap 6 of 9, the misses being socket.io, vite and vue. FixMap still leads the regression suite (69.2% vs 38.5% top-1) but that suite shaped the ranker, and even there the lead is not significant against this baseline (p = 0.125 top-1). Also fixes path-extraction, which peeled no segments off a GitHub permalink and so silently missed every URL-embedded path - the exact quantity that arm exists to measure. It now scores 100% on the named cohort and 0% on the unmentioned one, a clean separation that independently checks the classifier. README, docs/BENCHMARKS.md and the evidence page all state the corrected result, including that FixMap loses at Top-5 on unseen repositories. Co-Authored-By: Claude Opus 5 --- README.md | 22 +- apps/web/app/_lib/site-data.ts | 8 +- apps/web/app/evidence/page.tsx | 13 +- benchmarks/external/baseline-results.json | 1666 +++++++++++++++++++-- benchmarks/heldout/baseline-results.json | 1444 ++++++++++++++++-- docs/BENCHMARKS.md | 123 +- scripts/evaluate-baseline.mjs | 124 +- 7 files changed, 3052 insertions(+), 348 deletions(-) diff --git a/README.md b/README.md index a196370..d44db42 100644 --- a/README.md +++ b/README.md @@ -446,22 +446,28 @@ The same split on the regression suite barely moves it (69% → 69% Top-1), and #### Is this better than just searching the repository? -The fair question about a ranked file list is whether it beats what an agent already gets for free. So the same suites are scored against naive retrieval on **the same scanned corpus** — one repository scan per case, shared by every arm, so the only difference is ranking. Both keyword arms are case-insensitive and expand camelCase, which favours the baselines on purpose. +The fair question about a ranked file list is whether it beats what an agent already gets for free. The same suites are scored against naive retrieval on **the same scanned corpus** — one repository scan per case, shared by every arm. -Held-out, tasks that did not name the file (9 cases): +Candidate policy turned out to matter more than the ranking function. FixMap does not rank the raw scan: it gates on `isSource && !isTest` and then deprioritises documentation for an implementation task. A baseline pointed at every scanned file therefore returns `README.md` and `CONTRIBUTING.md` first and loses to the wrong thing. So each baseline is run under three candidate policies and **compared at its strongest**. + +Held-out, tasks that did not name the file (9 cases), each baseline at its best policy: | Arm | Top-1 | Top-3 | Top-5 | | --- | ---: | ---: | ---: | | Path extraction — read paths out of the task | 0% | 0% | 0% | -| Literal keyword search | 11% | 22% | 33% | -| BM25 retrieval | 11% | 22% | 33% | -| **FixMap** | **44%** | **56%** | **67%** | +| Literal keyword search, code files only | 22% | 44% | 67% | +| **BM25 retrieval, code files only** | **44%** | **56%** | **100%** | +| FixMap | 44% | 56% | 67% | + +**On repositories FixMap was never tuned against, BM25 over code files matches it at Top-1 and Top-3 and beats it at Top-5.** Paired McNemar exact tests put Top-1 and Top-3 at p = 1.0 — dead ties, two disagreements each way. At Top-5 the baseline wins 3 cases FixMap misses and FixMap wins none: BM25 has the fixing file in its top five for **9 of 9** cases, FixMap for 6 of 9. + +On the regression suite FixMap does lead — 69% vs 39% Top-1, 100% vs 62% Top-3 — but that is the suite whose cases shaped the ranker, and even there the lead is not significant against this baseline (p = 0.125 Top-1, p = 0.0625 Top-3). -FixMap wins on the regression suite too (69% vs 15% Top-1 against BM25 on its 13 unmentioned cases), and it never loses a head-to-head case against any baseline on that suite — a paired McNemar exact test puts that at p = 0.004–0.016. On the nine held-out cases the direction is identical and FixMap still never loses a disagreement, but it **cannot** reach significance there: with three disagreeing cases the smallest possible two-sided p-value is 0.25. That is a limit of the sample size, not a negative result, and it is the strongest argument for growing the held-out suite. +We are publishing this because it is what the measurement says. The honest reading is that FixMap's current advantage over plain BM25-over-code is **unproven on unseen repositories**, and that its Top-5 recall is behind. Closing that gap is the next piece of work, not a marketing line. -Path extraction scoring exactly 0% on this cohort and 67% on the named one is the check that the cohort split is measuring what it claims. +Path extraction scoring 0% on this cohort and 100% on the named one is the check that the cohort split measures what it claims. -Reproduce it with `node scripts/evaluate-baseline.mjs --suite heldout`; every arm's ranking is recorded in [`benchmarks/heldout/baseline-results.json`](benchmarks/heldout/baseline-results.json). +Reproduce it with `node scripts/evaluate-baseline.mjs --suite heldout`; every arm, policy, and ranking is recorded in [`benchmarks/heldout/baseline-results.json`](benchmarks/heldout/baseline-results.json). One thing the point estimates hide: held-out Top-1 stays close to Top-3 in both cohorts — **when FixMap finds the file at all, it usually ranks it first**, which is what matters to an agent that opens one file. The tuned suite's 100% Top-3 still conceals that in 31% of those cases something wrong ranks above the answer, so an agent following it opens the wrong file first. diff --git a/apps/web/app/_lib/site-data.ts b/apps/web/app/_lib/site-data.ts index 92791f4..3735dbb 100644 --- a/apps/web/app/_lib/site-data.ts +++ b/apps/web/app/_lib/site-data.ts @@ -32,12 +32,14 @@ const cohortOf = (suite: typeof heldout) => ({ }); // The comparison a ranked list actually has to win: naive retrieval on the same corpus. +// Each baseline is reported at its STRONGEST candidate policy — pointing a baseline at every +// scanned file makes it rank READMEs and turns the comparison into a strawman. const baselineOf = (suite: typeof heldoutBaseline) => ({ cases: suite.arms.fixmap.unmentioned.cases, fixmap: suite.arms.fixmap.unmentioned, - bm25: suite.arms.bm25.unmentioned, - lexical: suite.arms["lexical-literal"].unmentioned, - pathExtraction: suite.arms["path-extraction"].unmentioned + bm25: suite.arms["bm25:code"].unmentioned, + lexical: suite.arms["lexical-literal:code"].unmentioned, + pathExtraction: suite.arms["path-extraction:raw"].unmentioned }); export const siteStats = { diff --git a/apps/web/app/evidence/page.tsx b/apps/web/app/evidence/page.tsx index 1040bc5..d9513f4 100644 --- a/apps/web/app/evidence/page.tsx +++ b/apps/web/app/evidence/page.tsx @@ -85,18 +85,19 @@ export default function EvidencePage() {

A ranked list only earns its place if it beats what an agent already gets for free. Every arm below scores on the same repository scan — same files, same text - samples — so the only difference is ranking. Both keyword arms are case-insensitive and - expand camelCase, which favours the baselines on purpose. + samples. Each baseline is shown at its strongest candidate policy: + pointed at every scanned file a keyword search just returns README.md, which + would make this a strawman rather than a comparison.

ArmTop 1Top 3Top 5
Path extraction — read paths out of the task{rate(heldoutBaselines.pathExtraction.top1HitRate)}{rate(heldoutBaselines.pathExtraction.top3HitRate)}{rate(heldoutBaselines.pathExtraction.top5HitRate)}
-
Literal keyword search{rate(heldoutBaselines.lexical.top1HitRate)}{rate(heldoutBaselines.lexical.top3HitRate)}{rate(heldoutBaselines.lexical.top5HitRate)}
-
BM25 retrieval{rate(heldoutBaselines.bm25.top1HitRate)}{rate(heldoutBaselines.bm25.top3HitRate)}{rate(heldoutBaselines.bm25.top5HitRate)}
-
FixMap{rate(heldoutBaselines.fixmap.top1HitRate)}{rate(heldoutBaselines.fixmap.top3HitRate)}{rate(heldoutBaselines.fixmap.top5HitRate)}
+
Literal keyword search, code files only{rate(heldoutBaselines.lexical.top1HitRate)}{rate(heldoutBaselines.lexical.top3HitRate)}{rate(heldoutBaselines.lexical.top5HitRate)}
+
BM25 retrieval, code files only{rate(heldoutBaselines.bm25.top1HitRate)}{rate(heldoutBaselines.bm25.top3HitRate)}{rate(heldoutBaselines.bm25.top5HitRate)}
+
FixMap{rate(heldoutBaselines.fixmap.top1HitRate)}{rate(heldoutBaselines.fixmap.top3HitRate)}{rate(heldoutBaselines.fixmap.top5HitRate)}
-

The held-out lead is real in direction but underpowered. FixMap wins the same comparison on the regression suite at p = 0.004–0.016 by a paired McNemar exact test, and never loses a disagreeing case there. On these {heldoutBaselines.cases} held-out cases it still never loses a disagreement, but it cannot reach significance: with three disagreeing cases the smallest possible two-sided p-value is 0.25. That is a sample-size limit, not a negative result. Path extraction scoring 0% here — and 67% on the cases that named the file — is the check that the cohort split measures what it claims.

+

FixMap does not beat BM25 over code files on repositories it was never tuned against. Top 1 and Top 3 are exact ties — a paired McNemar exact test puts both at p = 1.0, with two disagreements each way. At Top 5 the baseline wins three cases FixMap misses and FixMap wins none: BM25 has the fixing file in its top five for 9 of 9 of these cases, FixMap for 6 of 9. FixMap does lead on the regression suite (69% vs 39% Top 1), but that is the suite whose cases shaped the ranker, and even there the lead is not significant against this baseline. We publish this because it is what the measurement says; closing the Top-5 recall gap is the next piece of work.

diff --git a/benchmarks/external/baseline-results.json b/benchmarks/external/baseline-results.json index cecf2f3..853377d 100644 --- a/benchmarks/external/baseline-results.json +++ b/benchmarks/external/baseline-results.json @@ -3,7 +3,7 @@ "cases": 16, "configuration": { "topN": 5, - "corpus": "one scanRepo() result per case, shared by every arm; ranking is the only difference", + "corpus": "one scanRepo() result per case, shared by every arm", "searchField": "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", "tokenizer": "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", "stopwords": 158, @@ -12,32 +12,42 @@ "k1": 1.2, "b": 0.75 }, + "candidatePolicies": { + "raw": "every scanned file; ranks READMEs first and is not a fair comparison", + "source": "isSource && !isTest — FixMap's own candidate gate", + "code": "isSource && !isTest && kind === 'code' — also drops documentation, as FixMap's scoring effectively does for implementation tasks" + }, + "bestPolicyPerFamily": { + "path-extraction": "raw", + "lexical-literal": "code", + "bm25": "code" + }, "armDescriptions": { "path-extraction": "path-shaped tokens read out of the task text, resolved against the corpus, ranked by order of appearance", "lexical-literal": "literal keyword search ranked by distinct query terms matched, then raw occurrence count", "bm25": "BM25 retrieval over the same text; a retrieval baseline, not a grep", - "fixmap": "rankContextFiles from @aryam/fixmap-core" + "fixmap": "rankContextFiles from @aryam/fixmap-core, which applies its own candidate gate internally" } }, "arms": { - "path-extraction": { + "path-extraction:raw": { "all": { "cases": 16, - "top1HitRate": 0.125, - "top3HitRate": 0.125, - "top5HitRate": 0.125, + "top1HitRate": 0.188, + "top3HitRate": 0.188, + "top5HitRate": 0.188, "intervals95": { "top1": [ - 0.035, - 0.36 + 0.066, + 0.43 ], "top3": [ - 0.035, - 0.36 + 0.066, + 0.43 ], "top5": [ - 0.035, - 0.36 + 0.066, + 0.43 ] } }, @@ -63,26 +73,150 @@ }, "mentioned": { "cases": 3, - "top1HitRate": 0.667, - "top3HitRate": 0.667, - "top5HitRate": 0.667, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, "intervals95": { "top1": [ - 0.208, - 0.939 + 0.438, + 1 ], "top3": [ - 0.208, - 0.939 + 0.438, + 1 ], "top5": [ - 0.208, - 0.939 + 0.438, + 1 + ] + } + } + }, + "path-extraction:source": { + "all": { + "cases": 16, + "top1HitRate": 0.188, + "top3HitRate": 0.188, + "top5HitRate": 0.188, + "intervals95": { + "top1": [ + 0.066, + 0.43 + ], + "top3": [ + 0.066, + 0.43 + ], + "top5": [ + 0.066, + 0.43 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0, + "top3HitRate": 0, + "top5HitRate": 0, + "intervals95": { + "top1": [ + 0, + 0.228 + ], + "top3": [ + 0, + 0.228 + ], + "top5": [ + 0, + 0.228 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + } + } + }, + "path-extraction:code": { + "all": { + "cases": 16, + "top1HitRate": 0.188, + "top3HitRate": 0.188, + "top5HitRate": 0.188, + "intervals95": { + "top1": [ + 0.066, + 0.43 + ], + "top3": [ + 0.066, + 0.43 + ], + "top5": [ + 0.066, + 0.43 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0, + "top3HitRate": 0, + "top5HitRate": 0, + "intervals95": { + "top1": [ + 0, + 0.228 + ], + "top3": [ + 0, + 0.228 + ], + "top5": [ + 0, + 0.228 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 ] } } }, - "lexical-literal": { + "lexical-literal:raw": { "all": { "cases": 16, "top1HitRate": 0.125, @@ -144,7 +278,131 @@ } } }, - "bm25": { + "lexical-literal:source": { + "all": { + "cases": 16, + "top1HitRate": 0.125, + "top3HitRate": 0.313, + "top5HitRate": 0.313, + "intervals95": { + "top1": [ + 0.035, + 0.36 + ], + "top3": [ + 0.142, + 0.556 + ], + "top5": [ + 0.142, + 0.556 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.077, + "top3HitRate": 0.308, + "top5HitRate": 0.308, + "intervals95": { + "top1": [ + 0.014, + 0.333 + ], + "top3": [ + 0.127, + 0.576 + ], + "top5": [ + 0.127, + 0.576 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "lexical-literal:code": { + "all": { + "cases": 16, + "top1HitRate": 0.313, + "top3HitRate": 0.313, + "top5HitRate": 0.313, + "intervals95": { + "top1": [ + 0.142, + 0.556 + ], + "top3": [ + 0.142, + 0.556 + ], + "top5": [ + 0.142, + 0.556 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.308, + "top3HitRate": 0.308, + "top5HitRate": 0.308, + "intervals95": { + "top1": [ + 0.127, + 0.576 + ], + "top3": [ + 0.127, + 0.576 + ], + "top5": [ + 0.127, + 0.576 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "bm25:raw": { "all": { "cases": 16, "top1HitRate": 0.188, @@ -206,6 +464,130 @@ } } }, + "bm25:source": { + "all": { + "cases": 16, + "top1HitRate": 0.25, + "top3HitRate": 0.375, + "top5HitRate": 0.375, + "intervals95": { + "top1": [ + 0.102, + 0.495 + ], + "top3": [ + 0.185, + 0.614 + ], + "top5": [ + 0.185, + 0.614 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.231, + "top3HitRate": 0.385, + "top5HitRate": 0.385, + "intervals95": { + "top1": [ + 0.082, + 0.503 + ], + "top3": [ + 0.177, + 0.645 + ], + "top5": [ + 0.177, + 0.645 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "bm25:code": { + "all": { + "cases": 16, + "top1HitRate": 0.375, + "top3HitRate": 0.563, + "top5HitRate": 0.563, + "intervals95": { + "top1": [ + 0.185, + 0.614 + ], + "top3": [ + 0.332, + 0.769 + ], + "top5": [ + 0.332, + 0.769 + ] + } + }, + "unmentioned": { + "cases": 13, + "top1HitRate": 0.385, + "top3HitRate": 0.615, + "top5HitRate": 0.615, + "intervals95": { + "top1": [ + 0.177, + 0.645 + ], + "top3": [ + 0.355, + 0.823 + ], + "top5": [ + 0.355, + 0.823 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.061, + 0.792 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, "fixmap": { "all": { "cases": 16, @@ -271,32 +653,32 @@ }, "pairedVsFixmapMcnemarExact": { "all": { - "path-extraction": { + "path-extraction:raw": { "top1": { - "aWins": 10, + "aWins": 9, "bWins": 1, - "discordant": 11, - "pValue": 0.0117 + "discordant": 10, + "pValue": 0.0215 }, "top3": { - "aWins": 14, + "aWins": 13, "bWins": 0, - "discordant": 14, - "pValue": 0.0001 + "discordant": 13, + "pValue": 0.0002 }, "top5": { - "aWins": 14, + "aWins": 13, "bWins": 0, - "discordant": 14, - "pValue": 0.0001 + "discordant": 13, + "pValue": 0.0002 } }, - "lexical-literal": { + "lexical-literal:code": { "top1": { - "aWins": 9, + "aWins": 6, "bWins": 0, - "discordant": 9, - "pValue": 0.0039 + "discordant": 6, + "pValue": 0.0313 }, "top3": { "aWins": 11, @@ -311,29 +693,29 @@ "pValue": 0.001 } }, - "bm25": { + "bm25:code": { "top1": { - "aWins": 8, + "aWins": 5, "bWins": 0, - "discordant": 8, - "pValue": 0.0078 + "discordant": 5, + "pValue": 0.0625 }, "top3": { - "aWins": 11, + "aWins": 7, "bWins": 0, - "discordant": 11, - "pValue": 0.001 + "discordant": 7, + "pValue": 0.0156 }, "top5": { - "aWins": 10, + "aWins": 7, "bWins": 0, - "discordant": 10, - "pValue": 0.002 + "discordant": 7, + "pValue": 0.0156 } } }, "unmentioned": { - "path-extraction": { + "path-extraction:raw": { "top1": { "aWins": 9, "bWins": 0, @@ -353,12 +735,12 @@ "pValue": 0.0002 } }, - "lexical-literal": { + "lexical-literal:code": { "top1": { - "aWins": 8, + "aWins": 5, "bWins": 0, - "discordant": 8, - "pValue": 0.0078 + "discordant": 5, + "pValue": 0.0625 }, "top3": { "aWins": 9, @@ -373,24 +755,24 @@ "pValue": 0.0039 } }, - "bm25": { + "bm25:code": { "top1": { - "aWins": 7, + "aWins": 4, "bWins": 0, - "discordant": 7, - "pValue": 0.0156 + "discordant": 4, + "pValue": 0.125 }, "top3": { - "aWins": 9, + "aWins": 5, "bWins": 0, - "discordant": 9, - "pValue": 0.0039 + "discordant": 5, + "pValue": 0.0625 }, "top5": { - "aWins": 8, + "aWins": 5, "bWins": 0, - "discordant": 8, - "pValue": 0.0078 + "discordant": 5, + "pValue": 0.0625 } } } @@ -402,11 +784,24 @@ "lib/request.js" ], "scannedFiles": 213, + "candidateCounts": { + "raw": 213, + "source": 65, + "code": 54 + }, "queryTermCount": 47, "mentionsExpectedPath": true, "mentionTier": "full-path", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [ + "lib/request.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:source": { "top5": [ "lib/request.js" ], @@ -414,7 +809,15 @@ "top3": true, "top5Hit": true }, - "lexical-literal": { + "path-extraction:code": { + "top5": [ + "lib/request.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:raw": { "top5": [ "lib/request.js", "lib/response.js", @@ -426,7 +829,31 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "lib/request.js", + "lib/response.js", + "lib/application.js", + "Readme.md", + "examples/auth/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:code": { + "top5": [ + "lib/request.js", + "lib/response.js", + "lib/application.js", + "examples/auth/index.js", + "examples/web-service/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "lib/request.js", "test/req.fresh.js", @@ -438,6 +865,30 @@ "top3": true, "top5Hit": true }, + "bm25:source": { + "top5": [ + "lib/request.js", + "lib/response.js", + "lib/application.js", + "lib/express.js", + "examples/mvc/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "lib/request.js", + "lib/response.js", + "lib/application.js", + "lib/express.js", + "examples/search/public/client.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/request.js", @@ -458,17 +909,34 @@ "lib/core/AxiosError.js" ], "scannedFiles": 454, + "candidateCounts": { + "raw": 454, + "source": 266, + "code": 80 + }, "queryTermCount": 36, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "tests/unit/prototypePollution.test.js", "lib/adapters/http.js", @@ -480,7 +948,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "lib/adapters/http.js", + "docs/pages/advanced/request-config.md", + "MIGRATION_GUIDE.md", + "AGENTS.md", + "docs/pages/advanced/api-reference.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "lib/adapters/http.js", + "index.d.cts", + "index.d.ts", + "lib/utils.js", + "lib/adapters/xhr.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "MIGRATION_GUIDE.md", "docs/pages/getting-started/first-steps.md", @@ -492,6 +984,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "MIGRATION_GUIDE.md", + "docs/pages/advanced/testing.md", + "docs/pages/getting-started/first-steps.md", + "docs/zh/pages/advanced/testing.md", + "docs/pages/advanced/x-www-form-urlencoded-format.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "lib/core/AxiosError.js", + "lib/core/InterceptorManager.js", + "gulpfile.js", + "lib/adapters/xhr.js", + "index.d.cts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/core/AxiosError.js", @@ -512,17 +1028,46 @@ "src/node.js" ], "scannedFiles": 14, + "candidateCounts": { + "raw": 14, + "source": 11, + "code": 6 + }, "queryTermCount": 30, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "README.md", + "src/node.js", + "src/common.js", + "src/browser.js", + ".github/PULL_REQUEST_TEMPLATE.md" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "lexical-literal:source": { "top5": [ "README.md", "src/node.js", @@ -534,7 +1079,31 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "src/node.js", + "src/common.js", + "src/browser.js", + "test.js", + "karma.conf.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { + "top5": [ + "src/node.js", + "README.md", + "src/common.js", + "src/browser.js", + ".github/PULL_REQUEST_TEMPLATE.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:source": { "top5": [ "src/node.js", "README.md", @@ -546,6 +1115,18 @@ "top3": true, "top5Hit": true }, + "bm25:code": { + "top5": [ + "src/node.js", + "src/browser.js", + "src/common.js", + "karma.conf.js", + "test.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "src/node.js", @@ -566,17 +1147,34 @@ "source/core/Ky.ts" ], "scannedFiles": 67, + "candidateCounts": { + "raw": 67, + "source": 36, + "code": 29 + }, "queryTermCount": 42, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "readme.md", "source/core/Ky.ts", @@ -588,7 +1186,31 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "readme.md", + "source/core/Ky.ts", + "source/types/options.ts", + "source/types/hooks.ts", + "source/core/constants.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "lexical-literal:code": { + "top5": [ + "source/core/Ky.ts", + "source/types/options.ts", + "source/types/hooks.ts", + "source/core/constants.ts", + "source/utils/merge.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "readme.md", "source/core/Ky.ts", @@ -600,6 +1222,30 @@ "top3": true, "top5Hit": true }, + "bm25:source": { + "top5": [ + "readme.md", + "source/core/Ky.ts", + "source/core/constants.ts", + "source/types/options.ts", + "source/errors/HTTPError.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "source/core/Ky.ts", + "source/core/constants.ts", + "source/types/options.ts", + "source/errors/HTTPError.ts", + "source/utils/merge.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "source/core/Ky.ts", @@ -620,17 +1266,34 @@ "packages/zod/src/v4/core/regexes.ts" ], "scannedFiles": 582, + "candidateCounts": { + "raw": 582, + "source": 302, + "code": 228 + }, "queryTermCount": 38, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "packages/docs-v3/README_ZH.md", "packages/zod/src/v4/mini/schemas.ts", @@ -642,7 +1305,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "packages/docs-v3/README_ZH.md", + "packages/zod/src/v4/mini/schemas.ts", + "packages/zod/src/v4/classic/from-json-schema.ts", + "packages/docs-v3/ERROR_HANDLING.md", + "packages/zod/src/v4/core/checks.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "packages/zod/src/v4/mini/schemas.ts", + "packages/zod/src/v4/classic/from-json-schema.ts", + "packages/zod/src/v4/core/checks.ts", + "packages/zod/src/v4/core/json-schema-processors.ts", + "packages/zod/src/v4/core/api.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "packages/zod/src/v4/mini/tests/string.test.ts", "packages/zod/src/v4/classic/tests/detached-methods.test.ts", @@ -654,6 +1341,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "packages/docs-v3/README_ZH.md", + "packages/zod/README.md", + "wiki/optionality.md", + "packages/resolution/src/index.mts", + "packages/resolution/src/index.cts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "packages/resolution/src/index.mts", + "packages/resolution/src/index.cts", + "packages/resolution/src/index.ts", + "packages/zod/src/v4/core/to-json-schema.ts", + "packages/docs/components/ecosystem.tsx" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "packages/zod/src/v4/core/regexes.ts", @@ -675,29 +1386,70 @@ "lib/worker.js" ], "scannedFiles": 200, + "candidateCounts": { + "raw": 200, + "source": 67, + "code": 40 + }, "queryTermCount": 28, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "path-extraction:code": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:raw": { + "top5": [ + "docs/transports.md", + "docs/api.md", + "pino.d.ts", + "docs/help.md", + "test/transport/core.test.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "docs/transports.md", "docs/api.md", "pino.d.ts", "docs/help.md", - "test/transport/core.test.js" + "pino.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "pino.d.ts", + "pino.js", + "lib/tools.js", + "examples/transport.js", + "browser.js" ], "top1": false, "top3": false, "top5Hit": false }, - "bm25": { + "bm25:raw": { "top5": [ "docs/api.md", "test/transport/core.test.js", @@ -709,6 +1461,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "docs/api.md", + "docs/transports.md", + "docs/help.md", + "examples/transport.js", + "pino.d.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "examples/transport.js", + "pino.d.ts", + "lib/transport.js", + "pino.js", + "lib/tools.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "pino.d.ts", @@ -729,17 +1505,46 @@ "lib/request.js" ], "scannedFiles": 393, + "candidateCounts": { + "raw": 393, + "source": 152, + "code": 72 + }, "queryTermCount": 20, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "docs/Reference/Errors.md", + "docs/Reference/Routes.md", + "SECURITY.md", + "docs/Reference/TypeScript.md", + "docs/Guides/Migration-Guide-V5.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "docs/Reference/Errors.md", "docs/Reference/Routes.md", @@ -751,7 +1556,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "lib/handle-request.js", + "scripts/validate-ecosystem-links.js", + "fastify.js", + "lib/reply.js", + "lib/server.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "SECURITY.md", "docs/Reference/Errors.md", @@ -763,6 +1580,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "SECURITY.md", + "docs/Reference/Errors.md", + ".github/workflows/links-check.yml", + "docs/Reference/Routes.md", + "docs/Guides/Serverless.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "scripts/validate-ecosystem-links.js", + "lib/handle-request.js", + "types/schema.d.ts", + "lib/logger-pino.js", + "lib/server.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "lib/request.js", @@ -783,17 +1624,34 @@ "source/vendor/supports-color/index.js" ], "scannedFiles": 33, + "candidateCounts": { + "raw": 33, + "source": 19, + "code": 12 + }, "queryTermCount": 34, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "readme.md", "source/vendor/supports-color/index.js", @@ -805,7 +1663,31 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "readme.md", + "source/vendor/supports-color/index.js", + "source/vendor/supports-color/index.d.ts", + "source/index.d.ts", + "source/vendor/supports-color/browser.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "lexical-literal:code": { + "top5": [ + "source/vendor/supports-color/index.js", + "source/vendor/supports-color/index.d.ts", + "source/index.d.ts", + "source/vendor/supports-color/browser.js", + "source/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "source/vendor/supports-color/index.js", "test/no-color-support.js", @@ -817,6 +1699,30 @@ "top3": true, "top5Hit": true }, + "bm25:source": { + "top5": [ + "source/vendor/supports-color/index.js", + "readme.md", + "source/vendor/supports-color/index.d.ts", + "source/vendor/supports-color/browser.js", + "source/index.d.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "source/vendor/supports-color/index.js", + "source/vendor/supports-color/index.d.ts", + "source/vendor/supports-color/browser.js", + "examples/rainbow.js", + "source/index.d.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "source/vendor/supports-color/index.js", @@ -837,17 +1743,46 @@ "packages/browser/src/client/tester/dialog.ts" ], "scannedFiles": 3050, + "candidateCounts": { + "raw": 3050, + "source": 1004, + "code": 610 + }, "queryTermCount": 46, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "docs/guide/browser/visual-regression-testing.md", + "docs/api/vi.md", + "docs/guide/migration.md", + "docs/guide/learn/debugging-tests.md", + "docs/blog/vitest-4-1.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "docs/guide/browser/visual-regression-testing.md", "docs/api/vi.md", @@ -859,7 +1794,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "packages/vitest/src/node/types/browser.ts", + "packages/browser/src/client/orchestrator.ts", + "packages/vitest/src/node/cli/cli-config.ts", + "packages/vitest/src/integrations/vi.ts", + "packages/browser/context.d.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ ".github/ISSUE_TEMPLATE/feature_request.yml", "docs/guide/browser/index.md", @@ -871,6 +1818,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + ".github/ISSUE_TEMPLATE/feature_request.yml", + "docs/guide/learn/debugging-tests.md", + "docs/guide/browser/index.md", + "docs/blog/vitest-4-1.md", + "docs/api/vi.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "packages/vitest/src/integrations/vi.ts", + "packages/browser/src/client/tester/dialog.ts", + "packages/vitest/src/node/cli/cli-config.ts", + "packages/spy/src/types.ts", + "packages/spy/src/index.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "packages/vitest/src/integrations/env/jsdom-keys.ts", @@ -891,17 +1862,46 @@ "lib/types/config-api.d.ts" ], "scannedFiles": 2297, + "candidateCounts": { + "raw": 2297, + "source": 953, + "code": 445 + }, "queryTermCount": 34, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "docs/src/extend/custom-rules.md", + "lib/linter/linter.js", + "docs/src/use/migrate-to-9.0.0.md", + "lib/types/index.d.ts", + "docs/src/use/configure/migration-guide.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "docs/src/extend/custom-rules.md", "lib/linter/linter.js", @@ -913,7 +1913,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "lib/linter/linter.js", + "lib/types/index.d.ts", + "lib/eslint/eslint.js", + "lib/languages/js/source-code/source-code.js", + "lib/eslint/eslint-helpers.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "docs/src/use/configure/migration-guide.md", "docs/src/extend/plugin-migration-flat-config.md", @@ -925,6 +1937,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "docs/src/use/configure/migration-guide.md", + "docs/src/use/migrate-to-9.0.0.md", + "docs/src/extend/plugin-migration-flat-config.md", + "docs/src/use/migrate-to-10.0.0.md", + "docs/src/extend/custom-rule-tutorial.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "lib/options.js", + "lib/linter/linter.js", + "lib/types/index.d.ts", + "lib/languages/js/source-code/source-code.js", + "messages/all-matched-files-ignored.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "lib/types/config-api.d.ts", @@ -945,29 +1981,70 @@ "lib/DefinePlugin.js" ], "scannedFiles": 13454, + "candidateCounts": { + "raw": 13454, + "source": 1509, + "code": 1158 + }, "queryTermCount": 38, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "AGENTS.md", + "lib/Module.js", + "CONTRIBUTING.md", + "lib/SourceMapDevToolPlugin.js", + "examples/css/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { + "top5": [ + "AGENTS.md", + "lib/Module.js", + "CONTRIBUTING.md", + "lib/SourceMapDevToolPlugin.js", + "examples/css/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { "top5": [ - "AGENTS.md", "lib/Module.js", - "CONTRIBUTING.md", "lib/SourceMapDevToolPlugin.js", - "examples/css/README.md" + "lib/dependencies/HtmlEntryDependency.js", + "lib/WebpackOptionsApply.js", + "lib/RuntimeTemplate.js" ], "top1": false, "top3": false, "top5Hit": false }, - "bm25": { + "bm25:raw": { "top5": [ "CONTRIBUTING.md", ".changeset/README.md", @@ -979,6 +2056,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "CONTRIBUTING.md", + "AGENTS.md", + "open-bot.yaml", + "lib/DotenvPlugin.js", + ".github/workflows/test.yml" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "lib/DotenvPlugin.js", + "lib/WebpackOptionsApply.js", + "lib/config/target.js", + "lib/CleanPlugin.js", + "bin/webpack.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "lib/dependencies/ImportMetaPlugin.js", @@ -999,17 +2100,34 @@ "lib/dispatcher/client-h2.js" ], "scannedFiles": 786, + "candidateCounts": { + "raw": 786, + "source": 300, + "code": 209 + }, "queryTermCount": 52, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "lib/dispatcher/client-h2.js", "test/http2-dispatcher.js", @@ -1021,7 +2139,31 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "lib/dispatcher/client-h2.js", + "docs/docs/index.md", + "README.md", + "lib/dispatcher/client-h1.js", + "lib/web/fetch/request.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:code": { + "top5": [ + "lib/dispatcher/client-h2.js", + "lib/dispatcher/client-h1.js", + "lib/web/fetch/request.js", + "types/dispatcher.d.ts", + "lib/web/fetch/util.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "test/http2-late-data.js", "lib/dispatcher/client-h2.js", @@ -1033,6 +2175,30 @@ "top3": true, "top5Hit": true }, + "bm25:source": { + "top5": [ + "lib/dispatcher/client-h2.js", + "lib/api/api-stream.js", + "types/dispatcher.d.ts", + "lib/dispatcher/client-h1.js", + "lib/web/eventsource/eventsource.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "lib/dispatcher/client-h2.js", + "lib/api/api-stream.js", + "lib/dispatcher/client-h1.js", + "types/dispatcher.d.ts", + "lib/web/eventsource/eventsource.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/dispatcher/client-h2.js", @@ -1053,11 +2219,32 @@ "packages/toolkit/src/query/react/buildHooks.ts" ], "scannedFiles": 1152, + "candidateCounts": { + "raw": 1152, + "source": 802, + "code": 595 + }, "queryTermCount": 47, "mentionsExpectedPath": true, "mentionTier": "path-suffix", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [ + "packages/toolkit/src/query/react/buildHooks.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:source": { + "top5": [ + "packages/toolkit/src/query/react/buildHooks.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:code": { "top5": [ "packages/toolkit/src/query/react/buildHooks.ts" ], @@ -1065,7 +2252,7 @@ "top3": true, "top5Hit": true }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "packages/toolkit/src/query/core/buildThunks.ts", "packages/toolkit/src/query/core/module.ts", @@ -1077,7 +2264,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/core/module.ts", + "packages/toolkit/src/query/core/apiState.ts", + "_artifacts/domain_map.yaml", + "packages/toolkit/src/query/endpointDefinitions.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/core/module.ts", + "packages/toolkit/src/query/core/apiState.ts", + "packages/toolkit/src/query/endpointDefinitions.ts", + "packages/toolkit/src/query/core/buildSlice.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "packages/toolkit/src/query/core/apiState.ts", "packages/toolkit/src/query/core/buildSlice.ts", @@ -1089,6 +2300,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "packages/toolkit/src/query/core/apiState.ts", + "packages/toolkit/src/query/core/buildSlice.ts", + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/core/buildSelectors.ts", + "packages/toolkit/src/query/core/buildMiddleware/cacheLifecycle.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "packages/toolkit/src/query/core/apiState.ts", + "packages/toolkit/src/query/core/buildSlice.ts", + "packages/toolkit/src/query/core/buildThunks.ts", + "packages/toolkit/src/query/core/buildSelectors.ts", + "packages/toolkit/src/query/core/buildMiddleware/queryLifecycle.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "packages/toolkit/src/query/core/apiState.ts", @@ -1109,17 +2344,34 @@ "src/language-css/print/comma-separated-value-group.js" ], "scannedFiles": 9319, + "candidateCounts": { + "raw": 9319, + "source": 881, + "code": 720 + }, "queryTermCount": 61, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "website/blog/2021-05-09-2.3.0.md", "tests/format/markdown/markdown/real-world-case.md", @@ -1131,7 +2383,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "website/blog/2021-05-09-2.3.0.md", + "website/blog/2018-04-11-1.12.0.md", + "website/blog/2017-11-07-1.8.0.md", + "website/blog/2020-03-21-2.0.0.md", + "website/blog/2018-05-27-1.13.0.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "src/index.d.ts", + "src/language-js/parentheses/needs-parentheses.js", + "src/language-js/types/flow-estree.d.ts", + "src/language-js/print/binaryish.js", + "src/language-css/parser-postcss.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "website/blog/2017-04-20-1.2.0.md", "README.md", @@ -1143,6 +2419,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "website/blog/2017-04-20-1.2.0.md", + "website/blog/2023-11-13-curious-ternaries.md", + "website/blog/2017-11-07-1.8.0.md", + "website/blog/2023-11-30-cli-deep-dive.md", + "docs/rationale.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "scripts/utilities/changelog.js", + "website/playground/utilities.js", + "src/main/core-options.evaluate.js", + "website/src/pages/index.jsx", + "src/language-css/parser-postcss.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "src/language-css/print/comma-separated-value-group.js", @@ -1163,17 +2463,34 @@ "src/adapter/aws-lambda/types.ts" ], "scannedFiles": 482, + "candidateCounts": { + "raw": 482, + "source": 304, + "code": 243 + }, "queryTermCount": 35, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:raw": { "top5": [ "runtime-tests/lambda/index.test.ts", "src/adapter/aws-lambda/handler.ts", @@ -1185,7 +2502,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "src/adapter/aws-lambda/handler.ts", + "docs/MIGRATION.md", + "src/jsx/dom/render.ts", + "src/context.ts", + "src/hono-base.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "src/adapter/aws-lambda/handler.ts", + "src/jsx/dom/render.ts", + "src/context.ts", + "src/hono-base.ts", + "src/adapter/lambda-edge/handler.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "runtime-tests/lambda/stream.test.ts", "runtime-tests/lambda/index.test.ts", @@ -1197,6 +2538,30 @@ "top3": false, "top5Hit": true }, + "bm25:source": { + "top5": [ + "src/adapter/aws-lambda/handler.ts", + "src/adapter/aws-lambda/types.ts", + "src/adapter/aws-lambda/index.ts", + "src/adapter/lambda-edge/handler.ts", + "runtime-tests/lambda/stream-mock.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "src/adapter/aws-lambda/handler.ts", + "src/adapter/aws-lambda/types.ts", + "src/adapter/aws-lambda/index.ts", + "runtime-tests/lambda/stream-mock.ts", + "runtime-tests/lambda/mock.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "src/adapter/aws-lambda/handler.ts", @@ -1217,17 +2582,52 @@ "source/core/index.ts" ], "scannedFiles": 120, + "candidateCounts": { + "raw": 120, + "source": 62, + "code": 31 + }, "queryTermCount": 42, "mentionsExpectedPath": true, "mentionTier": "full-path", "arms": { - "path-extraction": { - "top5": [], + "path-extraction:raw": { + "top5": [ + "source/core/index.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:source": { + "top5": [ + "source/core/index.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:code": { + "top5": [ + "source/core/index.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:raw": { + "top5": [ + "documentation/2-options.md", + "documentation/3-streams.md", + "documentation/tips.md", + "documentation/migration-guides/request.md", + "documentation/4-pagination.md" + ], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:source": { "top5": [ "documentation/2-options.md", "documentation/3-streams.md", @@ -1239,7 +2639,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "source/core/response.ts", + "source/types.ts", + "source/as-promise/index.ts", + "source/core/errors.ts", + "source/as-promise/types.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "documentation/2-options.md", "documentation/4-pagination.md", @@ -1251,6 +2663,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "documentation/2-options.md", + "source/core/response.ts", + "documentation/4-pagination.md", + "documentation/migration-guides/request.md", + "documentation/tips.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "source/core/response.ts", + "source/as-promise/types.ts", + "source/as-promise/index.ts", + "source/types.ts", + "source/core/errors.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "source/core/index.ts", diff --git a/benchmarks/heldout/baseline-results.json b/benchmarks/heldout/baseline-results.json index 19fcfd1..4e831c3 100644 --- a/benchmarks/heldout/baseline-results.json +++ b/benchmarks/heldout/baseline-results.json @@ -3,7 +3,7 @@ "cases": 12, "configuration": { "topN": 5, - "corpus": "one scanRepo() result per case, shared by every arm; ranking is the only difference", + "corpus": "one scanRepo() result per case, shared by every arm", "searchField": "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", "tokenizer": "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", "stopwords": 158, @@ -12,32 +12,42 @@ "k1": 1.2, "b": 0.75 }, + "candidatePolicies": { + "raw": "every scanned file; ranks READMEs first and is not a fair comparison", + "source": "isSource && !isTest — FixMap's own candidate gate", + "code": "isSource && !isTest && kind === 'code' — also drops documentation, as FixMap's scoring effectively does for implementation tasks" + }, + "bestPolicyPerFamily": { + "path-extraction": "raw", + "lexical-literal": "code", + "bm25": "code" + }, "armDescriptions": { "path-extraction": "path-shaped tokens read out of the task text, resolved against the corpus, ranked by order of appearance", "lexical-literal": "literal keyword search ranked by distinct query terms matched, then raw occurrence count", "bm25": "BM25 retrieval over the same text; a retrieval baseline, not a grep", - "fixmap": "rankContextFiles from @aryam/fixmap-core" + "fixmap": "rankContextFiles from @aryam/fixmap-core, which applies its own candidate gate internally" } }, "arms": { - "path-extraction": { + "path-extraction:raw": { "all": { "cases": 12, - "top1HitRate": 0.167, - "top3HitRate": 0.167, - "top5HitRate": 0.167, + "top1HitRate": 0.25, + "top3HitRate": 0.25, + "top5HitRate": 0.25, "intervals95": { "top1": [ - 0.047, - 0.448 + 0.089, + 0.532 ], "top3": [ - 0.047, - 0.448 + 0.089, + 0.532 ], "top5": [ - 0.047, - 0.448 + 0.089, + 0.532 ] } }, @@ -63,17 +73,327 @@ }, "mentioned": { "cases": 3, - "top1HitRate": 0.667, - "top3HitRate": 0.667, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + } + } + }, + "path-extraction:source": { + "all": { + "cases": 12, + "top1HitRate": 0.25, + "top3HitRate": 0.25, + "top5HitRate": 0.25, + "intervals95": { + "top1": [ + 0.089, + 0.532 + ], + "top3": [ + 0.089, + 0.532 + ], + "top5": [ + 0.089, + 0.532 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0, + "top3HitRate": 0, + "top5HitRate": 0, + "intervals95": { + "top1": [ + 0, + 0.299 + ], + "top3": [ + 0, + 0.299 + ], + "top5": [ + 0, + 0.299 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + } + } + }, + "path-extraction:code": { + "all": { + "cases": 12, + "top1HitRate": 0.25, + "top3HitRate": 0.25, + "top5HitRate": 0.25, + "intervals95": { + "top1": [ + 0.089, + 0.532 + ], + "top3": [ + 0.089, + 0.532 + ], + "top5": [ + 0.089, + 0.532 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0, + "top3HitRate": 0, + "top5HitRate": 0, + "intervals95": { + "top1": [ + 0, + 0.299 + ], + "top3": [ + 0, + 0.299 + ], + "top5": [ + 0, + 0.299 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 1, + "top3HitRate": 1, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.438, + 1 + ], + "top3": [ + 0.438, + 1 + ], + "top5": [ + 0.438, + 1 + ] + } + } + }, + "lexical-literal:raw": { + "all": { + "cases": 12, + "top1HitRate": 0.083, + "top3HitRate": 0.25, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.015, + 0.354 + ], + "top3": [ + 0.089, + 0.532 + ], + "top5": [ + 0.138, + 0.609 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.111, + "top3HitRate": 0.222, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.02, + 0.435 + ], + "top3": [ + 0.063, + 0.547 + ], + "top5": [ + 0.121, + 0.646 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0, + 0.562 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "lexical-literal:source": { + "all": { + "cases": 12, + "top1HitRate": 0.083, + "top3HitRate": 0.25, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.015, + 0.354 + ], + "top3": [ + 0.089, + 0.532 + ], + "top5": [ + 0.138, + 0.609 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.111, + "top3HitRate": 0.222, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0.02, + 0.435 + ], + "top3": [ + 0.063, + 0.547 + ], + "top5": [ + 0.121, + 0.646 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0, + "top3HitRate": 0.333, + "top5HitRate": 0.333, + "intervals95": { + "top1": [ + 0, + 0.562 + ], + "top3": [ + 0.061, + 0.792 + ], + "top5": [ + 0.061, + 0.792 + ] + } + } + }, + "lexical-literal:code": { + "all": { + "cases": 12, + "top1HitRate": 0.25, + "top3HitRate": 0.417, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.089, + 0.532 + ], + "top3": [ + 0.193, + 0.68 + ], + "top5": [ + 0.391, + 0.862 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.222, + "top3HitRate": 0.444, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.063, + 0.547 + ], + "top3": [ + 0.189, + 0.733 + ], + "top5": [ + 0.354, + 0.879 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.333, + "top3HitRate": 0.333, "top5HitRate": 0.667, "intervals95": { "top1": [ - 0.208, - 0.939 + 0.061, + 0.792 ], "top3": [ - 0.208, - 0.939 + 0.061, + 0.792 ], "top5": [ 0.208, @@ -82,16 +402,16 @@ } } }, - "lexical-literal": { + "bm25:raw": { "all": { "cases": 12, - "top1HitRate": 0.083, + "top1HitRate": 0.167, "top3HitRate": 0.25, "top5HitRate": 0.333, "intervals95": { "top1": [ - 0.015, - 0.354 + 0.047, + 0.448 ], "top3": [ 0.089, @@ -125,13 +445,13 @@ }, "mentioned": { "cases": 3, - "top1HitRate": 0, + "top1HitRate": 0.333, "top3HitRate": 0.333, "top5HitRate": 0.333, "intervals95": { "top1": [ - 0, - 0.562 + 0.061, + 0.792 ], "top3": [ 0.061, @@ -144,12 +464,12 @@ } } }, - "bm25": { + "bm25:source": { "all": { "cases": 12, "top1HitRate": 0.167, "top3HitRate": 0.25, - "top5HitRate": 0.333, + "top5HitRate": 0.5, "intervals95": { "top1": [ 0.047, @@ -160,8 +480,8 @@ 0.532 ], "top5": [ - 0.138, - 0.609 + 0.254, + 0.746 ] } }, @@ -169,7 +489,7 @@ "cases": 9, "top1HitRate": 0.111, "top3HitRate": 0.222, - "top5HitRate": 0.333, + "top5HitRate": 0.444, "intervals95": { "top1": [ 0.02, @@ -180,8 +500,8 @@ 0.547 ], "top5": [ - 0.121, - 0.646 + 0.189, + 0.733 ] } }, @@ -189,7 +509,7 @@ "cases": 3, "top1HitRate": 0.333, "top3HitRate": 0.333, - "top5HitRate": 0.333, + "top5HitRate": 0.667, "intervals95": { "top1": [ 0.061, @@ -200,8 +520,70 @@ 0.792 ], "top5": [ - 0.061, - 0.792 + 0.208, + 0.939 + ] + } + } + }, + "bm25:code": { + "all": { + "cases": 12, + "top1HitRate": 0.5, + "top3HitRate": 0.583, + "top5HitRate": 0.917, + "intervals95": { + "top1": [ + 0.254, + 0.746 + ], + "top3": [ + 0.32, + 0.807 + ], + "top5": [ + 0.646, + 0.985 + ] + } + }, + "unmentioned": { + "cases": 9, + "top1HitRate": 0.444, + "top3HitRate": 0.556, + "top5HitRate": 1, + "intervals95": { + "top1": [ + 0.189, + 0.733 + ], + "top3": [ + 0.267, + 0.811 + ], + "top5": [ + 0.701, + 1 + ] + } + }, + "mentioned": { + "cases": 3, + "top1HitRate": 0.667, + "top3HitRate": 0.667, + "top5HitRate": 0.667, + "intervals95": { + "top1": [ + 0.208, + 0.939 + ], + "top3": [ + 0.208, + 0.939 + ], + "top5": [ + 0.208, + 0.939 ] } } @@ -271,69 +653,69 @@ }, "pairedVsFixmapMcnemarExact": { "all": { - "path-extraction": { + "path-extraction:raw": { "top1": { + "aWins": 4, + "bWins": 0, + "discordant": 4, + "pValue": 0.125 + }, + "top3": { "aWins": 5, "bWins": 0, "discordant": 5, "pValue": 0.0625 }, - "top3": { + "top5": { "aWins": 6, "bWins": 0, "discordant": 6, "pValue": 0.0313 - }, - "top5": { - "aWins": 7, - "bWins": 0, - "discordant": 7, - "pValue": 0.0156 } }, - "lexical-literal": { + "lexical-literal:code": { "top1": { - "aWins": 6, - "bWins": 0, + "aWins": 5, + "bWins": 1, "discordant": 6, - "pValue": 0.0313 + "pValue": 0.2188 }, "top3": { - "aWins": 5, - "bWins": 0, + "aWins": 4, + "bWins": 1, "discordant": 5, - "pValue": 0.0625 + "pValue": 0.375 }, "top5": { - "aWins": 5, - "bWins": 0, - "discordant": 5, - "pValue": 0.0625 + "aWins": 2, + "bWins": 1, + "discordant": 3, + "pValue": 1 } }, - "bm25": { + "bm25:code": { "top1": { - "aWins": 5, - "bWins": 0, + "aWins": 3, + "bWins": 2, "discordant": 5, - "pValue": 0.0625 + "pValue": 1 }, "top3": { - "aWins": 5, - "bWins": 0, - "discordant": 5, - "pValue": 0.0625 + "aWins": 4, + "bWins": 3, + "discordant": 7, + "pValue": 1 }, "top5": { - "aWins": 6, - "bWins": 1, - "discordant": 7, - "pValue": 0.125 + "aWins": 1, + "bWins": 3, + "discordant": 4, + "pValue": 0.625 } } }, "unmentioned": { - "path-extraction": { + "path-extraction:raw": { "top1": { "aWins": 4, "bWins": 0, @@ -353,44 +735,44 @@ "pValue": 0.0313 } }, - "lexical-literal": { + "lexical-literal:code": { "top1": { "aWins": 3, - "bWins": 0, - "discordant": 3, - "pValue": 0.25 + "bWins": 1, + "discordant": 4, + "pValue": 0.625 }, "top3": { - "aWins": 3, - "bWins": 0, + "aWins": 2, + "bWins": 1, "discordant": 3, - "pValue": 0.25 + "pValue": 1 }, "top5": { - "aWins": 3, - "bWins": 0, - "discordant": 3, - "pValue": 0.25 + "aWins": 1, + "bWins": 1, + "discordant": 2, + "pValue": 1 } }, - "bm25": { + "bm25:code": { "top1": { - "aWins": 3, - "bWins": 0, - "discordant": 3, - "pValue": 0.25 + "aWins": 2, + "bWins": 2, + "discordant": 4, + "pValue": 1 }, "top3": { "aWins": 3, - "bWins": 0, - "discordant": 3, - "pValue": 0.25 + "bWins": 3, + "discordant": 6, + "pValue": 1 }, "top5": { - "aWins": 4, - "bWins": 1, - "discordant": 5, - "pValue": 0.375 + "aWins": 0, + "bWins": 3, + "discordant": 3, + "pValue": 0.25 } } } @@ -402,11 +784,32 @@ "lib/document.js" ], "scannedFiles": 702, + "candidateCounts": { + "raw": 702, + "source": 443, + "code": 317 + }, "queryTermCount": 36, "mentionsExpectedPath": true, "mentionTier": "full-path", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [ + "lib/document.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:source": { + "top5": [ + "lib/document.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:code": { "top5": [ "lib/document.js" ], @@ -414,7 +817,19 @@ "top3": true, "top5Hit": true }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "lib/connection.js", + "docs/schematypes.md", + "lib/schemaType.js", + "types/document.d.ts", + "lib/types/subdocument.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "lib/connection.js", "docs/schematypes.md", @@ -426,7 +841,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "lib/connection.js", + "lib/schemaType.js", + "types/document.d.ts", + "lib/types/subdocument.js", + "lib/mongoose.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ "lib/types/subdocument.js", "types/document.d.ts", @@ -438,6 +865,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "lib/types/subdocument.js", + "types/document.d.ts", + "docs/documents.md", + "lib/helpers/symbols.js", + "docs/middleware.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "lib/types/subdocument.js", + "types/document.d.ts", + "lib/helpers/symbols.js", + "lib/helpers/model/discriminator.js", + "lib/schemaType.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, "fixmap": { "top5": [ "lib/document.js", @@ -458,35 +909,100 @@ "src/types/types-external.ts" ], "scannedFiles": 165, + "candidateCounts": { + "raw": 165, + "source": 83, + "code": 28 + }, "queryTermCount": 24, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "src/types/types-external.ts", + "src/immer.ts", + "src/plugins/arrayMethods.ts", + "src/core/immerClass.ts", + "website/docs/pitfalls.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:source": { + "top5": [ + "src/types/types-external.ts", + "src/immer.ts", + "src/plugins/arrayMethods.ts", + "src/core/immerClass.ts", + "website/docs/pitfalls.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:code": { + "top5": [ + "src/types/types-external.ts", + "src/immer.ts", + "src/plugins/arrayMethods.ts", + "src/core/immerClass.ts", + "src/core/proxy.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "src/types/types-external.ts", + ".github/ISSUE_TEMPLATE/bug.md", "src/immer.ts", - "src/plugins/arrayMethods.ts", "src/core/immerClass.ts", - "website/docs/pitfalls.md" + "__tests__/produce.ts" ], "top1": true, "top3": true, "top5Hit": true }, - "bm25": { + "bm25:source": { "top5": [ "src/types/types-external.ts", ".github/ISSUE_TEMPLATE/bug.md", "src/immer.ts", "src/core/immerClass.ts", - "__tests__/produce.ts" + "website/docs/pitfalls.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "src/types/types-external.ts", + "src/immer.ts", + "src/core/immerClass.ts", + "src/utils/errors.ts", + "src/utils/common.ts" ], "top1": true, "top3": true, @@ -512,17 +1028,46 @@ "packages/jest-mock/src/index.ts" ], "scannedFiles": 3249, + "candidateCounts": { + "raw": 3249, + "source": 1622, + "code": 826 + }, "queryTermCount": 50, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "path-extraction:code": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:raw": { + "top5": [ + "docs/MockFunctionAPI.md", + "website/versioned_docs/version-30.4/MockFunctionAPI.md", + "website/versioned_docs/version-29.7/MockFunctionAPI.md", + "website/versioned_docs/version-30.0/MockFunctionAPI.md", + "packages/jest-mock/src/index.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "lexical-literal:source": { "top5": [ "docs/MockFunctionAPI.md", "website/versioned_docs/version-30.4/MockFunctionAPI.md", @@ -534,7 +1079,19 @@ "top3": false, "top5Hit": true }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "packages/jest-mock/src/index.ts", + "packages/jest-runtime/src/internals/EsmLoader.ts", + "packages/jest-environment/src/index.ts", + "packages/jest-config/src/normalize.ts", + "packages/jest-resolve/src/resolver.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "website/versioned_docs/version-29.7/MockFunctionAPI.md", "website/versioned_docs/version-30.0/MockFunctionAPI.md", @@ -546,6 +1103,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "website/versioned_docs/version-29.7/MockFunctionAPI.md", + "website/versioned_docs/version-30.0/MockFunctionAPI.md", + "website/versioned_docs/version-30.4/MockFunctionAPI.md", + "docs/MockFunctionAPI.md", + "packages/expect/src/types.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "packages/jest-mock/src/index.ts", + "packages/expect/src/types.ts", + "packages/jest-environment/src/index.ts", + "packages/jest-haste-map/src/watchers/index.ts", + "packages/jest-runtime/src/internals/EsmLoader.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "packages/jest-config/src/readConfigFileAndSetRootDir.ts", @@ -566,17 +1147,34 @@ "lib/dialects/postgres/query/pg-querycompiler.js" ], "scannedFiles": 600, + "candidateCounts": { + "raw": 600, + "source": 260, + "code": 188 + }, "queryTermCount": 31, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "path-extraction:code": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:raw": { "top5": [ "docs/src/guide/schema-builder.md", "docs/src/guide/index.md", @@ -588,7 +1186,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "docs/src/guide/schema-builder.md", + "docs/src/guide/index.md", + "docs/src/guide/migrations.md", + "UPGRADING.md", + "lib/query/querycompiler.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:code": { + "top5": [ + "lib/query/querycompiler.js", + "lib/query/querybuilder.js", + "lib/dialects/mssql/query/mssql-querycompiler.js", + "lib/dialects/postgres/query/pg-querycompiler.js", + "lib/schema/tablebuilder.js" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "docs/src/guide/schema-builder.md", "docs/src/guide/index.md", @@ -600,6 +1222,30 @@ "top3": true, "top5Hit": true }, + "bm25:source": { + "top5": [ + "docs/src/guide/schema-builder.md", + "docs/src/guide/index.md", + "lib/dialects/postgres/query/pg-querycompiler.js", + "docs/src/faq/recipes.md", + "UPGRADING.md" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "lib/dialects/postgres/query/pg-querycompiler.js", + "lib/dialects/mssql/query/mssql-querycompiler.js", + "lib/schema/tablecompiler.js", + "lib/query/querycompiler.js", + "lib/dialects/postgres/index.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/dialects/postgres/query/pg-querycompiler.js", @@ -620,17 +1266,50 @@ "lib/reporters/xunit.js" ], "scannedFiles": 661, + "candidateCounts": { + "raw": 661, + "source": 129, + "code": 69 + }, "queryTermCount": 46, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [ + ".github/CODE_OF_CONDUCT.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [ + ".github/CODE_OF_CONDUCT.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + ".github/CONTRIBUTING.md", + "MAINTAINERS.md", + ".github/ISSUE_TEMPLATE/01-bug.yml", + "README.md", + "PROJECT_CHARTER.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ ".github/CONTRIBUTING.md", "MAINTAINERS.md", @@ -642,7 +1321,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "lib/utils.cjs", + "lib/mocha.cjs", + "lib/cli/watch-run.cjs", + "lib/cli/lookup-files.js", + "lib/runner.cjs" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { "top5": [ ".github/ISSUE_TEMPLATE/01-bug.yml", "README.md", @@ -654,6 +1345,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + ".github/ISSUE_TEMPLATE/01-bug.yml", + ".github/CONTRIBUTING.md", + "README.md", + "PROJECT_CHARTER.md", + "MAINTAINERS.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "scripts/update-authors.cjs", + "lib/cli/cli.js", + "lib/nodejs/esm-utils.cjs", + "lib/reporters/xunit.js", + "lib/utils.cjs" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/errors.js", @@ -674,17 +1389,46 @@ "src/logic/createFormControl.ts" ], "scannedFiles": 490, + "candidateCounts": { + "raw": 490, + "source": 320, + "code": 257 + }, "queryTermCount": 52, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "CHANGELOG.md", + "src/types/form.ts", + "src/logic/createFormControl.ts", + "reports/api-extractor.md.api.md", + "reports/api-extractor.md" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "lexical-literal:source": { "top5": [ "CHANGELOG.md", "src/types/form.ts", @@ -696,7 +1440,19 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "src/types/form.ts", + "src/logic/createFormControl.ts", + "scripts/bench.ts", + "src/useFieldArray.ts", + "src/useForm.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "CHANGELOG.md", "scripts/bench.ts", @@ -708,6 +1464,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "CHANGELOG.md", + "scripts/bench.ts", + "src/types/form.ts", + "examples/README.md", + "src/logic/createFormControl.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "scripts/bench.ts", + "src/types/form.ts", + "src/useForm.ts", + "src/logic/createFormControl.ts", + "src/useFormState.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, "fixmap": { "top5": [ "src/logic/createFormControl.ts", @@ -728,17 +1508,46 @@ "packages/engine.io-client/lib/socket.ts" ], "scannedFiles": 829, + "candidateCounts": { + "raw": 829, + "source": 520, + "code": 260 + }, "queryTermCount": 27, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "packages/engine.io/lib/server.ts", + "packages/socket.io-client/dist/socket.io.esm.min.js", + "packages/socket.io/client-dist/socket.io.esm.min.js", + "packages/engine.io-client/dist/engine.io.esm.min.js", + "packages/socket.io-client/CHANGELOG.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "packages/engine.io/lib/server.ts", "packages/socket.io-client/dist/socket.io.esm.min.js", @@ -750,16 +1559,52 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "packages/engine.io/lib/server.ts", + "packages/socket.io-client/dist/socket.io.esm.min.js", + "packages/socket.io/client-dist/socket.io.esm.min.js", + "packages/engine.io-client/dist/engine.io.esm.min.js", + "packages/engine.io-client/lib/socket.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:raw": { + "top5": [ + "packages/engine.io-client/README.md", + "packages/engine.io/README.md", + "packages/engine.io/lib/server.ts", + "packages/engine.io-client/lib/transports/polling-xhr.ts", + "packages/engine.io-client/lib/socket.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:source": { "top5": [ + "packages/engine.io/lib/server.ts", "packages/engine.io-client/README.md", "packages/engine.io/README.md", + "packages/engine.io-client/lib/socket.ts", + "packages/engine.io-client/dist/engine.io.esm.min.js" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:code": { + "top5": [ "packages/engine.io/lib/server.ts", + "packages/engine.io-client/lib/socket.ts", + "packages/engine.io-client/dist/engine.io.esm.min.js", "packages/engine.io-client/lib/transports/polling-xhr.ts", - "packages/engine.io-client/lib/socket.ts" + "packages/socket.io-client/dist/socket.io.esm.min.js" ], "top1": false, - "top3": false, + "top3": true, "top5Hit": true }, "fixmap": { @@ -782,17 +1627,52 @@ "packages/svelte/src/internal/client/dom/blocks/boundary.js" ], "scannedFiles": 8974, + "candidateCounts": { + "raw": 8974, + "source": 610, + "code": 444 + }, "queryTermCount": 43, "mentionsExpectedPath": true, "mentionTier": "full-path", "arms": { - "path-extraction": { - "top5": [], + "path-extraction:raw": { + "top5": [ + "packages/svelte/src/internal/client/dom/blocks/boundary.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:source": { + "top5": [ + "packages/svelte/src/internal/client/dom/blocks/boundary.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:code": { + "top5": [ + "packages/svelte/src/internal/client/dom/blocks/boundary.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "lexical-literal:raw": { + "top5": [ + "documentation/docs/07-misc/07-v5-migration-guide.md", + "packages/svelte/src/internal/server/renderer.js", + "packages/svelte/messages/client-warnings/warnings.md", + "documentation/docs/98-reference/.generated/client-warnings.md", + "packages/svelte/src/internal/client/reactivity/batch.js" + ], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:source": { "top5": [ "documentation/docs/07-misc/07-v5-migration-guide.md", "packages/svelte/src/internal/server/renderer.js", @@ -804,7 +1684,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "packages/svelte/src/internal/server/renderer.js", + "packages/svelte/src/internal/client/reactivity/batch.js", + "packages/svelte/src/internal/client/errors.js", + "packages/svelte/src/internal/client/dom/blocks/boundary.js", + "packages/svelte/src/internal/client/reactivity/deriveds.js" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "documentation/docs/05-special-elements/01-svelte-boundary.md", "packages/svelte/src/internal/shared/errors.js", @@ -816,6 +1708,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + "packages/svelte/messages/client-errors/errors.md", + "documentation/docs/98-reference/.generated/client-errors.md", + "packages/svelte/messages/client-warnings/warnings.md", + "documentation/docs/98-reference/.generated/client-warnings.md", + "packages/svelte/src/internal/client/dom/blocks/boundary.js" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "packages/svelte/src/internal/client/dom/blocks/boundary.js", + "packages/svelte/src/internal/client/errors.js", + "packages/svelte/src/internal/server/renderer.js", + "packages/svelte/src/internal/client/warnings.js", + "packages/svelte/src/internal/shared/errors.js" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "packages/svelte/src/internal/client/dom/blocks/boundary.js", @@ -836,17 +1752,46 @@ "packages/vite/src/node/server/bundledDev.ts" ], "scannedFiles": 2758, + "candidateCounts": { + "raw": 2758, + "source": 1806, + "code": 1241 + }, "queryTermCount": 48, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:code": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "lexical-literal:raw": { + "top5": [ + "docs/config/shared-options.md", + "CONTRIBUTING.md", + "packages/vite/src/node/utils.ts", + "docs/guide/features.md", + "packages/vite/src/node/build.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "docs/config/shared-options.md", "CONTRIBUTING.md", @@ -858,7 +1803,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "packages/vite/src/node/utils.ts", + "packages/vite/src/node/build.ts", + "packages/vite/src/node/optimizer/index.ts", + "packages/vite/src/node/server/index.ts", + "packages/create-vite/src/index.ts" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { + "top5": [ + "CONTRIBUTING.md", + "docs/guide/static-deploy.md", + "docs/guide/index.md", + ".github/ISSUE_TEMPLATE/bug_report.yml", + "packages/create-vite/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:source": { "top5": [ "CONTRIBUTING.md", "docs/guide/static-deploy.md", @@ -870,6 +1839,18 @@ "top3": false, "top5Hit": false }, + "bm25:code": { + "top5": [ + "packages/create-vite/src/index.ts", + "packages/vite/src/node/utils.ts", + "packages/vite/src/node/build.ts", + "packages/vite/src/node/server/bundledDev.ts", + "packages/vite/src/node/server/transformRequest.ts" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, "fixmap": { "top5": [ "packages/vite/src/types/commonjs.d.ts", @@ -890,17 +1871,46 @@ "packages/runtime-dom/src/index.ts" ], "scannedFiles": 699, + "candidateCounts": { + "raw": 699, + "source": 408, + "code": 324 + }, "queryTermCount": 31, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "path-extraction:code": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:raw": { + "top5": [ + ".github/contributing.md", + "packages/compiler-sfc/src/script/resolveType.ts", + "packages/runtime-core/src/component.ts", + "packages/compiler-sfc/src/compileScript.ts", + "changelogs/CHANGELOG-3.1.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ ".github/contributing.md", "packages/compiler-sfc/src/script/resolveType.ts", @@ -912,7 +1922,31 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "packages/compiler-sfc/src/script/resolveType.ts", + "packages/runtime-core/src/component.ts", + "packages/compiler-sfc/src/compileScript.ts", + "packages/runtime-dom/src/jsx.ts", + "packages-private/dts-test/defineComponent.test-d.tsx" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:raw": { + "top5": [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/contributing.md", + "packages/compiler-sfc/README.md", + "changelogs/CHANGELOG-3.1.md", + "packages/vue-compat/README.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:source": { "top5": [ ".github/ISSUE_TEMPLATE/bug_report.yml", ".github/contributing.md", @@ -924,6 +1958,18 @@ "top3": false, "top5Hit": false }, + "bm25:code": { + "top5": [ + "packages/runtime-dom/src/index.ts", + "scripts/release.js", + "packages/runtime-core/src/apiSetupHelpers.ts", + "packages/runtime-dom/src/jsx.ts", + "packages/reactivity/src/reactive.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "tsconfig.json", @@ -944,17 +1990,46 @@ "lib/winston/transports/file.js" ], "scannedFiles": 115, + "candidateCounts": { + "raw": 115, + "source": 65, + "code": 46 + }, "queryTermCount": 38, "mentionsExpectedPath": false, "mentionTier": "none", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "path-extraction:source": { "top5": [], "top1": false, "top3": false, "top5Hit": false }, - "lexical-literal": { + "path-extraction:code": { + "top5": [], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:raw": { + "top5": [ + "docs/transports.md", + "README.md", + "CHANGELOG.md", + "UPGRADE-3.0.md", + "lib/winston/logger.js" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "lexical-literal:source": { "top5": [ "docs/transports.md", "README.md", @@ -966,7 +2041,19 @@ "top3": false, "top5Hit": false }, - "bm25": { + "lexical-literal:code": { + "top5": [ + "lib/winston/logger.js", + "lib/winston/transports/file.js", + "lib/winston.js", + "index.d.ts", + "lib/winston/exception-handler.js" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ ".github/ISSUE_TEMPLATE/bug_report.yml", "lib/winston/common.js", @@ -978,6 +2065,30 @@ "top3": false, "top5Hit": false }, + "bm25:source": { + "top5": [ + ".github/ISSUE_TEMPLATE/bug_report.yml", + "lib/winston/common.js", + "docs/transports.md", + "README.md", + "UPGRADE-3.0.md" + ], + "top1": false, + "top3": false, + "top5Hit": false + }, + "bm25:code": { + "top5": [ + "lib/winston/common.js", + "lib/winston.js", + "lib/winston/logger.js", + "lib/winston/transports/file.js", + "lib/winston/exception-handler.js" + ], + "top1": false, + "top3": false, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/winston/transports/file.js", @@ -998,11 +2109,32 @@ "lib/utils/apply-extends.ts" ], "scannedFiles": 170, + "candidateCounts": { + "raw": 170, + "source": 103, + "code": 57 + }, "queryTermCount": 39, "mentionsExpectedPath": true, "mentionTier": "full-path", "arms": { - "path-extraction": { + "path-extraction:raw": { + "top5": [ + "lib/utils/apply-extends.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:source": { + "top5": [ + "lib/utils/apply-extends.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "path-extraction:code": { "top5": [ "lib/utils/apply-extends.ts" ], @@ -1010,7 +2142,7 @@ "top3": true, "top5Hit": true }, - "lexical-literal": { + "lexical-literal:raw": { "top5": [ "docs/api.md", "docs/advanced.md", @@ -1022,7 +2154,31 @@ "top3": true, "top5Hit": true }, - "bm25": { + "lexical-literal:source": { + "top5": [ + "docs/api.md", + "docs/advanced.md", + "lib/utils/apply-extends.ts", + "lib/command.ts", + "lib/typings/yargs-parser-types.ts" + ], + "top1": false, + "top3": true, + "top5Hit": true + }, + "lexical-literal:code": { + "top5": [ + "lib/utils/apply-extends.ts", + "lib/command.ts", + "lib/typings/yargs-parser-types.ts", + "lib/validation.ts", + "lib/completion.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:raw": { "top5": [ "lib/utils/apply-extends.ts", "lib/typings/yargs-parser-types.ts", @@ -1034,6 +2190,30 @@ "top3": true, "top5Hit": true }, + "bm25:source": { + "top5": [ + "lib/utils/apply-extends.ts", + "lib/typings/yargs-parser-types.ts", + "docs/api.md", + "lib/utils/levenshtein.ts", + "docs/advanced.md" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, + "bm25:code": { + "top5": [ + "lib/utils/apply-extends.ts", + "lib/typings/yargs-parser-types.ts", + "lib/utils/levenshtein.ts", + "helpers/helpers.mjs", + "lib/command.ts" + ], + "top1": true, + "top3": true, + "top5Hit": true + }, "fixmap": { "top5": [ "lib/utils/apply-extends.ts", diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 78e15ee..abe0dc3 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -53,85 +53,94 @@ contain their own answer. A hit rate published on its own does not answer the question a reader actually has: is this better than what an agent already gets by searching the repository itself? -[`scripts/evaluate-baseline.mjs`](../scripts/evaluate-baseline.mjs) scores three naive arms and -FixMap on **one `scanRepo()` result per case, shared by every arm** — the same file list, the same -text samples, the same truncation — so a difference in score is a difference in ranking. +[`scripts/evaluate-baseline.mjs`](../scripts/evaluate-baseline.mjs) scores naive arms and FixMap on +**one `scanRepo()` result per case, shared by every arm**. + +### Candidate policy is the confound, not the ranking function + +FixMap does not rank the raw scan. `rankContextFiles` gates on `isSource && !isTest` (minus +lockfiles, excluded and generated paths) and then deprioritises documentation for a task that is not +about documentation. A baseline pointed at every scanned file competes on a larger, doc-heavy +population and returns `README.md`, `CONTRIBUTING.md` and issue templates first — it loses to +documentation, not to FixMap. + +An earlier revision of this page did exactly that and reported FixMap beating BM25 at p = 0.004. +That number was an artifact of the handicap and has been withdrawn. + +Each baseline is now run under three candidate policies and reported **at its strongest**: + +| Policy | Candidate set | +| --- | --- | +| `raw` | Every scanned file. Kept only for reference; ranks READMEs first. | +| `source` | `isSource && !isTest` — FixMap's own gate. | +| `code` | `isSource && !isTest && kind === "code"` — also drops documentation, as FixMap's scoring effectively does for implementation tasks. | | Arm | What it does | | --- | --- | -| `path-extraction` | Pulls path-shaped tokens out of the task text and keeps those resolving to a real file. Ranks nothing; it prices what the task text was carrying. | -| `lexical-literal` | Literal keyword search: distinct query terms matched, then raw occurrence count. No corpus statistics. | -| `bm25` | Standard BM25 (k1 = 1.2, b = 0.75) over the same text. A retrieval baseline, **not** a grep. | -| `fixmap` | `rankContextFiles` from `@aryam/fixmap-core`. | +| `path-extraction` | Path-shaped tokens pulled from the task text, resolved against the corpus. Prices what the task was carrying. | +| `lexical-literal` | Literal keyword search: distinct query terms matched, then raw occurrence count. | +| `bm25` | Standard BM25 (k1 = 1.2, b = 0.75). A retrieval baseline, **not** a grep. | +| `fixmap` | `rankContextFiles`, which applies its own gate internally. | -Both keyword arms are case-insensitive and expand camelCase, which favours the baselines. That is -deliberate — a handicapped baseline proves nothing. +Both keyword arms are case-insensitive and expand camelCase, which favours the baselines. -Held-out suite, cases whose task did not name the file (9): +### Held-out, tasks that did not name the file (9) | Arm | Top-1 | Top-3 | Top-5 | | --- | ---: | ---: | ---: | -| `path-extraction` | 0.0% | 0.0% | 0.0% | -| `lexical-literal` | 11.1% | 22.2% | 33.3% | -| `bm25` | 11.1% | 22.2% | 33.3% | -| `fixmap` | **44.4%** | **55.6%** | **66.7%** | +| `path-extraction` (any policy) | 0.0% | 0.0% | 0.0% | +| `lexical-literal:raw` | 11.1% | 22.2% | 33.3% | +| `lexical-literal:code` | 22.2% | 44.4% | 66.7% | +| `bm25:raw` | 11.1% | 22.2% | 33.3% | +| `bm25:source` | 11.1% | 22.2% | 44.4% | +| **`bm25:code`** | **44.4%** | **55.6%** | **100%** | +| `fixmap` | 44.4% | 55.6% | 66.7% | + +**FixMap does not beat BM25-over-code on repositories it was never tuned against.** Top-1 and Top-3 +are exact ties (McNemar p = 1.0, two disagreements each way). At Top-5 the baseline wins three cases +FixMap misses and FixMap wins none — BM25 has the fixing file in its top five for 9 of 9 cases, +FixMap for 6 of 9. -Regression suite, same cohort (13): +The three FixMap misses BM25 catches are `socketio/socket.io`, `vitejs/vite` and `vuejs/core`. + +### Regression, tasks that did not name the file (13) | Arm | Top-1 | Top-3 | Top-5 | | --- | ---: | ---: | ---: | -| `path-extraction` | 0.0% | 0.0% | 0.0% | -| `lexical-literal` | 7.7% | 30.8% | 30.8% | -| `bm25` | 15.4% | 30.8% | 38.5% | -| `fixmap` | **69.2%** | **100%** | **100%** | - -`path-extraction` scoring exactly 0.0% on the unmentioned cohort of both suites, and 66.7% on the -named cohort, is the independent check that the cohort classifier measures what it claims. - -Arms are compared with **McNemar's exact test** rather than by comparing two Wilson intervals: the -arms ran on the same cases, and that pairing carries information independent intervals discard. - -| Suite | Cohort | FixMap vs | Top-1 p | Top-3 p | Top-5 p | -| --- | --- | --- | ---: | ---: | ---: | -| Regression | unmentioned (13) | `lexical-literal` | 0.0078 | 0.0039 | 0.0039 | -| Regression | unmentioned (13) | `bm25` | 0.0156 | 0.0039 | 0.0078 | -| Held-out | unmentioned (9) | `lexical-literal` | 0.25 | 0.25 | 0.25 | -| Held-out | unmentioned (9) | `bm25` | 0.25 | 0.25 | 0.375 | - -FixMap never loses a disagreeing case to any baseline on the regression suite, and loses exactly one -across the entire held-out comparison. The held-out p-values are **not** evidence of no effect: with -three disagreeing cases the smallest attainable two-sided p-value is 0.25, so that cohort is -arithmetically incapable of reaching 0.05. It is underpowered, which is the strongest available -argument for growing it. +| `lexical-literal:code` | 30.8% | 30.8% | 30.8% | +| `bm25:source` | 23.1% | 38.5% | 38.5% | +| `bm25:code` | 38.5% | 61.5% | 61.5% | +| **`fixmap`** | **69.2%** | **100%** | **100%** | -```bash -node scripts/evaluate-baseline.mjs --suite heldout -node scripts/evaluate-baseline.mjs --suite external -node scripts/evaluate-baseline.mjs --suite heldout --record -``` +FixMap leads here, but this is the suite whose cases shaped the ranker, and the lead is **not +significant** against `bm25:code`: p = 0.125 at Top-1 and p = 0.0625 at Top-3 and Top-5. -Per-arm rankings for every case are recorded in `benchmarks//baseline-results.json`. +### Paired tests -The context comparison is intentionally a proxy: +Arms are compared with **McNemar's exact test** rather than by comparing Wilson intervals: the arms +ran on the same cases, and that pairing carries information independent intervals discard. -- **Assumed baseline:** send every scanned text-bearing file in FixMap's supported extension set—22,058,578 estimated tokens. -- **FixMap comparison:** send only the Top-5 ranked files—318,546 estimated tokens. -- **Estimator:** UTF-8 file bytes ÷ 4. This is not tokenizer output and does not include prompts, tool protocol, or generated responses. +| Suite | FixMap vs | Top-1 | Top-3 | Top-5 | +| --- | --- | ---: | ---: | ---: | +| Regression (13) | `lexical-literal:code` | 0.0625 | 0.0039 | 0.0039 | +| Regression (13) | `bm25:code` | 0.125 | 0.0625 | 0.0625 | +| Held-out (9) | `lexical-literal:code` | 0.625 | 1.0 | 1.0 | +| Held-out (9) | `bm25:code` | 1.0 | 1.0 | 0.25 (baseline ahead) | -The supported set is `.cjs`, `.css`, `.go`, `.js`, `.json`, `.jsx`, `.md`, `.mjs`, `.py`, `.rs`, `.ts`, `.tsx`, `.yaml`, and `.yml`, so the assumed baseline includes tests, documentation, and configuration—not only implementation code. +Nine and thirteen cases cannot settle this either way; what they do show is that the previously +published margin does not survive a fair baseline. Growing the held-out suite and closing the Top-5 +recall gap are the work this points at. -The visual also shows a 14.97-minute implied difference against an **assumed 15-minute manual-triage baseline**. That baseline was not measured in a controlled with/without-agent experiment, so it is not presented as a real-world time-savings claim. - -Run or deliberately refresh the evidence: +`path-extraction` scoring 0.0% on the unmentioned cohort of both suites and 100% on the named +cohort is the independent check that the cohort classifier measures what it claims. ```bash -npm run evaluate:external -npm run benchmark:savings -npm run benchmark:savings:record -npm run render:benchmark-card +node scripts/evaluate-baseline.mjs --suite heldout +node scripts/evaluate-baseline.mjs --suite external +node scripts/evaluate-baseline.mjs --suite heldout --record ``` -See [`benchmarks/external/README.md`](../benchmarks/external/README.md) for case selection, exact repositories, fresh baseline comparison, and every recorded ranking. +Per-arm, per-policy rankings for every case are recorded in `benchmarks//baseline-results.json`. ## Scanner performance diff --git a/scripts/evaluate-baseline.mjs b/scripts/evaluate-baseline.mjs index cbd1eb0..aa27580 100644 --- a/scripts/evaluate-baseline.mjs +++ b/scripts/evaluate-baseline.mjs @@ -94,6 +94,26 @@ function searchableText(file) { return `${file.path}\n${file.textSample ?? ""}`; } +// Candidate policy is the confound that matters most here, and getting it wrong produces a +// strawman. FixMap does not rank the raw scan: rankContextFiles gates on +// `isSource && !isTest` (minus lockfiles, excluded and generated paths) and then penalises +// documentation for a task that is not about documentation. A baseline pointed at every +// scanned file is therefore competing on a different, larger, doc-heavy population — and it +// loses to README.md, not to FixMap. +// +// So each keyword arm is scored under three policies, and the arm's headline number is its +// BEST policy. Comparing FixMap to the strongest baseline available is the only version of +// this comparison worth publishing. +const CANDIDATE_POLICIES = { + // Every scanned file. Kept for reference: this is the version that ranks READMEs first. + raw: () => true, + // FixMap's own candidate gate, using only public RepoFile fields. + source: (file) => file.isSource && !file.isTest, + // Additionally drops documentation, matching FixMap's deprioritisation of it for the + // implementation tasks that make up both suites. + code: (file) => file.isSource && !file.isTest && file.kind === "code" +}; + // ---------------------------------------------------------------------------- arms /** @@ -101,18 +121,27 @@ function searchableText(file) { * Ranked by order of appearance. This measures how much of a score the task text is * carrying on its own, before any ranking happens. */ -function rankByPathExtraction(repo, task) { +function rankByPathExtraction(files, task) { const text = String(task).replace(/\\/g, "/"); const candidates = text.match(/[A-Za-z0-9_.$-]+(?:\/[A-Za-z0-9_.$-]+)+\.[A-Za-z0-9]+/g) ?? []; - const byPath = new Map(repo.files.map((file) => [file.path, file])); + const byPath = new Set(files.map((file) => file.path)); const ranked = []; const seen = new Set(); for (const candidate of candidates) { - // Exact repo-relative path first, then a unique suffix match — a stack trace or tsc - // error often names a path relative to a package rather than to the repository root. - let match = byPath.has(candidate) ? candidate : null; - if (!match) { - const suffixMatches = repo.files.filter((file) => file.path.endsWith(`/${candidate}`)); + // A candidate lifted from a GitHub permalink still carries its URL prefix + // ("github.com/owner/repo/blob//packages/…"), so an exact match on the whole token + // never fires. Peel leading segments off and take the longest tail that is a real file. + // Without this the arm silently misses every permalink, which understates exactly the + // quantity it exists to measure. + const segments = candidate.split("/"); + let match = null; + for (let start = 0; start < segments.length && !match; start += 1) { + const tail = segments.slice(start).join("/"); + if (byPath.has(tail)) { + match = tail; + break; + } + const suffixMatches = files.filter((file) => file.path.endsWith(`/${tail}`)); if (suffixMatches.length === 1) { match = suffixMatches[0].path; } @@ -130,9 +159,9 @@ function rankByPathExtraction(repo, task) { * by total occurrences. This is the "I ran a few greps and looked at what matched most" * arm — no idf, no length normalisation, no weighting. */ -function rankByLexicalLiteral(repo, terms) { +function rankByLexicalLiteral(files, terms) { const scored = []; - for (const file of repo.files) { + for (const file of files) { const haystack = searchableText(file).toLowerCase(); let distinct = 0; let total = 0; @@ -157,8 +186,8 @@ function rankByLexicalLiteral(repo, terms) { } /** Standard BM25 over the same corpus. The harder of the two keyword baselines. */ -function rankByBm25(repo, terms, k1 = 1.2, b = 0.75) { - const documents = repo.files.map((file) => { +function rankByBm25(files, terms, k1 = 1.2, b = 0.75) { + const documents = files.map((file) => { const counts = new Map(); for (const token of tokenize(searchableText(file))) { counts.set(token, (counts.get(token) ?? 0) + 1); @@ -199,13 +228,23 @@ function rankByBm25(repo, terms, k1 = 1.2, b = 0.75) { // ---------------------------------------------------------------------------- run -const ARMS = ["path-extraction", "lexical-literal", "bm25", "fixmap"]; +// Each baseline is run under every candidate policy; FixMap applies its own internally. +const BASELINE_ARMS = ["path-extraction", "lexical-literal", "bm25"]; +const ARMS = []; +for (const arm of BASELINE_ARMS) { + for (const policy of Object.keys(CANDIDATE_POLICIES)) { + ARMS.push(`${arm}:${policy}`); + } +} +ARMS.push("fixmap"); + const perArmResults = Object.fromEntries(ARMS.map((arm) => [arm, []])); const perCase = []; for (const benchmark of dataset.cases) { const dir = await materializePinnedRepository(benchmark); - // One scan, shared by every arm, so the comparison isolates ranking. + // One scan, shared by every arm, so the comparison isolates ranking and candidate policy + // rather than what was read off disk. const repo = await scanRepo({ repoRoot: dir }); if (repo.files.length === 0) { throw new Error(`Baseline evaluation could not scan any files for ${benchmark.slug} at ${benchmark.sha}.`); @@ -214,16 +253,22 @@ for (const benchmark of dataset.cases) { const mention = classifyExpectedPathMention(benchmark); const ranked = { - "path-extraction": rankByPathExtraction(repo, benchmark.task), - "lexical-literal": rankByLexicalLiteral(repo, terms), - bm25: rankByBm25(repo, terms), fixmap: rankContextFiles(repo, { issueText: benchmark.task }, TOP_N).map((file) => file.path) }; + const policyCounts = {}; + for (const [policy, predicate] of Object.entries(CANDIDATE_POLICIES)) { + const files = repo.files.filter(predicate); + policyCounts[policy] = files.length; + ranked[`path-extraction:${policy}`] = rankByPathExtraction(files, benchmark.task); + ranked[`lexical-literal:${policy}`] = rankByLexicalLiteral(files, terms); + ranked[`bm25:${policy}`] = rankByBm25(files, terms); + } const caseRow = { slug: benchmark.slug, expected: benchmark.expected, scannedFiles: repo.files.length, + candidateCounts: policyCounts, queryTermCount: terms.length, mentionsExpectedPath: mention.mentionsExpectedPath, mentionTier: mention.mentionTier, @@ -298,20 +343,39 @@ for (const arm of ARMS) { }; } -// FixMap against each baseline, on the cohort the product claim rests on. +// The comparison worth publishing is against the STRONGEST form of each baseline, not the +// weakest. For every baseline family, pick the candidate policy that scored best on the +// unmentioned cohort and treat that as the arm to beat. +const bestPolicyPerFamily = {}; +for (const family of BASELINE_ARMS) { + let best = null; + for (const policy of Object.keys(CANDIDATE_POLICIES)) { + const score = arms[`${family}:${policy}`].unmentioned; + const key = [score.top1HitRate ?? 0, score.top3HitRate ?? 0, score.top5HitRate ?? 0]; + if (!best || key > best.key) { + best = { policy, key }; + } + } + bestPolicyPerFamily[family] = best.policy; +} + +// FixMap against each baseline family's best policy, on the cohort the claim rests on. const pairedVsFixmap = {}; for (const cohortName of ["all", "unmentioned"]) { const pick = (arm) => splitCohorts(perArmResults[arm])[cohortName]; const fixmapRows = pick("fixmap"); pairedVsFixmap[cohortName] = Object.fromEntries( - ARMS.filter((arm) => arm !== "fixmap").map((arm) => [ - arm, - { - top1: mcnemarExact(fixmapRows, pick(arm), "top1"), - top3: mcnemarExact(fixmapRows, pick(arm), "top3"), - top5: mcnemarExact(fixmapRows, pick(arm), "top5Hit") - } - ]) + BASELINE_ARMS.map((family) => { + const arm = `${family}:${bestPolicyPerFamily[family]}`; + return [ + arm, + { + top1: mcnemarExact(fixmapRows, pick(arm), "top1"), + top3: mcnemarExact(fixmapRows, pick(arm), "top3"), + top5: mcnemarExact(fixmapRows, pick(arm), "top5Hit") + } + ]; + }) ); } @@ -320,17 +384,23 @@ const summary = { cases: dataset.cases.length, configuration: { topN: TOP_N, - corpus: "one scanRepo() result per case, shared by every arm; ranking is the only difference", + corpus: "one scanRepo() result per case, shared by every arm", searchField: "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", tokenizer: "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", stopwords: STOPWORDS.size, caseSensitivity: "case-insensitive for both keyword arms, which favours the baselines", bm25: { k1: 1.2, b: 0.75 }, + candidatePolicies: { + raw: "every scanned file; ranks READMEs first and is not a fair comparison", + source: "isSource && !isTest — FixMap's own candidate gate", + code: "isSource && !isTest && kind === 'code' — also drops documentation, as FixMap's scoring effectively does for implementation tasks" + }, + bestPolicyPerFamily, armDescriptions: { "path-extraction": "path-shaped tokens read out of the task text, resolved against the corpus, ranked by order of appearance", "lexical-literal": "literal keyword search ranked by distinct query terms matched, then raw occurrence count", bm25: "BM25 retrieval over the same text; a retrieval baseline, not a grep", - fixmap: "rankContextFiles from @aryam/fixmap-core" + fixmap: "rankContextFiles from @aryam/fixmap-core, which applies its own candidate gate internally" } }, arms, From 62f256b6b7b612db61346925d1ca2daadbb1715b Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 13:06:26 +0530 Subject: [PATCH 5/7] fix: make the benchmark correction reproducible --- README.md | 4 +- apps/web/app/_lib/site-data.ts | 21 +++++-- benchmarks/external/README.md | 7 ++- benchmarks/heldout/README.md | 32 ++++++---- docs/BENCHMARKS.md | 9 +-- docs/GROWTH_LOG.md | 8 +++ docs/LAUNCH_KIT.md | 13 ++-- docs/assets/fixmap-benchmark.svg | 35 ++++++----- .../2026-08-04-benchmark-self-audit.md | 46 ++++++++++++++ scripts/evaluate-baseline.mjs | 29 ++++++++- scripts/render-benchmark-card.mjs | 63 ++++++++++--------- 11 files changed, 192 insertions(+), 75 deletions(-) create mode 100644 docs/releases/2026-08-04-benchmark-self-audit.md diff --git a/README.md b/README.md index d44db42..9d8b451 100644 --- a/README.md +++ b/README.md @@ -416,7 +416,7 @@ Public repository inputs accept only canonical credential-free `https://github.c Most tools show you the benchmark they tuned on. Here is both. -![FixMap benchmark: the fixing file ranked in the top three for 8 of 12 held-out repositories never tuned against and 16 of 16 in the regression suite, with a 1.75-second median scan and rank.](docs/assets/fixmap-benchmark.svg) +![FixMap evidence audit: on nine held-out tasks that did not name the fixing file, FixMap and BM25 both ranked it in the top three for five cases, while BM25 led six to nine at Top-5.](docs/assets/fixmap-benchmark.svg) FixMap is measured against real issues that were later fixed by a merged pull request. Each case pins the commit *before* the fix, feeds FixMap the issue text a maintainer actually wrote, and checks whether the file that fix changed appears in the ranking. Cases are chosen mechanically, and every input and output is checked in. @@ -465,6 +465,8 @@ On the regression suite FixMap does lead — 69% vs 39% Top-1, 100% vs 62% Top-3 We are publishing this because it is what the measurement says. The honest reading is that FixMap's current advantage over plain BM25-over-code is **unproven on unseen repositories**, and that its Top-5 recall is behind. Closing that gap is the next piece of work, not a marketing line. +[Read the benchmark self-audit.](docs/releases/2026-08-04-benchmark-self-audit.md) + Path extraction scoring 0% on this cohort and 100% on the named one is the check that the cohort split measures what it claims. Reproduce it with `node scripts/evaluate-baseline.mjs --suite heldout`; every arm, policy, and ranking is recorded in [`benchmarks/heldout/baseline-results.json`](benchmarks/heldout/baseline-results.json). diff --git a/apps/web/app/_lib/site-data.ts b/apps/web/app/_lib/site-data.ts index 3735dbb..6687ccf 100644 --- a/apps/web/app/_lib/site-data.ts +++ b/apps/web/app/_lib/site-data.ts @@ -34,12 +34,25 @@ const cohortOf = (suite: typeof heldout) => ({ // The comparison a ranked list actually has to win: naive retrieval on the same corpus. // Each baseline is reported at its STRONGEST candidate policy — pointing a baseline at every // scanned file makes it rank READMEs and turns the comparison into a strawman. -const baselineOf = (suite: typeof heldoutBaseline) => ({ +type BaselineSuite = typeof heldoutBaseline; +type BaselineFamily = keyof BaselineSuite["configuration"]["bestPolicyPerFamily"]; +type BaselineArm = keyof BaselineSuite["arms"]; + +const strongestBaseline = (suite: BaselineSuite, family: BaselineFamily) => { + const policy = suite.configuration.bestPolicyPerFamily[family]; + const arm = suite.arms[`${family}:${policy}` as BaselineArm]; + if (!arm) { + throw new Error(`Recorded baseline results do not contain ${family}:${policy}.`); + } + return arm.unmentioned; +}; + +const baselineOf = (suite: BaselineSuite) => ({ cases: suite.arms.fixmap.unmentioned.cases, fixmap: suite.arms.fixmap.unmentioned, - bm25: suite.arms["bm25:code"].unmentioned, - lexical: suite.arms["lexical-literal:code"].unmentioned, - pathExtraction: suite.arms["path-extraction:raw"].unmentioned + bm25: strongestBaseline(suite, "bm25"), + lexical: strongestBaseline(suite, "lexical-literal"), + pathExtraction: strongestBaseline(suite, "path-extraction") }); export const siteStats = { diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md index ca29e43..7ef4fa4 100644 --- a/benchmarks/external/README.md +++ b/benchmarks/external/README.md @@ -2,7 +2,7 @@ A reproducible evaluation of FixMap's context ranking against real, already-fixed issues in permissively licensed JavaScript/TypeScript repositories. It complements the small self-repository gate in [`benchmarks/cases.json`](../cases.json). -> **This suite measures regression, not generalization.** Its cases guided v0.7.1 ranking work — when a case missed, the ranker changed — so its hit rates describe how well FixMap fits code it has already been shaped by. For an estimate of behavior on unseen repositories, use [`benchmarks/heldout/`](../heldout), which was selected by the identical rule after the ranker was finished and is never tuned against. Quote the two together or quote the held-out number; quoting 100% alone overstates accuracy. +> **This suite measures regression, not generalization.** Its cases guided v0.7.1 ranking work — when a case missed, the ranker changed — so its hit rates describe how well FixMap fits code it has already been shaped by. For unseen evidence, use the unmentioned cohort in [`benchmarks/heldout/`](../heldout) with its BM25 baseline beside it. Quoting this suite's 100% alone overstates accuracy. ## Dataset @@ -41,13 +41,14 @@ npm run build:core node scripts/evaluate-external.mjs # report only node scripts/evaluate-external.mjs --gate # also fail below regression floors npm run evaluate:external:record # deliberately refresh results.json +node scripts/evaluate-baseline.mjs --suite external --check-recorded ``` The first run shallow-clones each repository at its pinned SHA into the OS temp directory (network required); later runs reuse the clones. Because of the network dependency this is not part of `npm run ci`; the [`external-eval` workflow](../../.github/workflows/external-eval.yml) runs it on a weekly schedule and on manual dispatch. Scheduled and release runs use `--check-recorded`, so a ranking change must deliberately refresh and review [`results.json`](results.json). ## Results -Measured 2026-08-01 on the dataset above (Node v24, `rankContextFiles` with a top-5 window): +Measured 2026-08-04 on the dataset above (Node v24, `rankContextFiles` with a top-5 window): | Metric | Hit rate | | --- | --- | @@ -55,6 +56,8 @@ Measured 2026-08-01 on the dataset above (Node v24, `rankContextFiles` with a to | top-3 | 16/16 (100%) | | top-5 | 16/16 (100%) | +Three tasks name an expected fixing path; the derived unmentioned cohort is 9/13 (69.2%) Top-1 and 13/13 Top-3/Top-5. On that cohort BM25-over-code measures 5/13 Top-1 and 8/13 Top-3/Top-5. FixMap leads here, but these cases shaped its ranker, so that difference remains regression evidence rather than a generalization claim. Per-arm rankings are in [`baseline-results.json`](baseline-results.json). + The freshly measured pre-change baseline on the original expanded 15-case dataset was 6/15 (40%) Top-1, 10/15 (67%) Top-3, and 10/15 (67%) Top-5. It was measured against the untouched pre-v0.7.1 ranker; it was not copied from the stale historical `results.json`. The v0.7.1 changes add honest identifier grounding and general ranking evidence for member references, explicit paths and literals, type-focused tasks, HTTP/2 naming, and issue-template noise. Those changes move the expected fixing file into the Top-3 for all five previously missed cases. They do not special-case repository names or expected paths. diff --git a/benchmarks/heldout/README.md b/benchmarks/heldout/README.md index 21af2ac..28d3ed9 100644 --- a/benchmarks/heldout/README.md +++ b/benchmarks/heldout/README.md @@ -31,21 +31,29 @@ These 12 repositories were selected by the same frozen rule. When a case informs ## Results -Measured 2026-08-01 (Node v24, `rankContextFiles` with a top-5 window): +Measured 2026-08-04 (Node v24, `rankContextFiles` with a top-5 window): -| Metric | Held-out (12 cases) | Regression suite (16 cases) | -| --- | ---: | ---: | -| top-1 | **7/12 (58.3%)** | 11/16 (68.8%) | -| top-3 | **8/12 (66.7%)** | 16/16 (100.0%) | -| top-5 | **9/12 (75.0%)** | 16/16 (100.0%) | +Three tasks name their expected fixing file in the issue text. They legitimately test FixMap's explicit-file-mention signal, but they do not test whether it can locate a file the task did not name. The evaluator therefore derives and reports both cohorts every run: + +| Cohort | Cases | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | ---: | +| Task did not name the file | 9 | **4/9 (44.4%)** | **5/9 (55.6%)** | **6/9 (66.7%)** | +| Task named the file | 3 | 3/3 (100%) | 3/3 (100%) | 3/3 (100%) | +| Pooled, previously published | 12 | 7/12 (58.3%) | 8/12 (66.7%) | 9/12 (75.0%) | + +**Plan around the unmentioned cohort.** At nine cases its Top-3 95% Wilson interval is 27–81%, so the point estimate is exploratory rather than a precise success probability. The three Top-5 misses — `socketio/socket.io`, `vitejs/vite`, and `vuejs/core` — remain recorded in [`results.json`](results.json) with their actual rankings. The Jest answer is fourth, so it also misses Top-3. -Read those two columns together, because the gap is the point. +## Baseline-relative result -Top-1 is lower on the untouched held-out repositories than on the development suite. The difference is small enough that the confidence intervals overlap substantially, but it is still the honest result to publish rather than a number to explain away. +[`baseline-results.json`](baseline-results.json) scores FixMap and naive retrieval against the same scan, text samples, and truncation. On the nine unmentioned cases: -Top-3 drops to 67% and Top-5 to 75%. That difference is what tuning bought on the regression suite and nothing more. **The held-out column is the honest evidence to plan around**, and 100% should not be quoted as an accuracy claim. +| Arm | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | +| Literal keyword retrieval, code files | 2/9 | 4/9 | 6/9 | +| BM25 retrieval, code files | **4/9** | **5/9** | **9/9** | +| FixMap | 4/9 | 5/9 | 6/9 | -The three Top-5 misses — `socketio/socket.io`, `vitejs/vite`, and `vuejs/core` — are recorded in [`results.json`](results.json) with their actual rankings. The Jest answer is fourth, so it also misses Top-3. These results are not removed, reweighted, or explained away. +FixMap does not beat BM25-over-code on this unseen cohort: Top-1 and Top-3 tie, while BM25 leads 9/9 to 6/9 at Top-5. With nine cases this is not a stable effect-size estimate, but the previously published advantage does not survive the baseline comparison. ## Confidence calibration @@ -65,10 +73,12 @@ The ordering is not monotonic in this small sample, so the labels must not be re npm run build:core npm run evaluate:heldout # report only npm run evaluate:heldout:record # deliberately refresh results.json +node scripts/evaluate-baseline.mjs --suite heldout +node scripts/evaluate-baseline.mjs --suite heldout --check-recorded ``` ## Rules for this suite 1. **Never tune against these cases.** The moment a ranking change is made because one of them missed, this suite becomes a second regression suite and stops measuring generalization. Move the case into `benchmarks/external/` and select a fresh replacement. 2. **Do not edit a case to match output.** When ranking behavior changes, rerun and update the recorded results. -3. **Report both columns.** Quoting the regression number alone overstates accuracy; quoting only the held-out number understates regression coverage. +3. **Report cohorts and the baseline.** Do not present the pooled held-out rate as generalization evidence, or present a FixMap rate without the naive retrieval result beside it. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index abe0dc3..a3d82d5 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -2,11 +2,11 @@ ## Cross-repository ranking and efficiency -![FixMap benchmark: the fixing file ranked in the top three for 8 of 12 held-out repositories never tuned against and 16 of 16 in the regression suite, with a 1.75-second median scan and rank.](assets/fixmap-benchmark.svg) +![FixMap evidence audit: on nine held-out tasks that did not name the fixing file, FixMap and BM25 both ranked it in the top three for five cases, while BM25 led six to nine at Top-5.](assets/fixmap-benchmark.svg) Two suites answer two different questions. The [regression suite](../benchmarks/external/README.md) uses 16 repositories whose cases have guided ranking work, so it measures fit rather than generalization. The [held-out suite](../benchmarks/heldout/README.md) uses 12 further repositories selected by the identical frozen rule and rotates any case that informs a ranking change, so it remains unseen evidence. Each case in both pins the repository state before the fix and freezes the fixing source paths before FixMap ranks anything. -Ranking outputs refreshed 2026-07-31 on Node v24.13.0, Windows 11 (10.0.26200), Intel Core i5-8350U; the scan-time measurement remains from 2026-07-26: +Ranking outputs refreshed 2026-08-04 on Node v24.13.0, Windows 11 (10.0.26200), Intel Core i5-8350U; the scan-time measurement remains from 2026-07-26: | Quantity | Held-out (12) | Regression (16) | Evidence type | | --- | ---: | ---: | --- | @@ -14,9 +14,8 @@ Ranking outputs refreshed 2026-07-31 on Node v24.13.0, Windows 11 (10.0.26200), | Expected fixing file in Top-3 | 8/12 (67%) | 16/16 (100%) | Measured, **pooled — see cohorts below** | | Expected fixing file in Top-5 | 9/12 (75%) | 16/16 (100%) | Measured, **pooled — see cohorts below** | | Median scan + rank time | — | 1,747.7 ms | Measured, three warm runs per pinned repository | -| Context proxy reduction | — | 98.56% | Estimated proxy, **not** a savings measurement | -**The held-out column is the one to plan around.** The regression column describes performance on cases that shaped the ranker and will overstate what happens on a repository FixMap has never seen. +**The held-out, unmentioned cohort below is the one to plan around.** The pooled held-out column includes three tasks that name their fixing file, while the regression column describes performance on cases that shaped the ranker. ### Cohorts: tasks that already name the fixing file @@ -131,6 +130,8 @@ Nine and thirteen cases cannot settle this either way; what they do show is that published margin does not survive a fair baseline. Growing the held-out suite and closing the Top-5 recall gap are the work this points at. +The dated narrative is in [the benchmark self-audit](releases/2026-08-04-benchmark-self-audit.md). + `path-extraction` scoring 0.0% on the unmentioned cohort of both suites and 100% on the named cohort is the independent check that the cohort classifier measures what it claims. diff --git a/docs/GROWTH_LOG.md b/docs/GROWTH_LOG.md index 9ccbc24..840ea8f 100644 --- a/docs/GROWTH_LOG.md +++ b/docs/GROWTH_LOG.md @@ -117,3 +117,11 @@ Separately, `docs/LAUNCH_KIT.md` held ready-to-paste post copy quoting `4/6` top X and LinkedIn posts published around 10:30 UTC on the misleading-rate finding: the regression suite's 100% top-3 was concealing that a wrong file ranked first in 40% of those cases, while the held-out suite does so in 8%. Record impressions, reactions, and profile views separately from repository outcomes at 24 and 72 hours. Stars, unique visitors, and qualified feedback remain the decision metrics. Add the post URLs to the experiments table once available; the rows are recorded with the timestamps and pending links rather than left out, so the checkpoints stay honest if either post produces nothing. + +## Evidence correction — 2026-08-04 + +An audit found that 3 of the 12 held-out tasks name their expected fixing file in the task text. Those cases remain useful tests of explicit-file-mention handling, but they no longer contribute to the generalization headline. On the nine tasks that did not name the file, FixMap measures 4/9 Top-1, 5/9 Top-3, and 6/9 Top-5; the 95% interval remains wide enough that these are exploratory point estimates. + +The same scanned corpora were then ranked with literal keyword retrieval and BM25. On the held-out unmentioned cohort, BM25-over-code ties FixMap at Top-1 and Top-3 and leads 9/9 to 6/9 at Top-5. The paired Top-1 and Top-3 comparisons are exact ties; at this sample size none establishes a stable effect size. + +**Decision gate:** FixMap does not currently beat the naive retrieval baseline on unseen repositories. Pause distribution built on a “better than search” claim and do not add workflow surface area on that premise. The next evidence work is to expand the mechanically selected unmentioned cohort and improve recall without tuning against cases that remain held out. diff --git a/docs/LAUNCH_KIT.md b/docs/LAUNCH_KIT.md index 1967b16..c704535 100644 --- a/docs/LAUNCH_KIT.md +++ b/docs/LAUNCH_KIT.md @@ -21,14 +21,15 @@ The report ranks likely files with reasons, suggests test routes, and names risk - CLI, MCP server, and GitHub Action share the same core ranker. - Public GitHub issue URLs supply both task context and the repository in one input; source is scanned in an isolated anonymous shallow checkout that is removed after analysis. - Two frozen evaluations use real fixed issues at pinned pre-fix commits, selected by a mechanical rule. -- **Held-out (12 repositories, never tuned against): top-1 `7/12` (58%), top-3 `8/12` (67%), top-5 `9/12` (75%).** +- **Held-out tasks that did not name the fixing file (9 repositories, never tuned against): top-1 `4/9` (44%), top-3 `5/9` (56%), top-5 `6/9` (67%).** Three further cases named their answer and are reported separately. +- On that same cohort and scanned corpus, BM25-over-code ties FixMap at Top-1 and Top-3 and leads `9/9` to `6/9` at Top-5. FixMap's advantage over naive retrieval is unproven. - Regression (16 repositories, guided development): top-1 `11/16` (69%), top-3 and top-5 `16/16` (100%). - Confidence labels are directional heuristics, not calibrated probabilities; all per-band counts remain public. -- An adversarial suite measures false confidence on fabricated identifiers, vague tasks, and absent features: `0.0` across 8 cases. +- An adversarial suite measures false confidence on fabricated identifiers, vague tasks, and absent features: `0.0` across 9 cases. - Every ranked output and both frozen selection rules are public in [`benchmarks/`](../benchmarks/). - A release cannot publish unless the local CI suite and external evaluation gate pass. -**Always quote the held-out figure, or both together — never the regression figure alone.** The regression suite guided ranking work, so its 100% describes fit rather than generalization. Quoting it on its own overstates accuracy and is the fastest way to lose a technical reader who checks. +**Always quote the unmentioned held-out cohort beside the BM25 baseline. Never use the pooled held-out or regression figure as a generalization claim.** The regression suite guided ranking work, while three held-out tasks already contained their fixing path. ## Truth guardrails @@ -124,7 +125,7 @@ Points for the maintainer to explain personally: 2. The one-sentence solution: deterministic repo context—ranked files, test routes, risks, and diagnostics. 3. The fastest trial: include the one-input public GitHub issue URL command. 4. The technical mechanism: path/content signals, real git diff signals, bounded static import proximity, file-kind priors, and workspace boundaries. -5. The honest evidence: 12 held-out repositories never tuned against, `7/12` top-1 and `8/12` top-3, alongside the regression suite it was developed on, with every per-case ranking linked. +5. The honest evidence: on nine held-out tasks that did not name their fixing file, FixMap scores `4/9` Top-1 and `5/9` Top-3, exactly tied with BM25-over-code; BM25 leads at Top-5. Link every per-case ranking. 6. The scope: JavaScript/TypeScript today; remote URLs are issue-only; suggested tests are not executed. 7. What the benchmarks did not catch: both suites passed while FixMap could not find chalk's own color-detection code, because a directory blocklist ran after git had already applied `.gitignore` and a frequency cutoff suppressed the word "color" in a library about color. Pointing it at a repository it had never been run on found that; the benchmark never would have. 8. Ask for technical criticism of the evaluation and useful next signals. @@ -145,7 +146,7 @@ These are angles and evidence, not identical copy to syndicate. - Problem: agents spend context and tokens discovering where to start. - Demo: run FixMap first on a public repository, then hand the report to the agent. -- Evidence: deterministic, zero model calls, inspectable reasons, `8/12` top-3 on repositories it was never tuned against. +- Evidence: deterministic, zero model calls, and inspectable reasons; on nine held-out tasks that did not name the file, FixMap ties BM25-over-code at Top-1 and Top-3 and trails it at Top-5. - Honest caveat: it is a routing aid, not semantic code understanding or a correctness oracle. ### Claude Code and Cursor communities @@ -172,7 +173,7 @@ claude mcp add fixmap -- npx -y @aryam/fixmap@latest mcp 1. One pain sentence. 2. The public repository command. 3. A screenshot or short terminal video of the real output. -4. One evidence sentence: 12 held-out pinned bugs, `7/12` top-1 and `8/12` top-3, with the full result and the misses public. +4. One evidence sentence: nine held-out pinned bugs whose tasks did not name the file, FixMap tied with BM25-over-code at `4/9` Top-1 and `5/9` Top-3, with every ranking public. 5. Repository link and a specific feedback question. Avoid generic feature lists and unsupported superlatives. diff --git a/docs/assets/fixmap-benchmark.svg b/docs/assets/fixmap-benchmark.svg index 400859d..ec44dc7 100644 --- a/docs/assets/fixmap-benchmark.svg +++ b/docs/assets/fixmap-benchmark.svg @@ -1,6 +1,6 @@ FixMap benchmark - FixMap ranked the fixing file in the top three for 8 of 12 held-out repositories it was never tuned against, and 16 of 16 in the regression suite, with a 1.75s median scan and rank. + On 9 held-out tasks that did not name the fixing file, FixMap and BM25 both ranked it in the top three for 5 cases. BM25 ranked it in the top five for 9, compared with 6 for FixMap. @@ -13,30 +13,31 @@ - FIXMAP · REPRODUCIBLE BENCHMARK - Find the right code before the first edit. + FIXMAP · EVIDENCE AUDIT + The baseline erased the published advantage. - HELD-OUT ACCURACY - 8/12 - top-3 on repos never tuned against - Top-1 7/12 · Top-5 9/12 + FIXMAP · UNMENTIONED + 5/9 + top-3 on held-out tasks + Top-1 4/9 · Top-5 6/9 - MEASURED RUNTIME - 1.75s - median scan + rank - 3 warm runs per pinned repository + BM25 · SAME CORPUS + 5/9 + top-3 on held-out tasks + Top-1 4/9 · Top-5 9/9 - REGRESSION SUITE - 16/16 - top-3 on cases that guided development - Top-1 11/16 · not a generalization estimate + MEASURED RUNTIME + 1.75s + median scan + rank + 3 warm runs per pinned repository - WHAT THIS DOES NOT CLAIM - No tokens-saved or minutes-saved figure: that needs a controlled with/without-agent run, which has not been done. + WHAT THE AUDIT FOUND + FixMap does not beat BM25 over code files on unseen repositories. + 3 of 12 tasks named their answer; those cases are now reported separately. Pinned commits · real merged fixes · expected files taken from the fixing PR · every case and ranking checked into the repository diff --git a/docs/releases/2026-08-04-benchmark-self-audit.md b/docs/releases/2026-08-04-benchmark-self-audit.md new file mode 100644 index 0000000..3aa268e --- /dev/null +++ b/docs/releases/2026-08-04-benchmark-self-audit.md @@ -0,0 +1,46 @@ +# I audited FixMap's benchmark and found it was leaking answers + +FixMap's held-out benchmark was meant to answer one question: when a task does not tell you where to look, does FixMap surface the file that later fixed it? + +Three of its twelve tasks did tell FixMap where to look. Mongoose named `lib/document.js` with a line number; the Svelte and yargs tasks included the expected path in their text. All three ranked Top-1. They remain valid tests of explicit-file-mention handling, but they cannot count as evidence that FixMap located an unnamed file. + +## The corrected cohorts + +| Held-out cohort | Cases | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | ---: | +| Task did not name the file | 9 | 4/9 (44.4%) | 5/9 (55.6%) | 6/9 (66.7%) | +| Task named the file | 3 | 3/3 | 3/3 | 3/3 | +| Pooled, previously published | 12 | 7/12 (58.3%) | 8/12 (66.7%) | 9/12 (75.0%) | + +The split is computed from the task and expected paths during every evaluation. It is not a hand-maintained dataset label. With nine unmentioned cases the uncertainty is still large: the Top-3 95% Wilson interval is 27–81%. + +## The missing baseline changed the conclusion + +The published hit rate had never been compared with naive retrieval. The audit now runs FixMap, literal keyword retrieval, and BM25 against one shared repository scan per case. Each arm sees the same file paths, text samples, and truncation. + +On held-out tasks that did not name the file: + +| Arm | Top-1 | Top-3 | Top-5 | +| --- | ---: | ---: | ---: | +| Literal retrieval over code files | 2/9 | 4/9 | 6/9 | +| BM25 over code files | 4/9 | 5/9 | 9/9 | +| FixMap | 4/9 | 5/9 | 6/9 | + +FixMap does not beat BM25-over-code on this unseen cohort. It ties at Top-1 and Top-3, while BM25 finds all nine expected files within five results and FixMap finds six. A paired exact test cannot establish a stable advantage at this sample size; the honest conclusion is that FixMap's advantage over naive retrieval is unproven. + +## What changes now + +- Public evidence leads with the unmentioned cohort, with the BM25 result beside it. +- Pooled results remain available for audit but are labeled as pooled rather than generalization evidence. +- Distribution based on a “better than search” claim is paused. +- Ranking and dataset work must improve recall on fresh, mechanically selected cases without tuning against cases that remain held out. + +Reproduce the recorded results: + +```bash +npm run build:core +node scripts/evaluate-external.mjs --suite heldout --check-recorded +node scripts/evaluate-baseline.mjs --suite heldout --check-recorded +``` + +The complete per-case outputs are in [`benchmarks/heldout/results.json`](../../benchmarks/heldout/results.json) and [`benchmarks/heldout/baseline-results.json`](../../benchmarks/heldout/baseline-results.json). diff --git a/scripts/evaluate-baseline.mjs b/scripts/evaluate-baseline.mjs index aa27580..0f35291 100644 --- a/scripts/evaluate-baseline.mjs +++ b/scripts/evaluate-baseline.mjs @@ -50,6 +50,7 @@ if (!["external", "heldout"].includes(suite)) { const suiteDir = join(repoRoot, "benchmarks", suite); const dataset = JSON.parse(await readFile(join(suiteDir, "dataset.json"), "utf8")); +const recordedResultsPath = join(suiteDir, "baseline-results.json"); const TOP_N = 5; @@ -347,12 +348,21 @@ for (const arm of ARMS) { // weakest. For every baseline family, pick the candidate policy that scored best on the // unmentioned cohort and treat that as the arm to beat. const bestPolicyPerFamily = {}; +function compareScoreTuples(left, right) { + for (let index = 0; index < left.length; index += 1) { + if (left[index] !== right[index]) { + return left[index] - right[index]; + } + } + return 0; +} + for (const family of BASELINE_ARMS) { let best = null; for (const policy of Object.keys(CANDIDATE_POLICIES)) { const score = arms[`${family}:${policy}`].unmentioned; const key = [score.top1HitRate ?? 0, score.top3HitRate ?? 0, score.top5HitRate ?? 0]; - if (!best || key > best.key) { + if (!best || compareScoreTuples(key, best.key) > 0) { best = { policy, key }; } } @@ -413,5 +423,20 @@ const rendered = `${JSON.stringify(summary, null, 2)}\n`; process.stdout.write(rendered); if (process.argv.includes("--record")) { - await writeFile(join(suiteDir, "baseline-results.json"), rendered, "utf8"); + await writeFile(recordedResultsPath, rendered, "utf8"); +} + +if (process.argv.includes("--check-recorded")) { + let recorded = ""; + try { + recorded = await readFile(recordedResultsPath, "utf8"); + } catch { + // The mismatch message below also covers a missing or unreadable artifact. + } + if (recorded !== rendered) { + process.stderr.write( + `Baseline evaluation differs from benchmarks/${suite}/baseline-results.json; rerun with --record and review the change.\n` + ); + process.exit(1); + } } diff --git a/scripts/render-benchmark-card.mjs b/scripts/render-benchmark-card.mjs index 7623a1d..fac80af 100644 --- a/scripts/render-benchmark-card.mjs +++ b/scripts/render-benchmark-card.mjs @@ -2,10 +2,10 @@ // // npm run render:benchmark-card // -// The card leads with the held-out suite because that is the only number that -// estimates behavior on a repository FixMap was never shaped by. The regression -// suite is shown beside it and labeled as such, so a reader can see both the -// tuned figure and the honest one rather than only the flattering one. +// The card leads with the held-out tasks that did not name their fixing file and +// places the strongest naive retrieval baseline beside FixMap. A pooled score or a +// FixMap score without that baseline would repeat the benchmark mistake this card +// now exists to disclose. // // Deliberately absent: any "tokens saved" or "minutes saved" headline. Both would // need a controlled with/without-agent experiment that has not been run, and a @@ -20,25 +20,31 @@ const readJson = async (...segments) => JSON.parse(await readFile(join(repoRoot, ...segments), "utf8")); const heldout = await readJson("benchmarks", "heldout", "results.json"); -const external = await readJson("benchmarks", "external", "results.json"); +const baseline = await readJson("benchmarks", "heldout", "baseline-results.json"); const savings = await readJson("benchmarks", "external", "savings-results.json"); const output = join(repoRoot, "docs", "assets", "fixmap-benchmark.svg"); -const count = (suite, key) => suite.results.filter((result) => result[key]).length; const runtime = savings.performance.medianScanAndRankMs; const runtimeLabel = runtime >= 1000 ? `${(runtime / 1000).toFixed(2)}s` : `${Math.round(runtime)}ms`; -const heldoutTop3 = count(heldout, "top3"); -const heldoutTop1 = count(heldout, "top1"); -const heldoutTop5 = count(heldout, "top5Hit"); -const externalTop3 = count(external, "top3"); -const externalTop1 = count(external, "top1"); +const unmentioned = heldout.results.filter((result) => !result.mentionsExpectedPath); +const fixmapTop1 = unmentioned.filter((result) => result.top1).length; +const fixmapTop3 = unmentioned.filter((result) => result.top3).length; +const fixmapTop5 = unmentioned.filter((result) => result.top5Hit).length; +const bm25Policy = baseline.configuration.bestPolicyPerFamily.bm25; +const bm25Arm = `bm25:${bm25Policy}`; +const baselineUnmentioned = baseline.results.filter((result) => !result.mentionsExpectedPath); +const bm25Top1 = baselineUnmentioned.filter((result) => result.arms[bm25Arm].top1).length; +const bm25Top3 = baselineUnmentioned.filter((result) => result.arms[bm25Arm].top3).length; +const bm25Top5 = baselineUnmentioned.filter((result) => result.arms[bm25Arm].top5Hit).length; +const cohortSize = unmentioned.length; +const namedCases = heldout.results.length - cohortSize; const svg = ` FixMap benchmark - FixMap ranked the fixing file in the top three for ${heldoutTop3} of ${heldout.cases} held-out repositories it was never tuned against, and ${externalTop3} of ${external.cases} in the regression suite, with a ${runtimeLabel} median scan and rank. + On ${cohortSize} held-out tasks that did not name the fixing file, FixMap and BM25 both ranked it in the top three for ${fixmapTop3} cases. BM25 ranked it in the top five for ${bm25Top5}, compared with ${fixmapTop5} for FixMap. @@ -51,30 +57,31 @@ const svg = ` - FIXMAP · REPRODUCIBLE BENCHMARK - Find the right code before the first edit. + FIXMAP · EVIDENCE AUDIT + The baseline erased the published advantage. - HELD-OUT ACCURACY - ${heldoutTop3}/${heldout.cases} - top-3 on repos never tuned against - Top-1 ${heldoutTop1}/${heldout.cases} · Top-5 ${heldoutTop5}/${heldout.cases} + FIXMAP · UNMENTIONED + ${fixmapTop3}/${cohortSize} + top-3 on held-out tasks + Top-1 ${fixmapTop1}/${cohortSize} · Top-5 ${fixmapTop5}/${cohortSize} - MEASURED RUNTIME - ${runtimeLabel} - median scan + rank - ${savings.environment.runsPerRepository} warm runs per pinned repository + BM25 · SAME CORPUS + ${bm25Top3}/${cohortSize} + top-3 on held-out tasks + Top-1 ${bm25Top1}/${cohortSize} · Top-5 ${bm25Top5}/${cohortSize} - REGRESSION SUITE - ${externalTop3}/${external.cases} - top-3 on cases that guided development - Top-1 ${externalTop1}/${external.cases} · not a generalization estimate + MEASURED RUNTIME + ${runtimeLabel} + median scan + rank + ${savings.environment.runsPerRepository} warm runs per pinned repository - WHAT THIS DOES NOT CLAIM - No tokens-saved or minutes-saved figure: that needs a controlled with/without-agent run, which has not been done. + WHAT THE AUDIT FOUND + FixMap does not beat BM25 over code files on unseen repositories. + ${namedCases} of ${heldout.cases} tasks named their answer; those cases are now reported separately. Pinned commits · real merged fixes · expected files taken from the fixing PR · every case and ranking checked into the repository From 47f3c31f093ad3384f07db9a159b77e0a0923749 Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 13:08:34 +0530 Subject: [PATCH 6/7] ci: gate recorded retrieval baselines --- .github/workflows/external-eval.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/external-eval.yml b/.github/workflows/external-eval.yml index 572c13c..8d1ec67 100644 --- a/.github/workflows/external-eval.yml +++ b/.github/workflows/external-eval.yml @@ -22,3 +22,5 @@ jobs: - run: npm run build:core - run: node scripts/evaluate-external.mjs --gate --check-recorded - run: node scripts/evaluate-external.mjs --suite heldout --gate --check-recorded + - run: node scripts/evaluate-baseline.mjs --suite external --check-recorded + - run: node scripts/evaluate-baseline.mjs --suite heldout --check-recorded From 3fbb0743cc452b84122d64b2d8b09e7abac9d3a3 Mon Sep 17 00:00:00 2001 From: Aryam Goyal Date: Tue, 4 Aug 2026 13:30:30 +0530 Subject: [PATCH 7/7] fix: make baseline records platform-stable --- benchmarks/external/baseline-results.json | 97 +---------------------- benchmarks/heldout/baseline-results.json | 73 +---------------- scripts/evaluate-baseline.mjs | 5 +- 3 files changed, 3 insertions(+), 172 deletions(-) diff --git a/benchmarks/external/baseline-results.json b/benchmarks/external/baseline-results.json index 853377d..20ab2c6 100644 --- a/benchmarks/external/baseline-results.json +++ b/benchmarks/external/baseline-results.json @@ -4,6 +4,7 @@ "configuration": { "topN": 5, "corpus": "one scanRepo() result per case, shared by every arm", + "recordedCorpusFields": "rankings and hit outcomes only; platform-dependent checkout file counts are deliberately omitted", "searchField": "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", "tokenizer": "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", "stopwords": 158, @@ -783,12 +784,6 @@ "expected": [ "lib/request.js" ], - "scannedFiles": 213, - "candidateCounts": { - "raw": 213, - "source": 65, - "code": 54 - }, "queryTermCount": 47, "mentionsExpectedPath": true, "mentionTier": "full-path", @@ -908,12 +903,6 @@ "expected": [ "lib/core/AxiosError.js" ], - "scannedFiles": 454, - "candidateCounts": { - "raw": 454, - "source": 266, - "code": 80 - }, "queryTermCount": 36, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1027,12 +1016,6 @@ "expected": [ "src/node.js" ], - "scannedFiles": 14, - "candidateCounts": { - "raw": 14, - "source": 11, - "code": 6 - }, "queryTermCount": 30, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1146,12 +1129,6 @@ "expected": [ "source/core/Ky.ts" ], - "scannedFiles": 67, - "candidateCounts": { - "raw": 67, - "source": 36, - "code": 29 - }, "queryTermCount": 42, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1265,12 +1242,6 @@ "expected": [ "packages/zod/src/v4/core/regexes.ts" ], - "scannedFiles": 582, - "candidateCounts": { - "raw": 582, - "source": 302, - "code": 228 - }, "queryTermCount": 38, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1385,12 +1356,6 @@ "lib/transport.js", "lib/worker.js" ], - "scannedFiles": 200, - "candidateCounts": { - "raw": 200, - "source": 67, - "code": 40 - }, "queryTermCount": 28, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1504,12 +1469,6 @@ "expected": [ "lib/request.js" ], - "scannedFiles": 393, - "candidateCounts": { - "raw": 393, - "source": 152, - "code": 72 - }, "queryTermCount": 20, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1623,12 +1582,6 @@ "expected": [ "source/vendor/supports-color/index.js" ], - "scannedFiles": 33, - "candidateCounts": { - "raw": 33, - "source": 19, - "code": 12 - }, "queryTermCount": 34, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1742,12 +1695,6 @@ "expected": [ "packages/browser/src/client/tester/dialog.ts" ], - "scannedFiles": 3050, - "candidateCounts": { - "raw": 3050, - "source": 1004, - "code": 610 - }, "queryTermCount": 46, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1861,12 +1808,6 @@ "expected": [ "lib/types/config-api.d.ts" ], - "scannedFiles": 2297, - "candidateCounts": { - "raw": 2297, - "source": 953, - "code": 445 - }, "queryTermCount": 34, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1980,12 +1921,6 @@ "expected": [ "lib/DefinePlugin.js" ], - "scannedFiles": 13454, - "candidateCounts": { - "raw": 13454, - "source": 1509, - "code": 1158 - }, "queryTermCount": 38, "mentionsExpectedPath": false, "mentionTier": "none", @@ -2099,12 +2034,6 @@ "expected": [ "lib/dispatcher/client-h2.js" ], - "scannedFiles": 786, - "candidateCounts": { - "raw": 786, - "source": 300, - "code": 209 - }, "queryTermCount": 52, "mentionsExpectedPath": false, "mentionTier": "none", @@ -2218,12 +2147,6 @@ "expected": [ "packages/toolkit/src/query/react/buildHooks.ts" ], - "scannedFiles": 1152, - "candidateCounts": { - "raw": 1152, - "source": 802, - "code": 595 - }, "queryTermCount": 47, "mentionsExpectedPath": true, "mentionTier": "path-suffix", @@ -2343,12 +2266,6 @@ "expected": [ "src/language-css/print/comma-separated-value-group.js" ], - "scannedFiles": 9319, - "candidateCounts": { - "raw": 9319, - "source": 881, - "code": 720 - }, "queryTermCount": 61, "mentionsExpectedPath": false, "mentionTier": "none", @@ -2462,12 +2379,6 @@ "expected": [ "src/adapter/aws-lambda/types.ts" ], - "scannedFiles": 482, - "candidateCounts": { - "raw": 482, - "source": 304, - "code": 243 - }, "queryTermCount": 35, "mentionsExpectedPath": false, "mentionTier": "none", @@ -2581,12 +2492,6 @@ "expected": [ "source/core/index.ts" ], - "scannedFiles": 120, - "candidateCounts": { - "raw": 120, - "source": 62, - "code": 31 - }, "queryTermCount": 42, "mentionsExpectedPath": true, "mentionTier": "full-path", diff --git a/benchmarks/heldout/baseline-results.json b/benchmarks/heldout/baseline-results.json index 4e831c3..6913ed5 100644 --- a/benchmarks/heldout/baseline-results.json +++ b/benchmarks/heldout/baseline-results.json @@ -4,6 +4,7 @@ "configuration": { "topN": 5, "corpus": "one scanRepo() result per case, shared by every arm", + "recordedCorpusFields": "rankings and hit outcomes only; platform-dependent checkout file counts are deliberately omitted", "searchField": "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", "tokenizer": "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", "stopwords": 158, @@ -783,12 +784,6 @@ "expected": [ "lib/document.js" ], - "scannedFiles": 702, - "candidateCounts": { - "raw": 702, - "source": 443, - "code": 317 - }, "queryTermCount": 36, "mentionsExpectedPath": true, "mentionTier": "full-path", @@ -908,12 +903,6 @@ "expected": [ "src/types/types-external.ts" ], - "scannedFiles": 165, - "candidateCounts": { - "raw": 165, - "source": 83, - "code": 28 - }, "queryTermCount": 24, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1027,12 +1016,6 @@ "expected": [ "packages/jest-mock/src/index.ts" ], - "scannedFiles": 3249, - "candidateCounts": { - "raw": 3249, - "source": 1622, - "code": 826 - }, "queryTermCount": 50, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1146,12 +1129,6 @@ "expected": [ "lib/dialects/postgres/query/pg-querycompiler.js" ], - "scannedFiles": 600, - "candidateCounts": { - "raw": 600, - "source": 260, - "code": 188 - }, "queryTermCount": 31, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1265,12 +1242,6 @@ "expected": [ "lib/reporters/xunit.js" ], - "scannedFiles": 661, - "candidateCounts": { - "raw": 661, - "source": 129, - "code": 69 - }, "queryTermCount": 46, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1388,12 +1359,6 @@ "expected": [ "src/logic/createFormControl.ts" ], - "scannedFiles": 490, - "candidateCounts": { - "raw": 490, - "source": 320, - "code": 257 - }, "queryTermCount": 52, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1507,12 +1472,6 @@ "expected": [ "packages/engine.io-client/lib/socket.ts" ], - "scannedFiles": 829, - "candidateCounts": { - "raw": 829, - "source": 520, - "code": 260 - }, "queryTermCount": 27, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1626,12 +1585,6 @@ "expected": [ "packages/svelte/src/internal/client/dom/blocks/boundary.js" ], - "scannedFiles": 8974, - "candidateCounts": { - "raw": 8974, - "source": 610, - "code": 444 - }, "queryTermCount": 43, "mentionsExpectedPath": true, "mentionTier": "full-path", @@ -1751,12 +1704,6 @@ "expected": [ "packages/vite/src/node/server/bundledDev.ts" ], - "scannedFiles": 2758, - "candidateCounts": { - "raw": 2758, - "source": 1806, - "code": 1241 - }, "queryTermCount": 48, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1870,12 +1817,6 @@ "expected": [ "packages/runtime-dom/src/index.ts" ], - "scannedFiles": 699, - "candidateCounts": { - "raw": 699, - "source": 408, - "code": 324 - }, "queryTermCount": 31, "mentionsExpectedPath": false, "mentionTier": "none", @@ -1989,12 +1930,6 @@ "expected": [ "lib/winston/transports/file.js" ], - "scannedFiles": 115, - "candidateCounts": { - "raw": 115, - "source": 65, - "code": 46 - }, "queryTermCount": 38, "mentionsExpectedPath": false, "mentionTier": "none", @@ -2108,12 +2043,6 @@ "expected": [ "lib/utils/apply-extends.ts" ], - "scannedFiles": 170, - "candidateCounts": { - "raw": 170, - "source": 103, - "code": 57 - }, "queryTermCount": 39, "mentionsExpectedPath": true, "mentionTier": "full-path", diff --git a/scripts/evaluate-baseline.mjs b/scripts/evaluate-baseline.mjs index 0f35291..bc6e5b2 100644 --- a/scripts/evaluate-baseline.mjs +++ b/scripts/evaluate-baseline.mjs @@ -256,10 +256,8 @@ for (const benchmark of dataset.cases) { const ranked = { fixmap: rankContextFiles(repo, { issueText: benchmark.task }, TOP_N).map((file) => file.path) }; - const policyCounts = {}; for (const [policy, predicate] of Object.entries(CANDIDATE_POLICIES)) { const files = repo.files.filter(predicate); - policyCounts[policy] = files.length; ranked[`path-extraction:${policy}`] = rankByPathExtraction(files, benchmark.task); ranked[`lexical-literal:${policy}`] = rankByLexicalLiteral(files, terms); ranked[`bm25:${policy}`] = rankByBm25(files, terms); @@ -268,8 +266,6 @@ for (const benchmark of dataset.cases) { const caseRow = { slug: benchmark.slug, expected: benchmark.expected, - scannedFiles: repo.files.length, - candidateCounts: policyCounts, queryTermCount: terms.length, mentionsExpectedPath: mention.mentionsExpectedPath, mentionTier: mention.mentionTier, @@ -395,6 +391,7 @@ const summary = { configuration: { topN: TOP_N, corpus: "one scanRepo() result per case, shared by every arm", + recordedCorpusFields: "rankings and hit outcomes only; platform-dependent checkout file counts are deliberately omitted", searchField: "file path + scanner text sample (files over the scanner's sample limit are truncated for every arm alike)", tokenizer: "lowercase [A-Za-z0-9_$]+ of length >= 3, plus camelCase and underscore sub-tokens", stopwords: STOPWORDS.size,