evidence: cohort-split the benchmark, and withdraw the baseline claim that did not survive a fair comparison

.github/workflows/external-eval.yml

@@ -22,3 +22,5 @@ jobs:
       - run: npm run build:core
       - run: node scripts/evaluate-external.mjs --gate --check-recorded
       - run: node scripts/evaluate-external.mjs --suite heldout --gate --check-recorded
+      - run: node scripts/evaluate-baseline.mjs --suite external --check-recorded
+      - run: node scripts/evaluate-baseline.mjs --suite heldout --check-recorded

README.md

@@ -416,21 +416,62 @@ Public repository inputs accept only canonical credential-free `https://github.c
 
 Most tools show you the benchmark they tuned on. Here is both.
 
-![FixMap benchmark: the fixing file ranked in the top three for 8 of 12 held-out repositories never tuned against and 16 of 16 in the regression suite, with a 1.75-second median scan and rank.](docs/assets/fixmap-benchmark.svg)
+![FixMap evidence audit: on nine held-out tasks that did not name the fixing file, FixMap and BM25 both ranked it in the top three for five cases, while BM25 led six to nine at Top-5.](docs/assets/fixmap-benchmark.svg)
 
 FixMap is measured against real issues that were later fixed by a merged pull request. Each case pins the commit *before* the fix, feeds FixMap the issue text a maintainer actually wrote, and checks whether the file that fix changed appears in the ranking. Cases are chosen mechanically, and every input and output is checked in.
 
 | | Held-out — 12 repos, **never tuned against** | Regression — 16 repos, guided development |
 | --- | ---: | ---: |
-| Fixing file ranked Top-1 | **7 / 12 — 58%** <br><sub>95% CI 32–81%</sub> | 11 / 16 — 69% <br><sub>95% CI 44–86%</sub> |
-| Fixing file ranked Top-3 | **8 / 12 — 67%** <br><sub>95% CI 39–86%</sub> | 16 / 16 — 100% <br><sub>95% CI 81–100%</sub> |
-| Wrong file ranked first while the right one was available | **2 / 12 — 17%** | 5 / 16 — 31% |
+| Fixing file ranked Top-1 | 7 / 12 — 58% <br><sub>95% CI 32–81%</sub> | 11 / 16 — 69% <br><sub>95% CI 44–86%</sub> |
+| Fixing file ranked Top-3 | 8 / 12 — 67% <br><sub>95% CI 39–86%</sub> | 16 / 16 — 100% <br><sub>95% CI 81–100%</sub> |
+| Wrong file ranked first while the right one was available | 2 / 12 — 17% | 5 / 16 — 31% |
 
-**Plan around the held-out column.** The regression suite is where the ranking heuristics were developed — a case missed, the ranker changed — so its 100% describes fit, not accuracy on your repository.
+#### Some of those tasks already contained their answer
 
-**And read the intervals, not the percentages.** At twelve cases one result flipping moves Top-3 by eight points. The honest statement is "roughly two thirds, with a wide interval", not a precise success probability. Anyone quoting these figures to two significant figures, including us, is overstating them.
+Three of the twelve held-out tasks name a fixing file in the task text itself. Mongoose's says `Location: lib/document.js:2339`; svelte's and yargs' link a GitHub permalink straight to the file and line range. A ranker that reads explicit file mentions — which FixMap has — answers those by reading the task, not by searching the repository. Pooling them into one rate lets three cases carry the headline.
 
-Two things the point estimates hide. Held-out Top-1 (58%) remains close to its Top-3 (67%) — **when FixMap finds the file at all, it usually ranks it first**, which is what actually matters to an agent that opens one file. The tuned suite's 100% Top-3 still conceals that in 31% of those cases something wrong ranks above the answer, so an agent following it opens the wrong file first.
+Split by whether the task named the file, the held-out suite reads:
+
+| Held-out cohort | Cases | Top-1 | Top-3 | Top-5 |
+| --- | ---: | ---: | ---: | ---: |
+| Task **did not** name the file — *plan around this one* | 9 | **44%** <br><sub>95% CI 19–73%</sub> | **56%** <br><sub>95% CI 27–81%</sub> | 67% |
+| Task named the file | 3 | 100% | 100% | 100% |
+| Pooled (what we published before) | 12 | 58% | 67% | 75% |
+
+The same split on the regression suite barely moves it (69% → 69% Top-1), and its three named cases are 2 / 3 rather than 3 / 3 — so being named does not guarantee a hit, and with three cases per cohort the *size* of this effect is not established. What is established is structural: a generalization headline should not be computed over tasks that contain their own answer. The cohort is now derived at evaluation time from the same task text the ranker reads, so it cannot drift.
+
+**Plan around the held-out, unmentioned cohort.** The regression suite is where the ranking heuristics were developed — a case missed, the ranker changed — so its 100% describes fit, not accuracy on your repository.
+
+**And read the intervals, not the percentages.** At nine cases one result flipping moves Top-3 by eleven points. The honest statement is "roughly half, with a wide interval", not a precise success probability. Anyone quoting these figures to two significant figures, including us, is overstating them.
+
+#### Is this better than just searching the repository?
+
+The fair question about a ranked file list is whether it beats what an agent already gets for free. The same suites are scored against naive retrieval on **the same scanned corpus** — one repository scan per case, shared by every arm.
+
+Candidate policy turned out to matter more than the ranking function. FixMap does not rank the raw scan: it gates on `isSource && !isTest` and then deprioritises documentation for an implementation task. A baseline pointed at every scanned file therefore returns `README.md` and `CONTRIBUTING.md` first and loses to the wrong thing. So each baseline is run under three candidate policies and **compared at its strongest**.
+
+Held-out, tasks that did not name the file (9 cases), each baseline at its best policy:
+
+| Arm | Top-1 | Top-3 | Top-5 |
+| --- | ---: | ---: | ---: |
+| Path extraction — read paths out of the task | 0% | 0% | 0% |
+| Literal keyword search, code files only | 22% | 44% | 67% |
+| **BM25 retrieval, code files only** | **44%** | **56%** | **100%** |
+| FixMap | 44% | 56% | 67% |
+
+**On repositories FixMap was never tuned against, BM25 over code files matches it at Top-1 and Top-3 and beats it at Top-5.** Paired McNemar exact tests put Top-1 and Top-3 at p = 1.0 — dead ties, two disagreements each way. At Top-5 the baseline wins 3 cases FixMap misses and FixMap wins none: BM25 has the fixing file in its top five for **9 of 9** cases, FixMap for 6 of 9.
+
+On the regression suite FixMap does lead — 69% vs 39% Top-1, 100% vs 62% Top-3 — but that is the suite whose cases shaped the ranker, and even there the lead is not significant against this baseline (p = 0.125 Top-1, p = 0.0625 Top-3).
+
+We are publishing this because it is what the measurement says. The honest reading is that FixMap's current advantage over plain BM25-over-code is **unproven on unseen repositories**, and that its Top-5 recall is behind. Closing that gap is the next piece of work, not a marketing line.
+
+[Read the benchmark self-audit.](docs/releases/2026-08-04-benchmark-self-audit.md)
+
+Path extraction scoring 0% on this cohort and 100% on the named one is the check that the cohort split measures what it claims.
+
+Reproduce it with `node scripts/evaluate-baseline.mjs --suite heldout`; every arm, policy, and ranking is recorded in [`benchmarks/heldout/baseline-results.json`](benchmarks/heldout/baseline-results.json).
+
+One thing the point estimates hide: held-out Top-1 stays close to Top-3 in both cohorts — **when FixMap finds the file at all, it usually ranks it first**, which is what matters to an agent that opens one file. The tuned suite's 100% Top-3 still conceals that in 31% of those cases something wrong ranks above the answer, so an agent following it opens the wrong file first.
 
 The three held-out misses are published with their real rankings in [`benchmarks/heldout/`](benchmarks/heldout), not removed or explained away.
 

apps/web/app/_lib/site-data.ts

@@ -1,5 +1,7 @@
 import heldout from "../../../../benchmarks/heldout/results.json";
 import regression from "../../../../benchmarks/external/results.json";
+import heldoutBaseline from "../../../../benchmarks/heldout/baseline-results.json";
+import regressionBaseline from "../../../../benchmarks/external/baseline-results.json";
 import savings from "../../../../benchmarks/external/savings-results.json";
 import cli from "../../../../packages/cli/package.json";
 
@@ -19,6 +21,40 @@ export const repoUrl = "https://github.com/aryamthecodebreaker/FixMap";
 export const npmUrl = "https://www.npmjs.com/package/@aryam/fixmap";
 export const marketplaceUrl = "https://github.com/marketplace/actions/fixmap";
 
+// Some benchmark tasks name the fixing file outright — a "Location: lib/document.js:2339"
+// line, or a GitHub permalink to the exact range. Those are answerable by reading the task
+// rather than by searching the repository, so the site reports the cohort that had to be
+// located separately from the pooled rate. Read straight from the recorded results so the
+// page cannot drift from the suite.
+const cohortOf = (suite: typeof heldout) => ({
+  unmentioned: suite.cohorts.unmentioned,
+  mentioned: suite.cohorts.mentioned
+});
+
+// The comparison a ranked list actually has to win: naive retrieval on the same corpus.
+// Each baseline is reported at its STRONGEST candidate policy — pointing a baseline at every
+// scanned file makes it rank READMEs and turns the comparison into a strawman.
+type BaselineSuite = typeof heldoutBaseline;
+type BaselineFamily = keyof BaselineSuite["configuration"]["bestPolicyPerFamily"];
+type BaselineArm = keyof BaselineSuite["arms"];
+
+const strongestBaseline = (suite: BaselineSuite, family: BaselineFamily) => {
+  const policy = suite.configuration.bestPolicyPerFamily[family];
+  const arm = suite.arms[`${family}:${policy}` as BaselineArm];
+  if (!arm) {
+    throw new Error(`Recorded baseline results do not contain ${family}:${policy}.`);
+  }
+  return arm.unmentioned;
+};
+
+const baselineOf = (suite: BaselineSuite) => ({
+  cases: suite.arms.fixmap.unmentioned.cases,
+  fixmap: suite.arms.fixmap.unmentioned,
+  bm25: strongestBaseline(suite, "bm25"),
+  lexical: strongestBaseline(suite, "lexical-literal"),
+  pathExtraction: strongestBaseline(suite, "path-extraction")
+});
+
 export const siteStats = {
   version: cli.version,
   medianSeconds: (savings.performance.medianScanAndRankMs / 1000).toFixed(2),
@@ -28,6 +64,7 @@ export const siteStats = {
     top3: hits(heldoutResults, "top3"),
     top5: hits(heldoutResults, "top5Hit"),
     intervals95: heldout.intervals95,
+    cohorts: cohortOf(heldout),
     results: heldout.results
   },
   regression: {
@@ -36,7 +73,12 @@ export const siteStats = {
     top3: hits(regressionResults, "top3"),
     top5: hits(regressionResults, "top5Hit"),
     intervals95: regression.intervals95,
+    cohorts: cohortOf(regression),
     results: regression.results
+  },
+  baselines: {
+    heldout: baselineOf(heldoutBaseline),
+    regression: baselineOf(regressionBaseline)
   }
 };