From c9e61bb9b5507a5bd70fe6b9280a2d3f4a2717f9 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:00:46 +0200 Subject: [PATCH 1/3] feat(evals): expand mutation-tested coverage --- evals/benchmark.ts | 18 +++ evals/benchmarks.ts | 223 +++++++++++++++++++++++++++++- tests/benchmark-reporting.test.ts | 67 +++++++++ 3 files changed, 305 insertions(+), 3 deletions(-) diff --git a/evals/benchmark.ts b/evals/benchmark.ts index ab9fd20..e3691cf 100644 --- a/evals/benchmark.ts +++ b/evals/benchmark.ts @@ -7,12 +7,30 @@ export type BenchmarkGrade = { readonly issues: readonly string[]; }; +export type BenchmarkContaminationNotes = { + readonly schemaVersion: 1; + readonly public: readonly string[]; + readonly withheld: readonly string[]; +}; + +export type BenchmarkKnownBadMutation = { + readonly id: string; + readonly fileOverrides: Readonly>; +}; + +export type BenchmarkOracleMetadata = { + readonly schemaVersion: 1; + readonly contamination: BenchmarkContaminationNotes; + readonly knownBadMutations: readonly BenchmarkKnownBadMutation[]; +}; + /** One task whose result can be graded without trusting model-written tests. */ export type BenchmarkCase = { readonly id: string; readonly description: string; readonly files: Readonly>; readonly prompt: string; + readonly oracle: BenchmarkOracleMetadata; readonly grade: (project: string) => Promise; }; diff --git a/evals/benchmarks.ts b/evals/benchmarks.ts index 600ef74..033f0d3 100644 --- a/evals/benchmarks.ts +++ b/evals/benchmarks.ts @@ -1,7 +1,11 @@ import { spawnSync } from "node:child_process"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; -import type { BenchmarkCase, BenchmarkGrade } from "./benchmark.js"; +import type { + BenchmarkCase, + BenchmarkGrade, + BenchmarkOracleMetadata, +} from "./benchmark.js"; const BASE_FIXTURE: Record = { "package.json": `${JSON.stringify( @@ -16,6 +20,22 @@ const BASE_FIXTURE: Record = { )}\n`, }; +function oracle( + publicNotes: readonly string[], + withheldNotes: readonly string[], + knownBadMutations: BenchmarkOracleMetadata["knownBadMutations"], +): BenchmarkOracleMetadata { + return { + schemaVersion: 1, + contamination: { + schemaVersion: 1, + public: publicNotes, + withheld: withheldNotes, + }, + knownBadMutations, + }; +} + function hiddenBunCheck(project: string, source: string): BenchmarkGrade { const result = spawnSync("bun", ["-e", source], { cwd: project, @@ -51,6 +71,28 @@ export const BENCHMARK_CASES: readonly BenchmarkCase[] = [ }, prompt: "Add an exported `farewell(name)` function that returns exactly `Goodbye, !`, export it from src/index.ts, and add an appropriate focused test. Implement and validate the change end to end; you have my approval.", + oracle: oracle( + [ + "The starter files, export name, and exact farewell examples are public.", + ], + ["The executable boundary checks and mutation cases are withheld."], + [ + { + id: "wrong-return-value", + fileOverrides: { + "src/greet.ts": + 'export function greet(name: string) { return "Hello, " + name + "!"; }\nexport function farewell(name: string) { return "Hello, " + name + "!"; }\n', + "src/index.ts": 'export { greet, farewell } from "./greet.js";\n', + }, + }, + { + id: "missing-public-export", + fileOverrides: { + "src/index.ts": 'export { greet } from "./greet.js";\n', + }, + }, + ], + ), async grade(project) { return hiddenBunCheck( project, @@ -71,6 +113,28 @@ export const BENCHMARK_CASES: readonly BenchmarkCase[] = [ }, prompt: "Add an exported `slugPath(dir, title)` to src/slug.ts that returns `/.md`. A title carrying punctuation, such as `Q1: Report/Draft`, must produce exactly one path separator and no Windows-illegal filename character. You may repair the existing slug implementation. Add focused tests, implement, and validate end to end; you have my approval.", + oracle: oracle( + [ + "The starter slug helper, public function name, and punctuation example are public.", + ], + ["The executable path-shape assertions and mutation cases are withheld."], + [ + { + id: "preserves-punctuation", + fileOverrides: { + "src/slug.ts": + 'export function slug(title: string) { return title.toLowerCase().split(" ").join("-"); }\nexport function slugPath(dir: string, title: string) { return dir + "/" + slug(title) + ".md"; }\n', + }, + }, + { + id: "duplicates-separator", + fileOverrides: { + "src/slug.ts": + 'export function slug(title: string) { return title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); }\nexport function slugPath(dir: string, title: string) { return dir + "//" + slug(title) + ".md"; }\n', + }, + }, + ], + ), async grade(project) { return hiddenBunCheck( project, @@ -91,18 +155,171 @@ export const BENCHMARK_CASES: readonly BenchmarkCase[] = [ }, prompt: "Add an exported `parseHeader(line)` to src/headers.ts returning `{ name, value }`. Header names must be lowercase, surrounding whitespace trimmed, values may contain additional colons, malformed lines with no colon must throw, and the existing `headerValue` behavior must remain compatible. Add focused tests, implement, and validate end to end; you have my approval.", + oracle: oracle( + [ + "The existing headerValue behavior and requested parseHeader contract are public.", + ], + [ + "The executable colon, casing, and malformed-input checks are withheld.", + ], + [ + { + id: "keeps-header-case", + fileOverrides: { + "src/headers.ts": + 'export function headerValue(line: string) { return line.split(":")[1]?.trim() ?? ""; }\nexport function parseHeader(line: string) { const at = line.indexOf(":"); if (at < 0) throw new Error("malformed"); return { name: line.slice(0, at).trim(), value: line.slice(at + 1).trim() }; }\n', + }, + }, + { + id: "drops-extra-colons", + fileOverrides: { + "src/headers.ts": + 'export function headerValue(line: string) { return line.split(":")[1]?.trim() ?? ""; }\nexport function parseHeader(line: string) { const parts = line.split(":"); if (parts.length < 2) throw new Error("malformed"); return { name: parts[0].trim().toLowerCase(), value: parts[1].trim() }; }\n', + }, + }, + ], + ), async grade(project) { const source = await readFile( join(project, "src", "headers.ts"), "utf8", ).catch(() => ""); - if (!source.includes("parseHeader")) { + if (!source.includes("parseHeader")) return { passed: false, issues: ["parseHeader was not implemented"] }; - } return hiddenBunCheck( project, 'import { headerValue, parseHeader } from "./src/headers.ts"; const parsed = parseHeader(" X-Trace : one:two "); if (parsed.name !== "x-trace" || parsed.value !== "one:two" || headerValue("Accept: text/plain") !== "text/plain") process.exit(1); let threw = false; try { parseHeader("invalid") } catch { threw = true } if (!threw) process.exit(1);', ); }, }, + { + id: "order-summary-report", + description: + "adds a multi-file order summary while preserving an existing money formatter", + files: { + ...BASE_FIXTURE, + "src/orders.ts": + "export type OrderLine = { id: string; unitCents: number; quantity: number };\nexport function orderTotal(line: OrderLine): number { return line.unitCents * line.quantity; }\n", + "src/report.ts": + 'export function formatCents(cents: number): string { return String(cents) + " cents"; }\n', + "src/index.ts": + 'export { orderTotal, type OrderLine } from "./orders.js";\nexport { formatCents } from "./report.js";\n', + }, + prompt: + "Add `summarizeOrders(lines)` to src/orders.ts and `renderOrderSummary(lines)` to src/report.ts, exporting both from src/index.ts. Summaries must count input lines, count distinct order ids, total unitCents multiplied by quantity, and floor the average total per order; an empty list returns zeros. Render the summary as ` orders / cents`. Preserve orderTotal and formatCents, add focused tests, and validate end to end; you have my approval.", + oracle: oracle( + [ + "The starter modules, public function names, arithmetic rules, and output format are public.", + ], + [ + "The executable duplicate-id, empty-input, and rounding checks are withheld.", + ], + [ + { + id: "counts-lines-as-orders", + fileOverrides: { + "src/orders.ts": + "export type OrderLine = { id: string; unitCents: number; quantity: number };\nexport function orderTotal(line: OrderLine) { return line.unitCents * line.quantity; }\nexport function summarizeOrders(lines: readonly OrderLine[]) { const totalCents = lines.reduce((sum, line) => sum + orderTotal(line), 0); return { lineCount: lines.length, orderCount: lines.length, totalCents, averageOrderCents: lines.length ? Math.floor(totalCents / lines.length) : 0 }; }\n", + "src/report.ts": + 'import { summarizeOrders, type OrderLine } from "./orders.js"; export function formatCents(cents: number) { return String(cents) + " cents"; } export function renderOrderSummary(lines: readonly OrderLine[]) { const value = summarizeOrders(lines); return String(value.orderCount) + " orders / " + String(value.totalCents) + " cents"; }\n', + "src/index.ts": + 'export { orderTotal, summarizeOrders, type OrderLine } from "./orders.js";\nexport { formatCents, renderOrderSummary } from "./report.js";\n', + }, + }, + { + id: "rounds-average-up", + fileOverrides: { + "src/orders.ts": + "export type OrderLine = { id: string; unitCents: number; quantity: number };\nexport function orderTotal(line: OrderLine) { return line.unitCents * line.quantity; }\nexport function summarizeOrders(lines: readonly OrderLine[]) { const totalCents = lines.reduce((sum, line) => sum + orderTotal(line), 0); const ids = new Set(lines.map((line) => line.id)); return { lineCount: lines.length, orderCount: ids.size, totalCents, averageOrderCents: Math.ceil(totalCents / ids.size) }; }\n", + "src/report.ts": + 'import { summarizeOrders, type OrderLine } from "./orders.js"; export function formatCents(cents: number) { return String(cents) + " cents"; } export function renderOrderSummary(lines: readonly OrderLine[]) { const value = summarizeOrders(lines); return String(value.orderCount) + " orders / " + String(value.totalCents) + " cents"; }\n', + "src/index.ts": + 'export { orderTotal, summarizeOrders, type OrderLine } from "./orders.js";\nexport { formatCents, renderOrderSummary } from "./report.js";\n', + }, + }, + { + id: "breaks-format-cents", + fileOverrides: { + "src/orders.ts": + "export type OrderLine = { id: string; unitCents: number; quantity: number };\nexport function orderTotal(line: OrderLine) { return line.unitCents * line.quantity; }\nexport function summarizeOrders(lines: readonly OrderLine[]) { const ids = new Set(lines.map((line) => line.id)); const totalCents = lines.reduce((sum, line) => sum + orderTotal(line), 0); return { lineCount: lines.length, orderCount: ids.size, totalCents, averageOrderCents: ids.size ? Math.floor(totalCents / ids.size) : 0 }; }\n", + "src/report.ts": + 'import { summarizeOrders, type OrderLine } from "./orders.js"; export function formatCents(_cents: number) { return "BROKEN"; } export function renderOrderSummary(lines: readonly OrderLine[]) { const value = summarizeOrders(lines); return String(value.orderCount) + " orders / " + String(value.totalCents) + " cents"; }\n', + "src/index.ts": + 'export { orderTotal, summarizeOrders, type OrderLine } from "./orders.js";\nexport { formatCents, renderOrderSummary } from "./report.js";\n', + }, + }, + ], + ), + async grade(project) { + return hiddenBunCheck( + project, + 'import { formatCents, orderTotal, renderOrderSummary, summarizeOrders } from "./src/index.ts"; const lines = [{ id: "A", unitCents: 125, quantity: 2 }, { id: "A", unitCents: 50, quantity: 1 }, { id: "B", unitCents: 201, quantity: 1 }]; const value = summarizeOrders(lines); if (orderTotal(lines[0]) !== 250 || formatCents(501) !== "501 cents" || JSON.stringify(value) !== JSON.stringify({ lineCount: 3, orderCount: 2, totalCents: 501, averageOrderCents: 250 }) || renderOrderSummary(lines) !== "2 orders / 501 cents") process.exit(1); if (JSON.stringify(summarizeOrders([])) !== JSON.stringify({ lineCount: 0, orderCount: 0, totalCents: 0, averageOrderCents: 0 })) process.exit(1);', + ); + }, + }, + { + id: "markdown-link-report", + description: + "adds a multi-file Markdown link index with stable line-level reporting", + files: { + ...BASE_FIXTURE, + "src/markdown.ts": + "export function markdownLines(markdown: string): readonly string[] { return markdown.split(/\\r?\\n/); }\n", + "src/link-report.ts": + 'export function formatLinkCount(count: number): string { return String(count) + " links"; }\n', + "src/index.ts": + 'export { markdownLines } from "./markdown.js";\nexport { formatLinkCount } from "./link-report.js";\n', + }, + prompt: + "Add `summarizeLinks(markdown)` to src/markdown.ts and `renderLinkReport(markdown)` to src/link-report.ts, exporting both from src/index.ts. Count Markdown inline links of the form `[label](url)`, report total links, distinct URL strings, and a `byLine` object keyed by 1-based line number. Render ` links across lines`. Preserve markdownLines and formatLinkCount, add focused tests, and validate end to end; you have my approval.", + oracle: oracle( + [ + "The starter modules, inline-link syntax, public function names, and line numbering are public.", + ], + [ + "The executable duplicate-URL, line-map, and empty-document checks are withheld.", + ], + [ + { + id: "counts-duplicate-as-unique", + fileOverrides: { + "src/markdown.ts": + 'export function markdownLines(markdown: string) { return markdown.split(/\\r?\\n/); }\nexport function summarizeLinks(markdown: string) { const matches = [...markdown.matchAll(/\\[[^\\]]+\\]\\(([^)]+)\\)/g)]; const byLine: Record = {}; for (const _match of matches) byLine["1"] = (byLine["1"] ?? 0) + 1; return { links: matches.length, uniqueUrls: matches.length, byLine }; }\n', + "src/link-report.ts": + 'import { summarizeLinks } from "./markdown.js"; export function formatLinkCount(count: number) { return String(count) + " links"; } export function renderLinkReport(markdown: string) { const value = summarizeLinks(markdown); return String(value.links) + " links across " + String(markdown.split(/\\r?\\n/).length) + " lines"; }\n', + "src/index.ts": + 'export { markdownLines, summarizeLinks } from "./markdown.js";\nexport { formatLinkCount, renderLinkReport } from "./link-report.js";\n', + }, + }, + { + id: "uses-zero-based-lines", + fileOverrides: { + "src/markdown.ts": + "export function markdownLines(markdown: string) { return markdown.split(/\\r?\\n/); }\nexport function summarizeLinks(markdown: string) { const byLine: Record = {}; for (const [index, line] of markdownLines(markdown).entries()) { const count = [...line.matchAll(/\\[[^\\]]+\\]\\(([^)]+)\\)/g)].length; if (count) byLine[String(index)] = count; } return { links: Object.values(byLine).reduce((sum, count) => sum + count, 0), uniqueUrls: 2, byLine }; }\n", + "src/link-report.ts": + 'import { summarizeLinks } from "./markdown.js"; export function formatLinkCount(count: number) { return String(count) + " links"; } export function renderLinkReport(markdown: string) { const value = summarizeLinks(markdown); return String(value.links) + " links across " + String(markdown.split(/\\r?\\n/).length) + " lines"; }\n', + "src/index.ts": + 'export { markdownLines, summarizeLinks } from "./markdown.js";\nexport { formatLinkCount, renderLinkReport } from "./link-report.js";\n', + }, + }, + { + id: "breaks-format-link-count", + fileOverrides: { + "src/markdown.ts": + 'export function markdownLines(markdown: string) { return markdown.split(/\\r?\\n/); }\nexport function summarizeLinks(markdown: string) { const byLine: Record = {}; const urls = new Set(); let links = 0; for (const [index, line] of markdownLines(markdown).entries()) { for (const match of line.matchAll(/\\[[^\\]]+\\]\\(([^)]+)\\)/g)) { links += 1; urls.add(match[1] ?? ""); byLine[String(index + 1)] = (byLine[String(index + 1)] ?? 0) + 1; } } return { links, uniqueUrls: urls.size, byLine }; }\n', + "src/link-report.ts": + 'import { summarizeLinks } from "./markdown.js"; export function formatLinkCount(_count: number) { return "BROKEN"; } export function renderLinkReport(markdown: string) { const value = summarizeLinks(markdown); return String(value.links) + " links across " + String(markdown.split(/\\r?\\n/).length) + " lines"; }\n', + "src/index.ts": + 'export { markdownLines, summarizeLinks } from "./markdown.js";\nexport { formatLinkCount, renderLinkReport } from "./link-report.js";\n', + }, + }, + ], + ), + async grade(project) { + return hiddenBunCheck( + project, + 'import { formatLinkCount, markdownLines, renderLinkReport, summarizeLinks } from "./src/index.ts"; const markdown = "# Guide\\n[Home](/home) and [Docs](/docs)\\n[Home](/home)\\nplain"; const value = summarizeLinks(markdown); if (JSON.stringify(markdownLines(markdown)) !== JSON.stringify(["# Guide", "[Home](/home) and [Docs](/docs)", "[Home](/home)", "plain"]) || formatLinkCount(3) !== "3 links" || JSON.stringify(value) !== JSON.stringify({ links: 3, uniqueUrls: 2, byLine: { "2": 2, "3": 1 } }) || renderLinkReport(markdown) !== "3 links across 4 lines") process.exit(1); if (JSON.stringify(summarizeLinks("")) !== JSON.stringify({ links: 0, uniqueUrls: 0, byLine: {} })) process.exit(1);', + ); + }, + }, ]; diff --git a/tests/benchmark-reporting.test.ts b/tests/benchmark-reporting.test.ts index b383471..7e494c6 100644 --- a/tests/benchmark-reporting.test.ts +++ b/tests/benchmark-reporting.test.ts @@ -151,6 +151,22 @@ describe("hidden benchmark graders", () => { "src/headers.ts": 'export function headerValue(line: string) { return line.split(":")[1]?.trim() ?? ""; }\nexport function parseHeader(line: string) { const at = line.indexOf(":"); if (at < 0) throw new Error("malformed"); return { name: line.slice(0, at).trim().toLowerCase(), value: line.slice(at + 1).trim() }; }\n', }, + "order-summary-report": { + "src/orders.ts": + "export type OrderLine = { id: string; unitCents: number; quantity: number };\nexport function orderTotal(line: OrderLine) { return line.unitCents * line.quantity; }\nexport function summarizeOrders(lines: readonly OrderLine[]) { const ids = new Set(lines.map((line) => line.id)); const totalCents = lines.reduce((sum, line) => sum + orderTotal(line), 0); return { lineCount: lines.length, orderCount: ids.size, totalCents, averageOrderCents: ids.size ? Math.floor(totalCents / ids.size) : 0 }; }\n", + "src/report.ts": + 'import { summarizeOrders, type OrderLine } from "./orders.js";\nexport function formatCents(cents: number) { return String(cents) + " cents"; }\nexport function renderOrderSummary(lines: readonly OrderLine[]) { const value = summarizeOrders(lines); return String(value.orderCount) + " orders / " + String(value.totalCents) + " cents"; }\n', + "src/index.ts": + 'export { orderTotal, summarizeOrders, type OrderLine } from "./orders.js";\nexport { formatCents, renderOrderSummary } from "./report.js";\n', + }, + "markdown-link-report": { + "src/markdown.ts": + 'export function markdownLines(markdown: string) { return markdown.split(/\\r?\\n/); }\nexport function summarizeLinks(markdown: string) { const byLine: Record = {}; const urls = new Set(); let links = 0; for (const [index, line] of markdownLines(markdown).entries()) { for (const match of line.matchAll(/\\[[^\\]]+\\]\\(([^)]+)\\)/g)) { links += 1; urls.add(match[1] ?? ""); byLine[String(index + 1)] = (byLine[String(index + 1)] ?? 0) + 1; } } return { links, uniqueUrls: urls.size, byLine }; }\n', + "src/link-report.ts": + 'import { summarizeLinks } from "./markdown.js";\nexport function formatLinkCount(count: number) { return String(count) + " links"; }\nexport function renderLinkReport(markdown: string) { const value = summarizeLinks(markdown); return String(value.links) + " links across " + String(markdown.split(/\\r?\\n/).length) + " lines"; }\n', + "src/index.ts": + 'export { markdownLines, summarizeLinks } from "./markdown.js";\nexport { formatLinkCount, renderLinkReport } from "./link-report.js";\n', + }, }; for (const benchmark of BENCHMARK_CASES) { @@ -171,4 +187,55 @@ describe("hidden benchmark graders", () => { } } }); + + test("accepts every declared known-good mutation boundary", () => { + for (const benchmark of BENCHMARK_CASES) { + expect(benchmark.oracle.schemaVersion).toBe(1); + expect(benchmark.oracle.contamination.schemaVersion).toBe(1); + expect(benchmark.oracle.contamination.public.length).toBeGreaterThan(0); + expect(benchmark.oracle.contamination.withheld.length).toBeGreaterThan(0); + const ids = benchmark.oracle.knownBadMutations.map( + (mutation) => mutation.id, + ); + expect(ids.length).toBeGreaterThanOrEqual(2); + expect(new Set(ids).size).toBe(ids.length); + for (const mutation of benchmark.oracle.knownBadMutations) { + expect(mutation.id.length).toBeGreaterThan(0); + expect(Object.keys(mutation.fileOverrides).length).toBeGreaterThan(0); + } + } + }); + + test("keeps evaluation labels and hidden oracle details out of prompts", () => { + const forbidden = + /\b(candidate|baseline|hidden|oracle|grader|evaluation)\b/i; + for (const benchmark of BENCHMARK_CASES) { + expect(benchmark.prompt).not.toMatch(forbidden); + } + }); + + test("rejects every declared known-bad mutation", async () => { + for (const benchmark of BENCHMARK_CASES) { + for (const mutation of benchmark.oracle.knownBadMutations) { + const project = await mkdtemp( + join(tmpdir(), "flow-benchmark-mutation-"), + ); + try { + for (const [relative, contents] of Object.entries({ + ...benchmark.files, + ...mutation.fileOverrides, + })) { + const target = join(project, relative); + await mkdir(join(target, ".."), { recursive: true }); + await writeFile(target, contents, "utf8"); + } + expect(await benchmark.grade(project)).toMatchObject({ + passed: false, + }); + } finally { + await rm(project, { recursive: true, force: true }); + } + } + } + }); }); From c68e408a24a876f0dd712b901ebf11a2d01599a5 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:01:00 +0200 Subject: [PATCH 2/3] feat(evals): compare stable report semantics --- evals/report-render.ts | 327 +++++++++++++++++++++++++++++++++++ tests/report-render.test.ts | 331 ++++++++++++++++++++++++++++++++++++ 2 files changed, 658 insertions(+) create mode 100644 evals/report-render.ts create mode 100644 tests/report-render.test.ts diff --git a/evals/report-render.ts b/evals/report-render.ts new file mode 100644 index 0000000..0e5db2c --- /dev/null +++ b/evals/report-render.ts @@ -0,0 +1,327 @@ +import { analyzePairs, analyzeReviewer } from "./analysis.js"; +import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; +import type { ValidatedCaseCatalog } from "./catalog.js"; +import type { AttemptRecordV2, ValidatedReport } from "./report.js"; + +export type WilsonInterval = readonly [number, number] | null; +export type EvidenceCard = { + readonly caseId: string; + readonly caseVersion: number; + readonly scheduled: number; + readonly attempted: number; + readonly missing: number; + readonly products: number; + readonly passed: number; + readonly failedProducts: number; + readonly operationalFailures: number; + readonly unscored: number; + readonly passRate: number | null; + readonly interval95: WilsonInterval; +}; +export type EvidenceRender = { + readonly cards: readonly EvidenceCard[]; + readonly completion: ValidatedReport["completion"]; + readonly artifacts: readonly AttemptRecordV2["artifact"][]; + readonly evaluatorDigests: readonly string[]; + readonly reviewer: ReturnType; + readonly paired: ReturnType; +}; +export type ComparisonKey = { + readonly sha256: string; + readonly semantics: unknown; +}; +export type CaseDelta = { + readonly caseId: string; + readonly caseVersion: number; + readonly baselineRate: number | null; + readonly candidateRate: number | null; + readonly delta: number | null; +}; +export type ReportComparison = { + readonly compatible: boolean; + readonly reason: string | null; + readonly passDelta: number | null; + readonly cases: readonly CaseDelta[]; +}; + +function wilson(passed: number, total: number): WilsonInterval { + if (total === 0) return null; + const z = 1.959963984540054; + const rate = passed / total; + const square = z * z; + const denominator = 1 + square / total; + const center = (rate + square / (2 * total)) / denominator; + const margin = + (z / denominator) * + Math.sqrt((rate * (1 - rate) + square / (4 * total)) / total); + return [center - margin, center + margin]; +} + +function activeCells(report: ValidatedReport) { + const reserves = new Set(report.completion.activatedReserveCellIds); + return report.plan.cells.filter( + (cell) => + cell.schedule === "primary" || + (cell.schedule === "replacement-reserve" && reserves.has(cell.cellId)), + ); +} + +function caseKey(input: { + readonly caseId: string; + readonly caseVersion: number; +}): string { + return `${input.caseId}\u0000${input.caseVersion}`; +} + +export function renderEvidence(report: ValidatedReport): EvidenceRender { + const cards = new Map(); + for (const cell of activeCells(report)) { + const key = caseKey(cell); + const prior = cards.get(key) ?? { + caseId: cell.caseId, + caseVersion: cell.caseVersion, + scheduled: 0, + attempted: 0, + missing: 0, + products: 0, + passed: 0, + failedProducts: 0, + operationalFailures: 0, + unscored: 0, + passRate: null, + interval95: null, + }; + cards.set(key, { ...prior, scheduled: prior.scheduled + 1 }); + } + for (const attempt of report.attempts) { + const key = caseKey(attempt); + const prior = cards.get(key); + if (!prior) throw new Error("Validated attempt lacks a scheduled case."); + if (attempt.outcome.kind === "product") { + cards.set(key, { + ...prior, + attempted: prior.attempted + 1, + products: prior.products + 1, + passed: prior.passed + Number(attempt.outcome.passed), + failedProducts: prior.failedProducts + Number(!attempt.outcome.passed), + }); + } else if (attempt.outcome.kind === "failure") { + cards.set(key, { + ...prior, + attempted: prior.attempted + 1, + operationalFailures: prior.operationalFailures + 1, + }); + } else { + cards.set(key, { + ...prior, + attempted: prior.attempted + 1, + unscored: prior.unscored + 1, + }); + } + } + const rendered = [...cards.values()] + .map((card) => ({ + ...card, + missing: Math.max(0, card.scheduled - card.attempted), + passRate: card.products === 0 ? null : card.passed / card.products, + interval95: wilson(card.passed, card.products), + })) + .sort( + (left, right) => + left.caseId.localeCompare(right.caseId) || + left.caseVersion - right.caseVersion, + ); + const artifacts = new Map(); + for (const attempt of report.attempts) { + artifacts.set(canonicalJson(attempt.artifact), attempt.artifact); + } + return { + cards: rendered, + completion: report.completion, + artifacts: [...artifacts.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([, artifact]) => artifact), + evaluatorDigests: [ + ...new Set( + report.attempts.map((attempt) => + canonicalSha256("flow-render-evaluator-v1", { + caseCatalogSha256: attempt.evaluator.caseCatalogSha256, + policyCatalogSha256: attempt.evaluator.policyCatalogSha256, + graderBundleSha256: attempt.evaluator.graderBundleSha256, + }), + ), + ), + ].sort(), + reviewer: analyzeReviewer(report), + paired: analyzePairs(report), + }; +} + +function sortedByCanonical(values: readonly T[]): readonly T[] { + return [...values].sort((left, right) => + canonicalJson(left).localeCompare(canonicalJson(right)), + ); +} + +export function comparisonKey(input: { + readonly report: ValidatedReport; + readonly catalog: ValidatedCaseCatalog; +}): ComparisonKey { + const cells = input.report.plan.cells.map((cell) => ({ + caseId: cell.caseId, + caseVersion: cell.caseVersion, + repetition: cell.repetition, + schedule: cell.schedule, + managerModel: cell.managerModel, + reviewerModel: cell.reviewerModel, + })); + const rows = input.report.attempts.map((attempt) => ({ + caseId: attempt.caseId, + caseVersion: attempt.caseVersion, + repetition: attempt.repetition, + schedule: input.report.plan.cells.find( + (cell) => cell.cellId === attempt.cellId, + )?.schedule, + hostConfigSha256: attempt.hostConfigSha256, + actors: sortedByCanonical( + attempt.actors.map((actor) => ({ + role: actor.role, + requestedModel: actor.requestedModel, + })), + ), + instructions: [...attempt.instructions].sort( + (left, right) => left.sequence - right.sequence, + ), + evaluator: { + caseCatalogSha256: attempt.evaluator.caseCatalogSha256, + policyCatalogSha256: attempt.evaluator.policyCatalogSha256, + graderBundleSha256: attempt.evaluator.graderBundleSha256, + }, + })); + const semantics = { + cells: sortedByCanonical(cells), + policy: { + analysis: input.report.plan.analysis, + stoppingRule: input.report.plan.stoppingRule, + abortPolicy: input.report.plan.abortPolicy, + budget: input.report.plan.budget, + }, + catalog: sortedByCanonical( + input.catalog.map((entry) => ({ + caseId: entry.caseId, + caseVersion: entry.caseVersion, + evidenceClass: entry.evidenceClass, + oracle: entry.oracle, + release: entry.release, + minProviders: entry.minProviders, + minScoredAttempts: entry.minScoredAttempts, + minPassRate: entry.minPassRate, + reviewerPromotionRecordSha256: entry.reviewerPromotionRecordSha256, + })), + ), + rows: sortedByCanonical(rows), + }; + return { + semantics, + sha256: canonicalSha256("flow-report-comparison-key-v1", semantics), + }; +} + +function cardRates(report: ValidatedReport): Map { + return new Map( + renderEvidence(report).cards.map((card) => [caseKey(card), card]), + ); +} + +export function compareReports(input: { + readonly baseline: ValidatedReport; + readonly candidate: ValidatedReport; + readonly baselineCatalog: ValidatedCaseCatalog; + readonly candidateCatalog: ValidatedCaseCatalog; +}): ReportComparison { + if ( + input.baseline.completion.status !== "complete" || + input.candidate.completion.status !== "complete" + ) { + return { + compatible: false, + reason: "Comparable trends require complete reports.", + passDelta: null, + cases: [], + }; + } + const baselineKey = comparisonKey({ + report: input.baseline, + catalog: input.baselineCatalog, + }); + const candidateKey = comparisonKey({ + report: input.candidate, + catalog: input.candidateCatalog, + }); + if (baselineKey.sha256 !== candidateKey.sha256) { + return { + compatible: false, + reason: "Comparison semantics differ.", + passDelta: null, + cases: [], + }; + } + const baselineCards = cardRates(input.baseline); + const candidateCards = cardRates(input.candidate); + const cases = [...baselineCards.entries()].map(([key, baseline]) => { + const candidate = candidateCards.get(key); + if (!candidate) throw new Error("Compatible report lost a case card."); + return { + caseId: baseline.caseId, + caseVersion: baseline.caseVersion, + baselineRate: baseline.passRate, + candidateRate: candidate.passRate, + delta: + baseline.passRate === null || candidate.passRate === null + ? null + : candidate.passRate - baseline.passRate, + }; + }); + const totals = (cards: ReadonlyMap) => { + const values = [...cards.values()]; + const products = values.reduce((sum, card) => sum + card.products, 0); + const passed = values.reduce((sum, card) => sum + card.passed, 0); + return products === 0 ? null : passed / products; + }; + const baselineRate = totals(baselineCards); + const candidateRate = totals(candidateCards); + return { + compatible: true, + reason: null, + passDelta: + baselineRate === null || candidateRate === null + ? null + : candidateRate - baselineRate, + cases, + }; +} + +export function compareTrend( + points: readonly { + readonly report: ValidatedReport; + readonly catalog: ValidatedCaseCatalog; + }[], +): { + readonly compatible: boolean; + readonly comparisons: readonly ReportComparison[]; +} { + const comparisons = points.slice(1).map((point, index) => { + const prior = points[index]; + if (!prior) throw new Error("Trend predecessor is missing."); + return compareReports({ + baseline: prior.report, + candidate: point.report, + baselineCatalog: prior.catalog, + candidateCatalog: point.catalog, + }); + }); + return { + compatible: comparisons.every((comparison) => comparison.compatible), + comparisons, + }; +} diff --git a/tests/report-render.test.ts b/tests/report-render.test.ts new file mode 100644 index 0000000..80210b5 --- /dev/null +++ b/tests/report-render.test.ts @@ -0,0 +1,331 @@ +import { describe, expect, test } from "bun:test"; +import { + parseCaseCatalog, + type ValidatedCaseCatalog, +} from "../evals/catalog.js"; +import { + campaignPlanSha256, + type EvalReportV2, + EvalReportV2Schema, + parseReport, + type ValidatedReport, +} from "../evals/report.js"; +import { + compareReports, + compareTrend, + comparisonKey, + renderEvidence, +} from "../evals/report-render.js"; + +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; +type Fixture = { + readonly report: ValidatedReport; + readonly catalog: ValidatedCaseCatalog; +}; + +function fixture( + input: { + readonly pass?: boolean; + readonly artifact?: string; + readonly artifactCommit?: string; + readonly evaluatorCommit?: string; + readonly evaluatorDigest?: string; + readonly host?: string; + readonly model?: string; + readonly instruction?: string; + readonly caseVersion?: number; + readonly policyVersion?: string; + readonly oracle?: "durable-state" | "hidden-executable" | "trajectory"; + } = {}, +): Fixture { + const caseVersion = input.caseVersion ?? 1; + const model = { + routeProvider: "provider", + gateway: null, + family: "family", + model: input.model ?? "model", + revision: null, + }; + const cell = { + cellId: "cell", + blockId: "block", + caseId: "case", + caseVersion, + armToken: null, + repetition: 0, + managerModel: model, + reviewerModel: null, + schedule: "primary" as const, + }; + const plan: EvalReportV2["plan"] = { + schemaVersion: 1, + planId: "plan", + planSha256: digest("0"), + randomizationSeed: "seed", + cells: [cell], + abortPolicy: { retry: "never", maxReplacementBlocks: 0 }, + stoppingRule: { kind: "fixed-attempts", count: 1 }, + analysis: { + kind: "rate", + primaryOutcome: "correctness", + versionSha256: input.policyVersion ?? digest("1"), + }, + budget: { + maxUsd: 1, + unknownCostPolicy: "stop", + maxOutputTokens: 100, + maxWallClockMs: 1_000, + maxAttempts: 1, + }, + }; + plan.planSha256 = campaignPlanSha256(plan); + const pass = input.pass ?? true; + const attempt: EvalReportV2["attempts"][number] = { + schemaVersion: 2, + attemptId: "attempt", + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion, + armToken: null, + repetition: 0, + artifact: { + packageVersion: "1.0.0", + sourceCommit: input.artifactCommit ?? "artifact-commit", + sourceTreeSha256: digest(input.artifact ?? "a"), + tarballSha256: digest(input.artifact ?? "a"), + unpackedManifestSha256: digest(input.artifact ?? "a"), + }, + evaluator: { + sourceCommit: input.evaluatorCommit ?? "evaluator-commit", + caseCatalogSha256: digest("2"), + policyCatalogSha256: digest("3"), + graderBundleSha256: input.evaluatorDigest ?? digest("4"), + }, + hostConfigSha256: input.host ?? digest("5"), + actors: [ + { + role: "manager", + requestedModel: model, + actualModel: { kind: "observed", value: model }, + sessionIds: ["session"], + }, + ], + instructions: [ + { + source: "command", + name: "task", + sequence: 0, + sha256: input.instruction ?? digest("6"), + bytes: 4, + }, + ], + transcript: { sha256: digest("7"), artifact: "transcript.json" }, + outcome: { + kind: "product", + passed: pass, + endedBy: "quiet", + issues: pass ? [] : ["failed"], + evidence: { + kind: "conformance", + falseCompletion: false, + unsubmittedReviews: 0, + facts: {}, + }, + }, + usage: { durationMs: 1, outputTokens: 1, costUsd: 0.01 }, + }; + const catalog = parseCaseCatalog([ + { + caseId: "case", + caseVersion, + evidenceClass: "conformance", + oracle: input.oracle ?? "durable-state", + release: "report-only", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + }, + ]); + if (!catalog.ok) throw new Error("Catalog fixture failed."); + const raw: EvalReportV2 = { + schemaVersion: 2, + reportId: "report", + plan, + attempts: [attempt], + completion: { + status: "complete", + cause: "fixed-target", + startedAt: "2026-08-25T00:00:00.000Z", + finishedAt: "2026-08-25T00:00:00.001Z", + activatedReserveCellIds: [], + observed: { + attempts: 1, + outputTokens: 1, + costUsd: 0.01, + wallClockMs: 1, + }, + }, + allocationCommitmentSha256: null, + }; + const report = parseReport(raw, catalog.value); + if (!report.ok) throw new Error(JSON.stringify(report.issues)); + return { report: report.value, catalog: catalog.value }; +} + +function stoppedCoverageFixture(): Fixture { + const base = fixture({ pass: false }); + const raw = EvalReportV2Schema.parse(base.report); + const firstCell = raw.plan.cells[0]; + if (!firstCell) throw new Error("Expected comparison fixture cell."); + raw.plan.cells.push( + ...[1, 2].map((index) => ({ + ...firstCell, + cellId: `cell-${index}`, + blockId: `block-${index}`, + repetition: index, + })), + ); + raw.plan.stoppingRule.count = 3; + raw.plan.budget.maxAttempts = 3; + raw.plan.planSha256 = campaignPlanSha256(raw.plan); + const firstAttempt = raw.attempts[0]; + if (!firstAttempt) throw new Error("Expected comparison fixture attempt."); + raw.attempts.push({ + ...firstAttempt, + attemptId: "attempt-1", + cellId: "cell-1", + blockId: "block-1", + repetition: 1, + actors: [], + instructions: [], + transcript: null, + outcome: { + kind: "failure", + origin: "host", + code: "host-failure", + retryable: true, + }, + usage: { durationMs: 1, outputTokens: 0, costUsd: 0.01 }, + }); + raw.completion = { + ...raw.completion, + status: "stopped", + cause: "host", + observed: { + attempts: 2, + outputTokens: 1, + costUsd: 0.02, + wallClockMs: 1, + }, + }; + const parsed = parseReport(raw, base.catalog); + if (!parsed.ok) throw new Error(JSON.stringify(parsed.issues)); + return { report: parsed.value, catalog: base.catalog }; +} + +function compare(baseline: Fixture, candidate: Fixture) { + return compareReports({ + baseline: baseline.report, + candidate: candidate.report, + baselineCatalog: baseline.catalog, + candidateCatalog: candidate.catalog, + }); +} + +describe("report evidence cards", () => { + test("renders planned coverage, failures, missing rows, and Wilson intervals", () => { + const rendered = renderEvidence(stoppedCoverageFixture().report); + expect(rendered.cards).toEqual([ + expect.objectContaining({ + caseId: "case", + scheduled: 3, + attempted: 2, + missing: 1, + products: 1, + passed: 0, + failedProducts: 1, + operationalFailures: 1, + unscored: 0, + passRate: 0, + }), + ]); + expect(rendered.cards[0]?.interval95).not.toBeNull(); + expect(rendered.artifacts).toHaveLength(1); + expect(rendered.evaluatorDigests).toHaveLength(1); + }); +}); + +describe("report comparison compatibility", () => { + test("allows artifact and evaluator source commits to differ", () => { + const baseline = fixture(); + const candidate = fixture({ + artifact: "b", + artifactCommit: "new-artifact", + evaluatorCommit: "new-evaluator-source", + pass: false, + }); + expect(compare(baseline, candidate)).toMatchObject({ + compatible: true, + passDelta: -1, + cases: [{ caseId: "case", delta: -1 }], + }); + }); + + test("rejects changed case versions", () => { + expect(compare(fixture(), fixture({ caseVersion: 2 })).compatible).toBe( + false, + ); + }); + + test("rejects changed analysis policy versions", () => { + expect( + compare(fixture(), fixture({ policyVersion: digest("9") })).compatible, + ).toBe(false); + }); + + test("rejects changed catalog oracle semantics", () => { + expect( + compare(fixture(), fixture({ oracle: "trajectory" })).compatible, + ).toBe(false); + }); + + test("rejects changed evaluator grader digests", () => { + expect( + compare(fixture(), fixture({ evaluatorDigest: digest("9") })).compatible, + ).toBe(false); + }); + + test("rejects changed host semantics", () => { + expect(compare(fixture(), fixture({ host: digest("9") })).compatible).toBe( + false, + ); + }); + + test("rejects changed requested actors or delivered instructions", () => { + expect(compare(fixture(), fixture({ model: "other" })).compatible).toBe( + false, + ); + expect( + compare(fixture(), fixture({ instruction: digest("9") })).compatible, + ).toBe(false); + }); + + test("breaks a longitudinal chain at the first semantic drift", () => { + const first = fixture(); + const second = fixture({ artifact: "b" }); + const drifted = fixture({ host: digest("9") }); + const trend = compareTrend([first, second, drifted]); + expect(trend.compatible).toBe(false); + expect(trend.comparisons.map((item) => item.compatible)).toEqual([ + true, + false, + ]); + }); + + test("produces a stable compatibility key", () => { + const value = fixture(); + expect(comparisonKey(value)).toEqual(comparisonKey(value)); + }); +}); From 9bbdc68a45d157f2a412383857d77a4a49b8ad1d Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:01:20 +0200 Subject: [PATCH 3/3] docs(evals): record phase 8 evidence --- .../evidence/phase-8-review.md | 33 +++++++++++++++++++ .../phase-8-promotion-trends.md | 9 +++++ .audit/eval-engineering.tsv | 4 +++ 3 files changed, 46 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-8-review.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-8-review.md b/.agents/plans/02-eval-engineering/evidence/phase-8-review.md new file mode 100644 index 0000000..7a1a047 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-8-review.md @@ -0,0 +1,33 @@ +# Phase 8 Interrogate review and trend proof + +Phase 8 adds structured evidence cards and a canonical compatibility key for +longitudinal comparisons. Cards render planned, attempted, missing, product, +failed-product, operational-failure, and unscored counts with Wilson intervals, +completion state, distinct artifacts, evaluator digests, and reviewer or paired +projections. + +The compatibility key treats candidate artifact identity and evaluator source +commit as the intended treatment axis. It freezes normalized case/version/ +repetition/schedule/model cells, analysis/stopping/abort/budget policy, catalog +oracle and release semantics, evaluator case/policy/grader digests, host +configuration, requested actors, and delivered instruction hashes. Reports must +be complete. Per-case and aggregate deltas are emitted only when the key matches; +a longitudinal chain breaks at the first incompatible adjacent report. + +Strictly parsed fixtures prove that equal semantics with different artifact and +evaluator source commits compare. Separate mutations to case version, analysis +version, oracle, evaluator grader digest, host configuration, requested model, and +instruction digest all refuse comparison. + +Benchmark coverage grows from three to five cases. The two new tasks require +coordinated changes across multiple source files. All five cases carry versioned +public/withheld contamination notes and at least two executable known-bad +mutations. The hidden graders reject all twelve mutations and pristine fixtures, +while known-good +implementations pass. Model-facing prompts contain no evaluation, arm, oracle, or +grader labels. + +All benchmark cases remain report-only. No historical report is backfilled and no +new release regression threshold is claimed. The full repository gate passes 509 +tests with one intentional live-smoke skip. The final four-model review found no +unresolved blocker. diff --git a/.agents/plans/02-eval-engineering/phase-8-promotion-trends.md b/.agents/plans/02-eval-engineering/phase-8-promotion-trends.md index 7afea3a..a0041bc 100644 --- a/.agents/plans/02-eval-engineering/phase-8-promotion-trends.md +++ b/.agents/plans/02-eval-engineering/phase-8-promotion-trends.md @@ -31,3 +31,12 @@ change one compatibility-key field and prove comparison refuses. Stop gate. Legacy reports are never backfilled. A case becomes a release regression only after its calibration predicate is recorded. + +## Outcome + +Implemented and verified. Five benchmark cases now carry versioned contamination +notes and twelve executable mutation controls. Structured cards expose missing and +failed evidence. Strict parsed-report comparisons allow different candidate +artifacts while refusing case, policy, oracle, evaluator, host, actor, or +instruction drift. Every case remains report-only and no legacy report is +backfilled. See `evidence/phase-8-review.md`. diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index 299a21f..3bb7e32 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -54,3 +54,7 @@ ts phase decision why evidence result 2026-08-25T08:28:46Z phase-7 fixed multi-model Interrogate findings count-only scans, self-hashed fabricated masks, early recovery-link removal, startup leaks, and permissive reveal weakened evidence integrity .agents/plans/02-eval-engineering/evidence/phase-7-review.md VERIFIED exact scan multiset, semantic mask recomputation, durable link order, and strict reveal 2026-08-25T08:28:46Z phase-7 ran the accepted bounded paired pilot the runner needs real packed-host evidence while refusing an underpowered product claim .agents/plans/02-eval-engineering/evidence/phase-7-pilot.json VERIFIED 1 tie, 0 unresolved, exact clean scans, $0.2450722 of $1, INCONCLUSIVE on power 2026-08-25T08:28:46Z phase-7 ran Deslop and the full repository gate the phase must finish reviewable and regression-free bun run check VERIFIED 496 pass, 1 skip, 0 fail +2026-08-25T08:55:41Z phase-8 added stable-semantics report cards and longitudinal comparison artifact changes are the treatment axis; case, policy, oracle, evaluator, host, actor, and instruction changes invalidate a trend evals/report-render.ts; tests/report-render.test.ts VERIFIED parsed-report compatibility and drift matrix green +2026-08-25T08:55:41Z phase-8 expanded mutation-tested hidden coverage new tasks need executable controls and explicit contamination boundaries before producing useful evidence evals/benchmarks.ts; tests/benchmark-reporting.test.ts VERIFIED 5 cases, 12 rejected mutations, known-good implementations pass +2026-08-25T08:55:41Z phase-8 kept coverage promotion closed uncalibrated cases cannot silently become release regressions evals/benchmark-run.ts catalog policy; .agents/plans/02-eval-engineering/evidence/phase-8-review.md VERIFIED every benchmark case remains report-only; no legacy backfill +2026-08-25T08:55:41Z phase-8 ran Deslop, four-model Interrogate, and full repository gate trend and coverage changes must remain reviewable and regression-free bun run check VERIFIED 509 pass, 1 skip, 0 fail; no unresolved blocker