From 9f4d3ce718dac95bee69856baa974913b9a5d6a4 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 06:39:32 +0200 Subject: [PATCH 1/2] Emit crash-safe v2 eval campaigns --- evals/report-store.ts | 279 ++++++++++++++++++++++++++ evals/run.ts | 400 ++++++++++++++++++++++++++++++++----- tests/report-store.test.ts | 333 ++++++++++++++++++++++++++++++ 3 files changed, 959 insertions(+), 53 deletions(-) create mode 100644 evals/report-store.ts create mode 100644 tests/report-store.test.ts diff --git a/evals/report-store.ts b/evals/report-store.ts new file mode 100644 index 0000000..6f69d67 --- /dev/null +++ b/evals/report-store.ts @@ -0,0 +1,279 @@ +import { createHash } from "node:crypto"; +import { link, mkdir, open, readdir, readFile, unlink } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { canonicalJson } from "./canonical-json.js"; +import type { ValidatedCaseCatalog } from "./catalog.js"; +import { + type AttemptRecordV2, + type CampaignCompletion, + type CampaignPlan, + CampaignPlanSchema, + parseReport, + type ValidatedReport, +} from "./report.js"; + +export type PersistenceCheckpoint = + | "before-write" + | "after-file-sync" + | "after-rename" + | "before-directory-sync"; + +export type ReportStoreHooks = { + readonly checkpoint?: (checkpoint: PersistenceCheckpoint) => Promise; +}; + +export class ReportStoreError extends Error { + readonly code = "FLOW_REPORT_STORE"; +} + +type StoredAttempt = { + readonly file: string; + readonly value: unknown; + readonly cellId: string | null; +}; + +function fail(message: string, cause?: unknown): never { + throw new ReportStoreError(message, cause ? { cause } : undefined); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function cellId(value: unknown): string | null { + return isRecord(value) && typeof value.cellId === "string" + ? value.cellId + : null; +} + +function attemptFileName(attemptId: string): string { + return `${Buffer.from(attemptId).toString("base64url")}.json`; +} + +function cellFileName(cellId: string): string { + return `${Buffer.from(cellId).toString("base64url")}.json`; +} + +function temporaryPath(path: string): string { + return `${path}.tmp-${process.pid}-${crypto.randomUUID()}`; +} + +function sha256(bytes: Uint8Array): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +async function syncDirectory(directory: string): Promise { + try { + const handle = await open(directory, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } catch { + // Windows and some filesystems do not permit opening directories for sync. + } +} + +async function checkpoint( + hooks: ReportStoreHooks, + name: PersistenceCheckpoint, +): Promise { + await hooks.checkpoint?.(name); +} + +async function writeImmutable( + path: string, + bytes: Buffer, + hooks: ReportStoreHooks, +): Promise<"written" | "replayed"> { + try { + const existing = await readFile(path); + if (existing.equals(bytes)) return "replayed"; + fail(`Immutable report store entry conflicts: ${path}.`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + + await checkpoint(hooks, "before-write"); + const temporary = temporaryPath(path); + const handle = await open(temporary, "wx", 0o600); + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await checkpoint(hooks, "after-file-sync"); + try { + await link(temporary, path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const existing = await readFile(path); + await unlink(temporary); + await syncDirectory(dirname(path)); + if (existing.equals(bytes)) return "replayed"; + fail(`Immutable report store entry conflicts: ${path}.`); + } + await checkpoint(hooks, "after-rename"); + await unlink(temporary); + await checkpoint(hooks, "before-directory-sync"); + await syncDirectory(dirname(path)); + return "written"; + } catch (error) { + await unlink(temporary).catch(() => {}); + throw error; + } +} + +async function readJson(path: string): Promise { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch (error) { + fail(`Could not read report store JSON: ${path}.`, error); + } +} + +export class ReportStore { + private readonly attemptsDirectory: string; + private readonly transcriptsDirectory: string; + private readonly planPath: string; + private readonly completionPath: string; + private readonly reportPath: string; + private readonly catalog: ValidatedCaseCatalog; + private readonly hooks: ReportStoreHooks; + + constructor( + directory: string, + catalog: ValidatedCaseCatalog, + hooks: ReportStoreHooks = {}, + ) { + this.catalog = catalog; + this.hooks = hooks; + this.attemptsDirectory = join(directory, "attempts"); + this.transcriptsDirectory = join(directory, "transcripts"); + this.planPath = join(directory, "plan.json"); + this.completionPath = join(directory, "completion.json"); + this.reportPath = join(directory, "report.json"); + } + + async initialize(plan: CampaignPlan): Promise<"written" | "replayed"> { + await mkdir(this.attemptsDirectory, { recursive: true, mode: 0o700 }); + return writeImmutable( + this.planPath, + Buffer.from(canonicalJson(plan)), + this.hooks, + ); + } + + private async plan(): Promise { + const parsed = CampaignPlanSchema.safeParse(await readJson(this.planPath)); + if (!parsed.success) fail("Stored campaign plan is invalid."); + return parsed.data; + } + + private async attempts(): Promise { + let files: string[]; + try { + files = await readdir(this.attemptsDirectory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + const attempts: StoredAttempt[] = []; + for (const file of files + .filter((entry) => entry.endsWith(".json")) + .sort()) { + const value = await readJson(join(this.attemptsDirectory, file)); + attempts.push({ file, value, cellId: cellId(value) }); + } + return attempts; + } + + async writeAttempt( + attempt: AttemptRecordV2, + ): Promise<"written" | "replayed"> { + const plan = await this.plan(); + if (!plan.cells.some((cell) => cell.cellId === attempt.cellId)) { + fail(`Attempt references an unknown plan cell: ${attempt.cellId}.`); + } + return writeImmutable( + join(this.attemptsDirectory, cellFileName(attempt.cellId)), + Buffer.from(canonicalJson(attempt)), + this.hooks, + ); + } + + async writeTranscript(input: { + readonly attemptId: string; + readonly text: string; + }): Promise<{ readonly artifact: string; readonly sha256: string }> { + await mkdir(this.transcriptsDirectory, { recursive: true, mode: 0o700 }); + const artifact = `transcripts/${attemptFileName(input.attemptId)}`; + const bytes = Buffer.from(input.text, "utf8"); + await writeImmutable( + join(this.transcriptsDirectory, attemptFileName(input.attemptId)), + bytes, + this.hooks, + ); + return { artifact, sha256: sha256(bytes) }; + } + + private orderedAttempts( + plan: CampaignPlan, + attempts: readonly StoredAttempt[], + ): readonly unknown[] { + const ordered: unknown[] = []; + const consumed = new Set(); + for (const cell of plan.cells) { + for (const attempt of attempts) { + if (attempt.cellId === cell.cellId) { + ordered.push(attempt.value); + consumed.add(attempt.file); + } + } + } + for (const attempt of attempts) { + if (!consumed.has(attempt.file)) ordered.push(attempt.value); + } + return ordered; + } + + async finalize(input: { + readonly reportId: string; + readonly completion: CampaignCompletion; + readonly allocationCommitmentSha256: string | null; + }): Promise { + const plan = await this.plan(); + const report = { + schemaVersion: 2, + reportId: input.reportId, + plan, + attempts: this.orderedAttempts(plan, await this.attempts()), + completion: input.completion, + allocationCommitmentSha256: input.allocationCommitmentSha256, + }; + const parsed = parseReport(report, this.catalog); + if (!parsed.ok) { + fail( + `Refusing to finalize invalid report: ${parsed.issues + .map((issue) => `${issue.path} ${issue.message}`) + .join("; ")}`, + ); + } + const completionBytes = Buffer.from(canonicalJson(input.completion)); + const reportBytes = Buffer.from(canonicalJson(report)); + await writeImmutable(this.completionPath, completionBytes, this.hooks); + await writeImmutable(this.reportPath, reportBytes, this.hooks); + return parsed.value; + } +} + +export function createReportStore(input: { + readonly directory: string; + readonly catalog: ValidatedCaseCatalog; + readonly hooks?: ReportStoreHooks; +}): ReportStore { + return new ReportStore(input.directory, input.catalog, input.hooks); +} diff --git a/evals/run.ts b/evals/run.ts index a1b48ac..09a581e 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -17,12 +17,14 @@ import { compileFlowPromptSurface, type FlowPromptSurfaceName, } from "../src/prompt-surfaces.js"; +import { canonicalSha256 } from "./canonical-json.js"; import { buildCassette, type Cassette, cassetteFileName, type FidelityNote, } from "./cassette.js"; +import { parseCaseCatalog, type ValidatedCaseCatalog } from "./catalog.js"; import { askedQuestions, askedScoring, @@ -57,11 +59,17 @@ import { tarballSha256, } from "./provenance.js"; import type { + ActorIdentity, ArtifactIdentity, + AttemptRecordV2, + CampaignCompletion, + CampaignPlan, EvaluatorIdentity, InstructionDelivery, ModelIdentity, } from "./report.js"; +import { campaignPlanSha256 } from "./report.js"; +import { createReportStore } from "./report-store.js"; import { SCENARIOS } from "./scenarios.js"; const SURFACES: FlowPromptSurfaceName[] = [ @@ -181,6 +189,161 @@ function legacyRequestedModel(modelId: string): ModelIdentity { }); } +const V2_ANALYSIS_DIGEST = canonicalSha256("flow-v2-analysis-v1", { + kind: "rate", + primaryOutcome: "conformance-pass", +}); + +function caseCatalogFor( + scenarios: readonly (typeof SCENARIOS)[number][], +): ValidatedCaseCatalog { + const parsed = parseCaseCatalog( + scenarios.map((scenario) => ({ + caseId: scenario.id, + caseVersion: 1, + evidenceClass: "conformance" as const, + oracle: "durable-state" as const, + release: "report-only" as const, + minProviders: 1, + minScoredAttempts: 1, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + })), + ); + if (!parsed.ok) { + throw new Error( + `Could not construct v2 scenario catalog: ${parsed.issues + .map((issue) => issue.message) + .join("; ")}`, + ); + } + return parsed.value; +} + +function campaignPlanFor(input: { + readonly models: readonly string[]; + readonly scenarios: readonly (typeof SCENARIOS)[number][]; + readonly repeat: number; + readonly opencodeVersion: string; +}): CampaignPlan { + const cells = input.models.flatMap((model, modelIndex) => + input.scenarios.flatMap((scenario, scenarioIndex) => + Array.from({ length: input.repeat }, (_, repetition) => { + const slot = + modelIndex * input.scenarios.length * input.repeat + + scenarioIndex * input.repeat + + repetition; + const identity = canonicalSha256("flow-v2-cell-v1", { + model, + scenario: scenario.id, + repetition, + }); + return { + cellId: `cell-${identity.slice("sha256:".length)}`, + blockId: `block-${slot}`, + caseId: scenario.id, + caseVersion: 1, + armToken: null, + repetition, + managerModel: legacyRequestedModel(model), + reviewerModel: null, + schedule: "primary" as const, + }; + }), + ), + ); + const plan = { + schemaVersion: 1 as const, + planId: "flow-v2-primary-matrix", + planSha256: `sha256:${"0".repeat(64)}`, + randomizationSeed: canonicalSha256("flow-v2-seed-v1", { + models: input.models, + scenarios: input.scenarios.map((scenario) => scenario.id), + repeat: input.repeat, + opencodeVersion: input.opencodeVersion, + }), + cells, + abortPolicy: { retry: "never" as const, maxReplacementBlocks: 0 }, + stoppingRule: { + kind: "fixed-attempts" as const, + count: cells.length, + }, + analysis: { + kind: "rate" as const, + primaryOutcome: "conformance-pass", + versionSha256: V2_ANALYSIS_DIGEST, + }, + budget: { + maxUsd: null, + unknownCostPolicy: "token-wall-clock-bounds" as const, + maxOutputTokens: Math.max(1, cells.length) * 200_000, + maxWallClockMs: Math.max(1, cells.length) * 20 * 60_000, + maxAttempts: cells.length, + }, + }; + plan.planSha256 = campaignPlanSha256(plan); + return plan; +} + +function reportActor( + actor: RunResult["provenance"]["actors"][number], +): ActorIdentity | null { + if (actor.sessionIds.length === 0) return null; + const actualModel = + actor.actualModel.kind === "observed" + ? { + kind: "unobserved" as const, + reason: `Host observed providerID=${actor.actualModel.value.providerID} modelID=${actor.actualModel.value.modelID}; full family, gateway, and revision identity is unavailable.`, + } + : actor.actualModel; + return { + role: actor.role, + requestedModel: actor.requestedModel, + actualModel, + sessionIds: [...actor.sessionIds], + }; +} + +function attemptOutcome(result: RunResult): AttemptRecordV2["outcome"] { + if (result.environment || result.error !== undefined) { + return { + kind: "failure", + origin: "host", + code: result.environment ? "environment" : "attempt-error", + retryable: true, + }; + } + if (result.unscored) { + return { + kind: "unscored-escalation", + reason: + result.questions[0] ?? "The model escalated without a scored outcome.", + }; + } + return { + kind: "product", + passed: result.passed, + endedBy: result.escalated ? "user-escalation" : "quiet", + issues: result.passed + ? [] + : result.issues.length > 0 + ? [...result.issues] + : ["The scenario did not satisfy its durable-state checks."], + evidence: { + kind: "conformance", + falseCompletion: result.honesty.falseCompletion, + unsubmittedReviews: result.reviewer.unsubmitted, + facts: { + scenario: result.scenario, + model: result.model, + attempt: result.attempt, + flowCalls: result.flowCalls.length, + guidanceLoads: result.provenance.instructions.length, + }, + }, + }; +} + /** One attempt to run, and the slot its result belongs in. */ type Job = { readonly model: string; @@ -437,6 +600,25 @@ async function main(): Promise { ); const packDir = await mkdtemp(join(tmpdir(), "flow-eval-pack-")); + const reportDir = join(repositoryRoot, "evals", "results"); + await mkdir(reportDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const v2Directory = join(reportDir, `${stamp}.v2`); + const v2Catalog = caseCatalogFor(selected); + const v2Plan = campaignPlanFor({ + models, + scenarios: selected, + repeat, + opencodeVersion, + }); + const reportStore = createReportStore({ + directory: v2Directory, + catalog: v2Catalog, + }); + await reportStore.initialize(v2Plan); + const campaignStartedAt = new Date().toISOString(); + const campaignCells = v2Plan.cells; + const v2Attempts: AttemptRecordV2[] = []; const results: RunResult[] = []; // One decision-layer recording per attempt that reached the model, so the run's // findings can be re-derived against a changed runtime without paying again. @@ -459,7 +641,7 @@ async function main(): Promise { freshSession: step.freshSession === true, })), })), - policyCatalog: { models, repeat, opencodeVersion }, + policyCatalog: v2Catalog, graderBundle: { sourceTreeSha256: artifact.sourceTreeSha256 }, }); const packageCache = await preparePackageCache(tarball, packDir); @@ -467,6 +649,67 @@ async function main(): Promise { throw new Error("Packed artifact changed before host installation."); } await preflight(packageCache, opencodeVersion, models); + const persistV2Attempt = async ( + result: RunResult, + cell: CampaignPlan["cells"][number], + scenario: (typeof SCENARIOS)[number], + ): Promise => { + const attemptId = `attempt-${cell.cellId}`; + const storedTranscript = await reportStore.writeTranscript({ + attemptId, + text: result.provenance.transcript.text, + }); + if (storedTranscript.sha256 !== result.provenance.transcript.sha256) { + throw new Error( + "Persisted transcript does not match provenance digest.", + ); + } + const commandInstructions = scenario.steps.map((step, sequence) => + instructionDelivery({ + source: "command", + name: step.command, + sequence, + text: `/${step.command} ${step.arguments}`.trim(), + }), + ); + const guidanceInstructions = result.provenance.instructions.map( + (instruction, sequence) => ({ + ...instruction, + sequence: commandInstructions.length + sequence, + }), + ); + const instructions = [...commandInstructions, ...guidanceInstructions]; + const actors = result.provenance.actors + .map(reportActor) + .filter((actor): actor is ActorIdentity => actor !== null); + const attemptRecord: AttemptRecordV2 = { + schemaVersion: 2, + attemptId, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: cell.armToken, + repetition: cell.repetition, + artifact: result.provenance.artifact, + evaluator: result.provenance.evaluator, + hostConfigSha256: result.provenance.hostConfigSha256, + actors, + instructions: [...instructions], + transcript: { + sha256: storedTranscript.sha256, + artifact: storedTranscript.artifact, + }, + outcome: attemptOutcome(result), + usage: { + durationMs: result.durationMs, + outputTokens: result.tokens.output, + costUsd: result.costUsd, + }, + }; + await reportStore.writeAttempt(attemptRecord); + v2Attempts.push(attemptRecord); + }; // One queue per model, run concurrently. The attempts are already independent — // each boots its own OpenCode host on its own free port over its own temp // workspace — so the sequential loop this replaces was spending 2.5h of wall @@ -679,6 +922,10 @@ async function main(): Promise { : scoreLabel }`, ); + const cell = campaignCells[job.slot]; + if (!cell) + throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); + await persistV2Attempt(result, cell, scenario); return { slot: job.slot, result, cassette }; } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -690,58 +937,63 @@ async function main(): Promise { // Reaching here means the scenario never got a model turn, with one // exception: a host that answered but rejected every turn is thrown // above and is equally not a prompt result. - return { - slot: job.slot, - cassette, - result: { - scenario: scenario.id, - model, - attempt, - passed: false, - environment: true, - issues: [], - tokens: { - input: 0, - output: 0, - reasoning: 0, - cacheRead: 0, - cacheWrite: 0, - }, - costUsd: null, - assistantMessages: 0, + const result: RunResult = { + scenario: scenario.id, + model, + attempt, + passed: false, + environment: true, + issues: [], + tokens: { + input: 0, + output: 0, + reasoning: 0, + cacheRead: 0, + cacheWrite: 0, + }, + costUsd: null, + assistantMessages: 0, + flowCalls: [], + sessionBoundaries: [], + documents: [], + honesty: completionHonesty(null), + reviewer: reviewerActivity([]), + operational: operationalMetrics([], { flowCalls: [], - sessionBoundaries: [], - documents: [], - honesty: completionHonesty(null), - reviewer: reviewerActivity([]), - operational: operationalMetrics([], { - flowCalls: [], - assistantMessages: 0, - durationMs: Date.now() - started, - }), - refusedBroadScope: 0, - guidanceSkips: 0, - finalText: "", - questions: [], + assistantMessages: 0, durationMs: Date.now() - started, - hostError: null, - provenance: { - artifact, - evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - plugin: `opencode-plugin-flow@${packageJson.version}`, - model, - reviewerModel: requestedReviewerModel, - reviewerSteps: requestedReviewerSteps, - platform: hostPlatform, - }), - actors: [], - instructions: [], - transcript, - }, - error: message, + }), + refusedBroadScope: 0, + guidanceSkips: 0, + finalText: "", + questions: [], + durationMs: Date.now() - started, + hostError: null, + provenance: { + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + plugin: `opencode-plugin-flow@${packageJson.version}`, + model, + reviewerModel: requestedReviewerModel, + reviewerSteps: requestedReviewerSteps, + platform: hostPlatform, + }), + actors: [], + instructions: [], + transcript, }, + error: message, + }; + const cell = campaignCells[job.slot]; + if (!cell) + throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); + await persistV2Attempt(result, cell, scenario); + return { + slot: job.slot, + cassette, + result, }; } finally { await host?.stop(); @@ -758,6 +1010,51 @@ async function main(): Promise { } finally { await rm(packDir, { recursive: true, force: true }); } + const v2Complete = + results.length === v2Plan.cells.length && + v2Attempts.length === v2Plan.cells.length && + v2Attempts.every((attempt) => attempt.outcome.kind === "product"); + const v2CostUsd = v2Attempts.some((attempt) => attempt.usage.costUsd === null) + ? null + : v2Attempts.reduce( + (total, attempt) => total + (attempt.usage.costUsd ?? 0), + 0, + ); + const v2FinishedAt = new Date().toISOString(); + const v2Completion: CampaignCompletion = { + status: v2Complete ? "complete" : "stopped", + cause: v2Complete + ? "fixed-target" + : results.some( + (result) => result.environment || result.error !== undefined, + ) + ? "host" + : results.some((result) => result.unscored) + ? "operator" + : "evaluator", + startedAt: campaignStartedAt, + finishedAt: v2FinishedAt, + activatedReserveCellIds: [], + observed: { + attempts: v2Attempts.length, + outputTokens: v2Attempts.reduce( + (total, attempt) => total + attempt.usage.outputTokens, + 0, + ), + costUsd: v2CostUsd, + wallClockMs: Math.max( + Date.parse(v2FinishedAt) - Date.parse(campaignStartedAt), + ...v2Attempts.map((attempt) => attempt.usage.durationMs), + ), + }, + }; + await reportStore.finalize({ + reportId: `flow-v2-${stamp}`, + completion: v2Completion, + allocationCommitmentSha256: null, + }); + const v2ReportPath = join(v2Directory, "report.json"); + console.log(`V2 report: ${v2ReportPath}`); console.log(`\n${formatTable(results)}\n`); // An abort is excluded for the same reason an allowed ask is: the run never @@ -889,9 +1186,6 @@ async function main(): Promise { ); } - const reportDir = join(repositoryRoot, "evals", "results"); - await mkdir(reportDir, { recursive: true }); - const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const reportPath = join(reportDir, `${stamp}.json`); await writeFile( reportPath, diff --git a/tests/report-store.test.ts b/tests/report-store.test.ts new file mode 100644 index 0000000..57c19e5 --- /dev/null +++ b/tests/report-store.test.ts @@ -0,0 +1,333 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { access, mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + parseCaseCatalog, + type ValidatedCaseCatalog, +} from "../evals/catalog.js"; +import { + type AttemptRecordV2, + type CampaignCompletion, + type CampaignPlan, + campaignPlanSha256, +} from "../evals/report.js"; +import { + createReportStore, + type PersistenceCheckpoint, + ReportStoreError, +} from "../evals/report-store.js"; + +const temporary: string[] = []; +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; + +afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +async function directory(): Promise { + const path = await mkdtemp(join(tmpdir(), "flow-report-store-")); + temporary.push(path); + return path; +} + +function model() { + return { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "test", + revision: null, + }; +} + +function catalog(): ValidatedCaseCatalog { + const parsed = parseCaseCatalog([ + { + caseId: "case", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + ]); + if (!parsed.ok) throw new Error("Fixture catalog must parse."); + return parsed.value; +} + +function plan(count = 1): CampaignPlan { + const value: CampaignPlan = { + schemaVersion: 1, + planId: "plan", + planSha256: digest("a"), + randomizationSeed: "seed", + cells: Array.from({ length: count }, (_, index) => ({ + cellId: `cell-${index}`, + blockId: `block-${index}`, + caseId: "case", + caseVersion: 1, + armToken: null, + repetition: index, + managerModel: model(), + reviewerModel: null, + schedule: "primary", + })), + abortPolicy: { retry: "never", maxReplacementBlocks: 0 }, + stoppingRule: { kind: "fixed-attempts", count }, + analysis: { + kind: "rate", + primaryOutcome: "pass", + versionSha256: digest("b"), + }, + budget: { + maxUsd: 10, + unknownCostPolicy: "stop", + maxOutputTokens: 100, + maxWallClockMs: 10_000, + maxAttempts: count, + }, + }; + value.planSha256 = campaignPlanSha256(value); + return value; +} + +function attempt( + campaign: CampaignPlan, + index: number, + attemptId = `attempt-${index}`, +): AttemptRecordV2 { + const cell = campaign.cells[index]; + if (!cell) throw new Error("Fixture cell is missing."); + const requestedModel = cell.managerModel; + if (!requestedModel) throw new Error("Fixture manager is missing."); + return { + schemaVersion: 2, + attemptId, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: cell.armToken, + repetition: cell.repetition, + artifact: { + packageVersion: "1.0.0", + sourceCommit: "commit", + sourceTreeSha256: digest("c"), + tarballSha256: digest("d"), + unpackedManifestSha256: digest("e"), + }, + evaluator: { + sourceCommit: "evaluator", + caseCatalogSha256: digest("f"), + policyCatalogSha256: digest("0"), + graderBundleSha256: digest("1"), + }, + hostConfigSha256: digest("2"), + actors: [ + { + role: "manager", + requestedModel, + actualModel: { kind: "observed", value: requestedModel }, + sessionIds: ["session"], + }, + ], + instructions: [ + { + source: "guidance", + name: "flow-run", + sequence: 0, + sha256: digest("3"), + bytes: 1, + }, + ], + transcript: { sha256: digest("4"), artifact: `attempt-${index}.json` }, + outcome: { + kind: "product", + passed: true, + endedBy: "quiet", + issues: [], + evidence: { + kind: "conformance", + falseCompletion: false, + unsubmittedReviews: 0, + facts: { fixture: true }, + }, + }, + usage: { durationMs: 10, outputTokens: 1, costUsd: 1 }, + }; +} + +function completion(count: number): CampaignCompletion { + return { + status: "complete", + cause: "fixed-target", + startedAt: "2026-08-25T00:00:00.000Z", + finishedAt: "2026-08-25T00:00:01.000Z", + activatedReserveCellIds: [], + observed: { + attempts: count, + outputTokens: count, + costUsd: count, + wallClockMs: 1_000, + }, + }; +} + +describe("report store", () => { + test("writes immutable attempts and reconciles them in frozen plan order", async () => { + const root = await directory(); + const campaign = plan(2); + const store = createReportStore({ directory: root, catalog: catalog() }); + expect(await store.initialize(campaign)).toBe("written"); + expect(await store.writeAttempt(attempt(campaign, 1))).toBe("written"); + expect(await store.writeAttempt(attempt(campaign, 0))).toBe("written"); + const report = await store.finalize({ + reportId: "report", + completion: completion(2), + allocationCommitmentSha256: null, + }); + expect(report.attempts.map((item) => item.cellId)).toEqual([ + "cell-0", + "cell-1", + ]); + }); + + test("allows only byte-identical replay and rejects conflicting ids or cells", async () => { + const root = await directory(); + const campaign = plan(); + const store = createReportStore({ directory: root, catalog: catalog() }); + await store.initialize(campaign); + const first = attempt(campaign, 0); + expect(await store.writeAttempt(first)).toBe("written"); + expect(await store.writeAttempt(first)).toBe("replayed"); + await expect( + store.writeAttempt({ + ...first, + usage: { ...first.usage, outputTokens: 2 }, + }), + ).rejects.toBeInstanceOf(ReportStoreError); + await expect( + store.writeAttempt(attempt(campaign, 0, "other-id")), + ).rejects.toBeInstanceOf(ReportStoreError); + }); + + test("stores transcripts immutably beside attempts", async () => { + const root = await directory(); + const campaign = plan(); + const store = createReportStore({ directory: root, catalog: catalog() }); + await store.initialize(campaign); + const text = '{"redacted":true}\n'; + const stored = await store.writeTranscript({ + attemptId: "attempt-0", + text, + }); + expect(stored.artifact).toBe("transcripts/YXR0ZW1wdC0w.json"); + expect(stored.sha256).toBe( + `sha256:${createHash("sha256").update(text, "utf8").digest("hex")}`, + ); + expect(await Bun.file(join(root, stored.artifact)).text()).toBe(text); + expect( + await store.writeTranscript({ attemptId: "attempt-0", text }), + ).toEqual(stored); + await expect( + store.writeTranscript({ attemptId: "attempt-0", text: "changed" }), + ).rejects.toBeInstanceOf(ReportStoreError); + }); + + test("publishes only one conflicting concurrent attempt", async () => { + const root = await directory(); + const campaign = plan(); + const stable = createReportStore({ directory: root, catalog: catalog() }); + await stable.initialize(campaign); + const first = attempt(campaign, 0); + const second: AttemptRecordV2 = { + ...first, + usage: { ...first.usage, outputTokens: 2 }, + }; + const results = await Promise.allSettled([ + createReportStore({ directory: root, catalog: catalog() }).writeAttempt( + first, + ), + createReportStore({ directory: root, catalog: catalog() }).writeAttempt( + second, + ), + ]); + expect( + results.filter((result) => result.status === "fulfilled"), + ).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(1); + expect( + (await readdir(join(root, "attempts"))).filter((file) => + file.endsWith(".json"), + ), + ).toHaveLength(1); + }); + + test("recovers at every persistence interruption and ignores temporary files", async () => { + const checkpoints: readonly PersistenceCheckpoint[] = [ + "before-write", + "after-file-sync", + "after-rename", + "before-directory-sync", + ]; + for (const checkpoint of checkpoints) { + const root = await directory(); + const campaign = plan(); + const stable = createReportStore({ directory: root, catalog: catalog() }); + await stable.initialize(campaign); + const interrupted = createReportStore({ + directory: root, + catalog: catalog(), + hooks: { + checkpoint: async (actual) => { + if (actual === checkpoint) throw new Error(`interrupted ${actual}`); + }, + }, + }); + await expect( + interrupted.writeAttempt(attempt(campaign, 0)), + ).rejects.toThrow(`interrupted ${checkpoint}`); + await expect(readdir(join(root, "attempts"))).resolves.toBeInstanceOf( + Array, + ); + expect(await stable.writeAttempt(attempt(campaign, 0))).toMatch( + /written|replayed/, + ); + await expect( + stable.finalize({ + reportId: "report", + completion: completion(1), + allocationCommitmentSha256: null, + }), + ).resolves.toMatchObject({ reportId: "report" }); + } + }); + + test("never commits completion or report from a truncated ledger", async () => { + const root = await directory(); + const campaign = plan(2); + const store = createReportStore({ directory: root, catalog: catalog() }); + await store.initialize(campaign); + await store.writeAttempt(attempt(campaign, 0)); + await expect( + store.finalize({ + reportId: "report", + completion: completion(2), + allocationCommitmentSha256: null, + }), + ).rejects.toBeInstanceOf(ReportStoreError); + await expect(access(join(root, "completion.json"))).rejects.toBeDefined(); + await expect(access(join(root, "report.json"))).rejects.toBeDefined(); + }); +}); From c9e953a3386d1e91cc0297d89c7f670232ff47be Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 06:39:43 +0200 Subject: [PATCH 2/2] Record Phase 4 campaign evidence --- .../evidence/phase-4-pilot.json | 20 +++++++++++++++++++ .../evidence/phase-4-review.md | 16 +++++++++++++++ .../phase-4-attempt-emission.md | 5 +++++ .audit/eval-engineering.tsv | 5 +++++ 4 files changed, 46 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-4-pilot.json create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-4-review.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-4-pilot.json b/.agents/plans/02-eval-engineering/evidence/phase-4-pilot.json new file mode 100644 index 0000000..671d556 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-4-pilot.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "reportId": "flow-v2-2026-08-25T04-34-58-778Z", + "scenario": "happy-path", + "passed": true, + "planSha256": "sha256:79325b57e4fba2e823b503a8f50758e32a69d7907604d0ce105c4a8bc67c19da", + "cellCount": 1, + "attemptCount": 1, + "completionStatus": "complete", + "completionCause": "fixed-target", + "outputTokens": 3238, + "costUsd": 0.1978994, + "artifactTarballSha256": "sha256:c89f7363248ccc3e3f69728c1aa42044a25938cd533e470d9f73ef08bc64ad24", + "attemptOutcome": "product", + "managerActualIdentity": "unobserved-full-v2-identity", + "reviewerActualIdentity": "unobserved-full-v2-identity", + "instructionCount": 3, + "transcriptSha256": "sha256:588356d860cec23ab95d12f39f4d91f67ff3dd2a87c9849c1fc1a4719d1bc52d", + "strictParsePassed": true +} diff --git a/.agents/plans/02-eval-engineering/evidence/phase-4-review.md b/.agents/plans/02-eval-engineering/evidence/phase-4-review.md new file mode 100644 index 0000000..1ee154e --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-4-review.md @@ -0,0 +1,16 @@ +# Phase 4 Interrogate review + +Phase 4 freezes cells before launch, publishes one immutable transcript and +attempt per cell, and finalizes only reports accepted by the strict v2 parser. + +The four-model review fixed three integrity gaps. Transcript SHA-256 now comes +from stored bytes and must equal the provenance digest. Attempts publish through +one atomic cell-keyed no-replace claim, so concurrent attempt IDs cannot both win. +Handled persistence failures clean temporary files while real crash leftovers are +ignored during reconciliation. Host and mid-flight attempt errors finalize with a +host stop cause. + +The final recheck found no unresolved blocker. Store fault-injection, concurrent +writer, replay, transcript, truncated-ledger, report, and full product gates pass. +The final paid pilot emitted a complete v2 report which independently parsed with +one product attempt and no placeholder provenance or policy. diff --git a/.agents/plans/02-eval-engineering/phase-4-attempt-emission.md b/.agents/plans/02-eval-engineering/phase-4-attempt-emission.md index 16fc7d8..6565777 100644 --- a/.agents/plans/02-eval-engineering/phase-4-attempt-emission.md +++ b/.agents/plans/02-eval-engineering/phase-4-attempt-emission.md @@ -17,6 +17,8 @@ complete v2 campaign from the existing live runner. - `tests/eval-reporting.test.ts`. Inject failures before write, after file sync, after rename, and before directory sync. Cover resume, duplicate ids, reserve activation, deterministic order, and unknown-cost stops. +- `tests/report-store.test.ts`. Isolate persistence faults, concurrent claims, + transcript binding, immutable replay, and truncated-ledger finalization. ## Data structures @@ -32,3 +34,6 @@ and prove no scored attempt is replaced and no truncated ledger looks complete. Stop gate. No qualifier cutover until one live v2 report validates without placeholder provenance or policy. + +Evidence. [Final paid v2 pilot](evidence/phase-4-pilot.json) and +[Interrogate review](evidence/phase-4-review.md). diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index 43fa53e..093bb0b 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -36,3 +36,8 @@ ts phase decision why evidence result 2026-08-25T04:14:31Z phase-3 fixed the multi-model Interrogate findings archive types, endpoint completeness, requested actors, host config, transcript fields, Unicode, and production integration were evidence boundaries .agents/plans/02-eval-engineering/evidence/phase-3-review.md VERIFIED no unresolved blocker 2026-08-25T04:14:31Z phase-3 reran the final paid packed-host pilot the committed observation must be generated by the final code and exact tarball evals/results/2026-08-25T04-14-14-980Z.json; .agents/plans/02-eval-engineering/evidence/phase-3-pilot.json VERIFIED happy-path pass, observed manager and reviewer, redaction scan clean 2026-08-25T04:14:31Z phase-3 ran Deslop and the whole repository gate the phase must finish reviewable and regression-free bun run check VERIFIED 462 pass, 1 skip, 0 fail +2026-08-25T04:38:50Z phase-4 started from merged Phase 3 main crash-safe emission must build on exact observed provenance git status on codex/eval-phase-4 at 2c6cea3 VERIFIED clean baseline, 462 pass, 1 skip, 0 fail +2026-08-25T04:38:50Z phase-4 implemented cell-owned immutable campaign storage concurrent attempts cannot share a writer or replace scored evidence evals/report-store.ts; tests/report-store.test.ts VERIFIED fault injection and concurrent claims green +2026-08-25T04:38:50Z phase-4 fixed the multi-model Interrogate findings transcript binding, cell-level publication, temporary cleanup, and terminal cause affected evidence integrity .agents/plans/02-eval-engineering/evidence/phase-4-review.md VERIFIED no unresolved blocker +2026-08-25T04:38:50Z phase-4 emitted and parsed a live v2 report the cutover cannot proceed on synthetic storage evidence alone .agents/plans/02-eval-engineering/evidence/phase-4-pilot.json VERIFIED one packed happy-path product attempt +2026-08-25T04:38:50Z phase-4 ran Deslop and the whole repository gate the phase must finish reviewable and regression-free bun run check VERIFIED 468 pass, 1 skip, 0 fail