Emit crash-safe v2 eval campaigns

.agents/plans/02-eval-engineering/evidence/phase-4-pilot.json

@@ -0,0 +1,20 @@
+{
+  "schemaVersion": 1,
+  "reportId": "flow-v2-2026-08-25T04-34-58-778Z",
+  "scenario": "happy-path",
+  "passed": true,
+  "planSha256": "sha256:79325b57e4fba2e823b503a8f50758e32a69d7907604d0ce105c4a8bc67c19da",
+  "cellCount": 1,
+  "attemptCount": 1,
+  "completionStatus": "complete",
+  "completionCause": "fixed-target",
+  "outputTokens": 3238,
+  "costUsd": 0.1978994,
+  "artifactTarballSha256": "sha256:c89f7363248ccc3e3f69728c1aa42044a25938cd533e470d9f73ef08bc64ad24",
+  "attemptOutcome": "product",
+  "managerActualIdentity": "unobserved-full-v2-identity",
+  "reviewerActualIdentity": "unobserved-full-v2-identity",
+  "instructionCount": 3,
+  "transcriptSha256": "sha256:588356d860cec23ab95d12f39f4d91f67ff3dd2a87c9849c1fc1a4719d1bc52d",
+  "strictParsePassed": true
+}

.agents/plans/02-eval-engineering/evidence/phase-4-review.md

@@ -0,0 +1,16 @@
+# Phase 4 Interrogate review
+
+Phase 4 freezes cells before launch, publishes one immutable transcript and
+attempt per cell, and finalizes only reports accepted by the strict v2 parser.
+
+The four-model review fixed three integrity gaps. Transcript SHA-256 now comes
+from stored bytes and must equal the provenance digest. Attempts publish through
+one atomic cell-keyed no-replace claim, so concurrent attempt IDs cannot both win.
+Handled persistence failures clean temporary files while real crash leftovers are
+ignored during reconciliation. Host and mid-flight attempt errors finalize with a
+host stop cause.
+
+The final recheck found no unresolved blocker. Store fault-injection, concurrent
+writer, replay, transcript, truncated-ledger, report, and full product gates pass.
+The final paid pilot emitted a complete v2 report which independently parsed with
+one product attempt and no placeholder provenance or policy.

.agents/plans/02-eval-engineering/phase-4-attempt-emission.md

@@ -17,6 +17,8 @@ complete v2 campaign from the existing live runner.
 - `tests/eval-reporting.test.ts`. Inject failures before write, after file sync,
   after rename, and before directory sync. Cover resume, duplicate ids, reserve
   activation, deterministic order, and unknown-cost stops.
+- `tests/report-store.test.ts`. Isolate persistence faults, concurrent claims,
+  transcript binding, immutable replay, and truncated-ledger finalization.
 
 ## Data structures
 
@@ -32,3 +34,6 @@ and prove no scored attempt is replaced and no truncated ledger looks complete.
 
 Stop gate. No qualifier cutover until one live v2 report validates without
 placeholder provenance or policy.
+
+Evidence. [Final paid v2 pilot](evidence/phase-4-pilot.json) and
+[Interrogate review](evidence/phase-4-review.md).

.audit/eval-engineering.tsv

@@ -36,3 +36,8 @@ ts	phase	decision	why	evidence	result
 2026-08-25T04:14:31Z	phase-3	fixed the multi-model Interrogate findings	archive types, endpoint completeness, requested actors, host config, transcript fields, Unicode, and production integration were evidence boundaries	.agents/plans/02-eval-engineering/evidence/phase-3-review.md	VERIFIED no unresolved blocker
 2026-08-25T04:14:31Z	phase-3	reran the final paid packed-host pilot	the committed observation must be generated by the final code and exact tarball	evals/results/2026-08-25T04-14-14-980Z.json; .agents/plans/02-eval-engineering/evidence/phase-3-pilot.json	VERIFIED happy-path pass, observed manager and reviewer, redaction scan clean
 2026-08-25T04:14:31Z	phase-3	ran Deslop and the whole repository gate	the phase must finish reviewable and regression-free	bun run check	VERIFIED 462 pass, 1 skip, 0 fail
+2026-08-25T04:38:50Z	phase-4	started from merged Phase 3 main	crash-safe emission must build on exact observed provenance	git status on codex/eval-phase-4 at 2c6cea3	VERIFIED clean baseline, 462 pass, 1 skip, 0 fail
+2026-08-25T04:38:50Z	phase-4	implemented cell-owned immutable campaign storage	concurrent attempts cannot share a writer or replace scored evidence	evals/report-store.ts; tests/report-store.test.ts	VERIFIED fault injection and concurrent claims green
+2026-08-25T04:38:50Z	phase-4	fixed the multi-model Interrogate findings	transcript binding, cell-level publication, temporary cleanup, and terminal cause affected evidence integrity	.agents/plans/02-eval-engineering/evidence/phase-4-review.md	VERIFIED no unresolved blocker
+2026-08-25T04:38:50Z	phase-4	emitted and parsed a live v2 report	the cutover cannot proceed on synthetic storage evidence alone	.agents/plans/02-eval-engineering/evidence/phase-4-pilot.json	VERIFIED one packed happy-path product attempt
+2026-08-25T04:38:50Z	phase-4	ran Deslop and the whole repository gate	the phase must finish reviewable and regression-free	bun run check	VERIFIED 468 pass, 1 skip, 0 fail

evals/report-store.ts

@@ -0,0 +1,279 @@
+import { createHash } from "node:crypto";
+import { link, mkdir, open, readdir, readFile, unlink } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { canonicalJson } from "./canonical-json.js";
+import type { ValidatedCaseCatalog } from "./catalog.js";
+import {
+	type AttemptRecordV2,
+	type CampaignCompletion,
+	type CampaignPlan,
+	CampaignPlanSchema,
+	parseReport,
+	type ValidatedReport,
+} from "./report.js";
+
+export type PersistenceCheckpoint =
+	| "before-write"
+	| "after-file-sync"
+	| "after-rename"
+	| "before-directory-sync";
+
+export type ReportStoreHooks = {
+	readonly checkpoint?: (checkpoint: PersistenceCheckpoint) => Promise<void>;
+};
+
+export class ReportStoreError extends Error {
+	readonly code = "FLOW_REPORT_STORE";
+}
+
+type StoredAttempt = {
+	readonly file: string;
+	readonly value: unknown;
+	readonly cellId: string | null;
+};
+
+function fail(message: string, cause?: unknown): never {
+	throw new ReportStoreError(message, cause ? { cause } : undefined);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function cellId(value: unknown): string | null {
+	return isRecord(value) && typeof value.cellId === "string"
+		? value.cellId
+		: null;
+}
+
+function attemptFileName(attemptId: string): string {
+	return `${Buffer.from(attemptId).toString("base64url")}.json`;
+}
+
+function cellFileName(cellId: string): string {
+	return `${Buffer.from(cellId).toString("base64url")}.json`;
+}
+
+function temporaryPath(path: string): string {
+	return `${path}.tmp-${process.pid}-${crypto.randomUUID()}`;
+}
+
+function sha256(bytes: Uint8Array): string {
+	return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+}
+
+async function syncDirectory(directory: string): Promise<void> {
+	try {
+		const handle = await open(directory, "r");
+		try {
+			await handle.sync();
+		} finally {
+			await handle.close();
+		}
+	} catch {
+		// Windows and some filesystems do not permit opening directories for sync.
+	}
+}
+
+async function checkpoint(
+	hooks: ReportStoreHooks,
+	name: PersistenceCheckpoint,
+): Promise<void> {
+	await hooks.checkpoint?.(name);
+}
+
+async function writeImmutable(
+	path: string,
+	bytes: Buffer,
+	hooks: ReportStoreHooks,
+): Promise<"written" | "replayed"> {
+	try {
+		const existing = await readFile(path);
+		if (existing.equals(bytes)) return "replayed";
+		fail(`Immutable report store entry conflicts: ${path}.`);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+	}
+
+	await checkpoint(hooks, "before-write");
+	const temporary = temporaryPath(path);
+	const handle = await open(temporary, "wx", 0o600);
+	try {
+		await handle.writeFile(bytes);
+		await handle.sync();
+	} finally {
+		await handle.close();
+	}
+	try {
+		await checkpoint(hooks, "after-file-sync");
+		try {
+			await link(temporary, path);
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+			const existing = await readFile(path);
+			await unlink(temporary);
+			await syncDirectory(dirname(path));
+			if (existing.equals(bytes)) return "replayed";
+			fail(`Immutable report store entry conflicts: ${path}.`);
+		}
+		await checkpoint(hooks, "after-rename");
+		await unlink(temporary);
+		await checkpoint(hooks, "before-directory-sync");
+		await syncDirectory(dirname(path));
+		return "written";
+	} catch (error) {
+		await unlink(temporary).catch(() => {});
+		throw error;
+	}
+}
+
+async function readJson(path: string): Promise<unknown> {
+	try {
+		return JSON.parse(await readFile(path, "utf8"));
+	} catch (error) {
+		fail(`Could not read report store JSON: ${path}.`, error);
+	}
+}
+
+export class ReportStore {
+	private readonly attemptsDirectory: string;
+	private readonly transcriptsDirectory: string;
+	private readonly planPath: string;
+	private readonly completionPath: string;
+	private readonly reportPath: string;
+	private readonly catalog: ValidatedCaseCatalog;
+	private readonly hooks: ReportStoreHooks;
+
+	constructor(
+		directory: string,
+		catalog: ValidatedCaseCatalog,
+		hooks: ReportStoreHooks = {},
+	) {
+		this.catalog = catalog;
+		this.hooks = hooks;
+		this.attemptsDirectory = join(directory, "attempts");
+		this.transcriptsDirectory = join(directory, "transcripts");
+		this.planPath = join(directory, "plan.json");
+		this.completionPath = join(directory, "completion.json");
+		this.reportPath = join(directory, "report.json");
+	}
+
+	async initialize(plan: CampaignPlan): Promise<"written" | "replayed"> {
+		await mkdir(this.attemptsDirectory, { recursive: true, mode: 0o700 });
+		return writeImmutable(
+			this.planPath,
+			Buffer.from(canonicalJson(plan)),
+			this.hooks,
+		);
+	}
+
+	private async plan(): Promise<CampaignPlan> {
+		const parsed = CampaignPlanSchema.safeParse(await readJson(this.planPath));
+		if (!parsed.success) fail("Stored campaign plan is invalid.");
+		return parsed.data;
+	}
+
+	private async attempts(): Promise<readonly StoredAttempt[]> {
+		let files: string[];
+		try {
+			files = await readdir(this.attemptsDirectory);
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+			throw error;
+		}
+		const attempts: StoredAttempt[] = [];
+		for (const file of files
+			.filter((entry) => entry.endsWith(".json"))
+			.sort()) {
+			const value = await readJson(join(this.attemptsDirectory, file));
+			attempts.push({ file, value, cellId: cellId(value) });
+		}
+		return attempts;
+	}
+
+	async writeAttempt(
+		attempt: AttemptRecordV2,
+	): Promise<"written" | "replayed"> {
+		const plan = await this.plan();
+		if (!plan.cells.some((cell) => cell.cellId === attempt.cellId)) {
+			fail(`Attempt references an unknown plan cell: ${attempt.cellId}.`);
+		}
+		return writeImmutable(
+			join(this.attemptsDirectory, cellFileName(attempt.cellId)),
+			Buffer.from(canonicalJson(attempt)),
+			this.hooks,
+		);
+	}
+
+	async writeTranscript(input: {
+		readonly attemptId: string;
+		readonly text: string;
+	}): Promise<{ readonly artifact: string; readonly sha256: string }> {
+		await mkdir(this.transcriptsDirectory, { recursive: true, mode: 0o700 });
+		const artifact = `transcripts/${attemptFileName(input.attemptId)}`;
+		const bytes = Buffer.from(input.text, "utf8");
+		await writeImmutable(
+			join(this.transcriptsDirectory, attemptFileName(input.attemptId)),
+			bytes,
+			this.hooks,
+		);
+		return { artifact, sha256: sha256(bytes) };
+	}
+
+	private orderedAttempts(
+		plan: CampaignPlan,
+		attempts: readonly StoredAttempt[],
+	): readonly unknown[] {
+		const ordered: unknown[] = [];
+		const consumed = new Set<string>();
+		for (const cell of plan.cells) {
+			for (const attempt of attempts) {
+				if (attempt.cellId === cell.cellId) {
+					ordered.push(attempt.value);
+					consumed.add(attempt.file);
+				}
+			}
+		}
+		for (const attempt of attempts) {
+			if (!consumed.has(attempt.file)) ordered.push(attempt.value);
+		}
+		return ordered;
+	}
+
+	async finalize(input: {
+		readonly reportId: string;
+		readonly completion: CampaignCompletion;
+		readonly allocationCommitmentSha256: string | null;
+	}): Promise<ValidatedReport> {
+		const plan = await this.plan();
+		const report = {
+			schemaVersion: 2,
+			reportId: input.reportId,
+			plan,
+			attempts: this.orderedAttempts(plan, await this.attempts()),
+			completion: input.completion,
+			allocationCommitmentSha256: input.allocationCommitmentSha256,
+		};
+		const parsed = parseReport(report, this.catalog);
+		if (!parsed.ok) {
+			fail(
+				`Refusing to finalize invalid report: ${parsed.issues
+					.map((issue) => `${issue.path} ${issue.message}`)
+					.join("; ")}`,
+			);
+		}
+		const completionBytes = Buffer.from(canonicalJson(input.completion));
+		const reportBytes = Buffer.from(canonicalJson(report));
+		await writeImmutable(this.completionPath, completionBytes, this.hooks);
+		await writeImmutable(this.reportPath, reportBytes, this.hooks);
+		return parsed.value;
+	}
+}
+
+export function createReportStore(input: {
+	readonly directory: string;
+	readonly catalog: ValidatedCaseCatalog;
+	readonly hooks?: ReportStoreHooks;
+}): ReportStore {
+	return new ReportStore(input.directory, input.catalog, input.hooks);
+}