Add strict eval report boundary

.agents/plans/02-eval-engineering/architecture.md

@@ -110,8 +110,7 @@ type AttemptRecordV2 = {
 	readonly hostConfigSha256: string;
 	readonly actors: readonly ActorIdentity[];
 	readonly instructions: readonly InstructionDelivery[];
-	readonly transcriptSha256: string;
-	readonly transcriptArtifact: string;
+	readonly transcript: { readonly sha256: string; readonly artifact: string } | null;
 	readonly outcome: AttemptOutcome;
 	readonly usage: { readonly durationMs: number; readonly outputTokens: number; readonly costUsd: number | null };
 };
@@ -120,6 +119,7 @@ type ScheduledCell = {
 	readonly cellId: string;
 	readonly blockId: string;
 	readonly caseId: string;
+	readonly caseVersion: number;
 	readonly armToken: string | null;
 	readonly repetition: number;
 	readonly managerModel: ModelIdentity | null;
@@ -351,7 +351,10 @@ remain. New code adds evidence semantics and analysis around them.
 
 ## Module map
 
-- `evals/report.ts` owns v2 types, strict parsing, integrity checks, and verdicts.
+- `evals/report.ts` owns v2 types, strict parsing, integrity checks, and verdicts;
+  `evals/canonical-json.ts` owns canonical serialization and domain-separated hashes.
+- `evals/validated.ts` owns the shared deep-readonly and runtime-freeze boundary.
+- `evals/report-pairing.ts` owns paired plan and complete-pair invariants.
 - `evals/report-store.ts` owns canonical write-once attempt files and finalization.
 - `evals/provenance.ts` owns source, artifact, evaluator, host, actor, and instruction digests.
 - `evals/catalog.ts` owns typed case policies and campaign planning.

.agents/plans/02-eval-engineering/evidence/phase-1-review.md

@@ -0,0 +1,60 @@
+# Phase 1 Interrogate review
+
+## Intent
+
+Phase 1 must add a fail-closed, schema-derived v2 boundary for frozen campaign
+plans, atomic attempts, completion records, and case policy. It must reject
+malformed, legacy, summary-only, cross-record, budget, and paired-design drift
+without changing the runner or release qualifier.
+
+## Reviewers
+
+- `gpt-5.6-terra` reviewed the first implementation and the final corrected diff.
+- `gpt-5.6-luna` reviewed the first implementation and the final corrected diff.
+- `gpt-5.5` reviewed the first implementation and the final corrected diff.
+- `gpt-5.4` reviewed the first implementation and the final corrected diff.
+
+## Acted on
+
+- Declared attempt, token, wall-clock, cost, and unknown-cost budgets now constrain
+  completion state. Wall-clock evidence has timestamp and longest-attempt lower
+  bounds.
+- Complete and stopped states now follow scored outcomes. Failure rows cannot
+  satisfy fixed targets.
+- Paired plans now require complete scored pairs and atomic reserve activation.
+  Non-paired plans cannot use the paired replacement policy.
+- Paired hidden evidence and reviewer fixed-label evidence now agree with their
+  top-level scored outcome.
+- Validated reports and catalogs are deeply readonly and recursively frozen.
+- Canonical JSON rejects malformed Unicode, sparse arrays, and non-JSON objects.
+  Catalog identifiers use the same Unicode scalar boundary.
+- Canonicalization, immutable validation, and paired-plan rules moved into small
+  reusable modules. No changed TypeScript file exceeds 1,000 lines.
+
+## Deferred by phase boundary
+
+- Catalog sample floors belong to Phase 2 decision analysis. A structurally valid
+  under-sampled report must remain analyzable as `INCONCLUSIVE`.
+- Expected artifact, evaluator, host, and actor comparison belongs to Phase 3.
+  Paired campaigns intentionally contain different candidate and baseline
+  artifacts, so one report-wide artifact equality rule would be wrong.
+
+## Dismissed
+
+- Opaque arm tokens are treatment keys and are intentionally reused across
+  blocks. They must differ within a block, not across the full plan.
+- A budget stop may occur below an observed ceiling when the next atomic attempt
+  cannot safely start. Exact budget exhaustion is not a valid completion
+  invariant.
+- An unsubmitted reviewer result is valid failed evidence, not malformed data.
+  The explicit nullable verdict and submitted flag preserve it for Phase 2
+  qualification.
+- Global collection limits do not protect the JSON parse allocation and would
+  introduce arbitrary campaign caps. Frozen plan budgets constrain executable
+  work instead.
+
+## Verdict
+
+`VERIFIED`. The final four-model recheck found no unresolved blocker after lead
+judgment. Focused tests pass with 19 cases. The full repository gate passes with
+430 tests, one intentional live-host skip, and zero failures.

.agents/plans/02-eval-engineering/phase-1-report-boundary.md

@@ -12,11 +12,18 @@ finalization record, and strict structural and semantic validation boundary.
 - `evals/report.ts`. Add v2 report, campaign, cell, attempt, disposition,
   evidence-payload, completion, provenance, actor, and structured issue types plus
   strict structural parsing.
+- `evals/canonical-json.ts`. Add the reusable RFC 8785-compatible canonical JSON
+  and domain-separated digest primitive used by frozen plans.
+- `evals/validated.ts`. Keep validated boundary values deeply readonly in types
+  and recursively frozen at runtime.
+- `evals/report-pairing.ts`. Isolate complete-pair shape, retry, activation, and
+  scored-target invariants from the general report boundary.
 - `evals/catalog.ts`. Add the minimal case identity, evidence class, oracle, sample
   floor, release disposition, and analysis policy needed for semantic validation.
 - `tests/eval-report.test.ts`. Cover valid round trips, missing fields, duplicate
   ids, unknown cases against a supplied catalog, inconsistent pairs, numeric
-  bounds, finalization causes, non-finite counters, and the summary-only exploit.
+  bounds, cell-to-case-version drift, finalization causes, non-finite counters,
+  transcript availability by outcome, and the summary-only exploit.
 
 ## Data structures
 

.audit/eval-engineering.tsv

@@ -21,3 +21,8 @@ ts	phase	decision	why	evidence	result
 2026-08-24T22:35:43Z	phase-0	reran the full repository gate after review fixes	accepted review changes require the same whole-product proof as the original slice	bun run check	VERIFIED 404 pass, 1 skip, 0 fail
 2026-08-24T22:50:31Z	phase-0	simplified the probe and tests before commit	the first green implementation carried repeated types and fixtures that did not earn their maintenance cost	scripts/probe-opencode-eval-metadata.ts; tests/live-opencode-smoke.test.ts	VERIFIED 170 lines removed from the test file and shared capability shapes consolidated
 2026-08-24T22:50:31Z	phase-0	reran the exact paid probe and whole repository gate after simplification	the committed observation must be generated by the final code shape	.agents/plans/02-eval-engineering/evidence/opencode-metadata-probe.json; bun run check	VERIFIED 411 pass, 1 skip, 0 fail
+2026-08-24T22:55:54Z	phase-1	started from the merged Phase 0 main	sequence verifiable units requires a clean verified predecessor before the report cutover begins	git status on codex/eval-phase-1 at 26d2aec	VERIFIED clean baseline, 411 pass, 1 skip, 0 fail
+2026-08-24T22:55:54Z	phase-1	bound scheduled cells to exact case versions	a cell that names only case id cannot detect semantic drift between case revisions	.agents/plans/02-eval-engineering/architecture.md	VERIFIED architecture corrected before code
+2026-08-24T23:49:00Z	phase-1	implemented the strict report and catalog boundary	atomic attempts must fail closed on malformed structure and inconsistent plans	evals/report.ts; evals/catalog.ts; tests/eval-report.test.ts	VERIFIED 19 focused cases
+2026-08-24T23:49:00Z	phase-1	fixed the multi-model Interrogate findings	budgets, terminal state, pair retries, evidence coherence, canonical JSON, and immutable validated values were trust boundaries	.agents/plans/02-eval-engineering/evidence/phase-1-review.md	VERIFIED no unresolved blocker
+2026-08-24T23:49:00Z	phase-1	ran Deslop and the whole repository gate	the phase must finish in a reviewable state without monolithic growth or product regressions	bun run check	VERIFIED 430 pass, 1 skip, 0 fail

evals/canonical-json.ts

@@ -0,0 +1,54 @@
+import { createHash } from "node:crypto";
+
+function canonicalString(value: string): string {
+	if (!value.isWellFormed()) {
+		throw new Error("Canonical JSON requires Unicode scalar values.");
+	}
+	return JSON.stringify(value);
+}
+
+export function canonicalJson(value: unknown): string {
+	if (value === null) return "null";
+	if (typeof value === "string") return canonicalString(value);
+	if (typeof value === "boolean") return JSON.stringify(value);
+	if (typeof value === "number") {
+		if (!Number.isFinite(value)) {
+			throw new Error("Canonical JSON requires finite numbers.");
+		}
+		return JSON.stringify(value);
+	}
+	if (Array.isArray(value)) {
+		if (Object.keys(value).length !== value.length) {
+			throw new Error("Canonical JSON requires dense arrays.");
+		}
+		for (let index = 0; index < value.length; index += 1) {
+			if (!Object.hasOwn(value, index)) {
+				throw new Error("Canonical JSON requires dense arrays.");
+			}
+		}
+		return `[${value.map(canonicalJson).join(",")}]`;
+	}
+	if (typeof value === "object") {
+		const prototype = Object.getPrototypeOf(value);
+		if (
+			(prototype !== Object.prototype && prototype !== null) ||
+			Object.getOwnPropertySymbols(value).length > 0
+		) {
+			throw new Error("Canonical JSON requires plain JSON objects.");
+		}
+		const entries = Object.entries(value).sort(([left], [right]) =>
+			left < right ? -1 : left > right ? 1 : 0,
+		);
+		return `{${entries
+			.map(([key, entry]) => `${canonicalString(key)}:${canonicalJson(entry)}`)
+			.join(",")}}`;
+	}
+	throw new Error("Canonical JSON requires JSON values.");
+}
+
+export function canonicalSha256(domain: string, value: unknown): string {
+	return `sha256:${createHash("sha256")
+		.update(`${domain}\u0000`)
+		.update(canonicalJson(value))
+		.digest("hex")}`;
+}

evals/catalog.ts

@@ -0,0 +1,174 @@
+import { z } from "zod";
+import { type DeepReadonly, freezeTree } from "./validated.js";
+
+const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/);
+const CaseIdSchema = z
+	.string()
+	.min(1)
+	.max(256)
+	.regex(/\S/)
+	.refine((value) => value.isWellFormed());
+const CaseVersionSchema = z.number().int().safe().positive();
+const CountSchema = z.number().int().safe().nonnegative();
+const RateSchema = z.number().finite().min(0).max(1);
+
+export const EvidenceClassSchema = z.enum([
+	"conformance",
+	"regression",
+	"capability",
+	"compatibility",
+	"reviewer-only",
+	"paired-value",
+]);
+
+export const CasePolicySchema = z
+	.object({
+		caseId: CaseIdSchema,
+		caseVersion: CaseVersionSchema,
+		evidenceClass: EvidenceClassSchema,
+		oracle: z.enum([
+			"durable-state",
+			"hidden-executable",
+			"trajectory",
+			"fixed-review-label",
+		]),
+		release: z.enum(["required", "report-only"]),
+		minProviders: CountSchema.positive(),
+		minScoredAttempts: CountSchema.positive(),
+		minPassRate: RateSchema.nullable(),
+		reviewerPromotionRecordSha256: DigestSchema.nullable(),
+	})
+	.strict()
+	.superRefine((policy, context) => {
+		const reviewerPolicy = policy.evidenceClass === "reviewer-only";
+		if (reviewerPolicy !== (policy.oracle === "fixed-review-label")) {
+			context.addIssue({
+				code: "custom",
+				path: ["oracle"],
+				message:
+					"Fixed-review-label oracles are required exactly for reviewer-only cases.",
+			});
+		}
+		if (
+			policy.evidenceClass === "paired-value" &&
+			policy.oracle !== "hidden-executable"
+		) {
+			context.addIssue({
+				code: "custom",
+				path: ["oracle"],
+				message: "Paired-value cases require a hidden-executable oracle.",
+			});
+		}
+		if (policy.release === "required" && policy.minPassRate === null) {
+			context.addIssue({
+				code: "custom",
+				path: ["minPassRate"],
+				message: "Required release cases require a minimum pass rate.",
+			});
+		}
+		if (
+			reviewerPolicy &&
+			policy.release === "required" &&
+			policy.reviewerPromotionRecordSha256 === null
+		) {
+			context.addIssue({
+				code: "custom",
+				path: ["reviewerPromotionRecordSha256"],
+				message:
+					"Required reviewer-only cases require a promotion record digest.",
+			});
+		}
+		if (!reviewerPolicy && policy.reviewerPromotionRecordSha256 !== null) {
+			context.addIssue({
+				code: "custom",
+				path: ["reviewerPromotionRecordSha256"],
+				message:
+					"Only reviewer-only cases may declare a promotion record digest.",
+			});
+		}
+	});
+
+export type EvidenceClass = z.infer<typeof EvidenceClassSchema>;
+export type CasePolicy = z.infer<typeof CasePolicySchema>;
+
+export type CatalogIssue = {
+	readonly path: string;
+	readonly code: "schema" | "missing" | "duplicate" | "policy";
+	readonly message: string;
+};
+
+export const CaseCatalogSchema = z
+	.array(CasePolicySchema)
+	.min(1)
+	.superRefine((catalog, context) => {
+		const known = new Set<string>();
+		for (const [index, policy] of catalog.entries()) {
+			const key = `${policy.caseId}\u0000${policy.caseVersion}`;
+			if (known.has(key)) {
+				context.addIssue({
+					code: "custom",
+					path: [index],
+					message: "Case id and version must be unique.",
+				});
+			}
+			known.add(key);
+		}
+	});
+
+const ValidatedCaseCatalogSchema =
+	CaseCatalogSchema.brand<"ValidatedCaseCatalog">();
+export type ValidatedCaseCatalog = DeepReadonly<
+	z.infer<typeof ValidatedCaseCatalogSchema>
+>;
+
+function pathText(path: readonly PropertyKey[]): string {
+	return path.length === 0 ? "$" : `$.${path.join(".")}`;
+}
+
+function hasPath(input: unknown, path: readonly PropertyKey[]): boolean {
+	let current = input;
+	for (const segment of path) {
+		if (current === null || typeof current !== "object") return false;
+		if (!Object.hasOwn(current, segment)) return false;
+		current = Reflect.get(current, segment);
+	}
+	return true;
+}
+
+function catalogIssue(input: unknown, issue: z.core.$ZodIssue): CatalogIssue {
+	const missing = issue.code === "invalid_type" && !hasPath(input, issue.path);
+	return {
+		path: pathText(issue.path),
+		code: missing
+			? "missing"
+			: issue.message === "Case id and version must be unique."
+				? "duplicate"
+				: issue.code === "custom"
+					? "policy"
+					: "schema",
+		message:
+			issue.message === "Case id and version must be unique." ||
+			issue.code === "custom"
+				? issue.message
+				: missing
+					? "Missing required value."
+					: "Invalid catalog value.",
+	};
+}
+
+export function parseCaseCatalog(
+	input: unknown,
+):
+	| { readonly ok: true; readonly value: ValidatedCaseCatalog }
+	| { readonly ok: false; readonly issues: readonly CatalogIssue[] } {
+	const parsed = CaseCatalogSchema.safeParse(input);
+	if (!parsed.success) {
+		return {
+			ok: false,
+			issues: parsed.error.issues.map((item) => catalogIssue(input, item)),
+		};
+	}
+	const value = ValidatedCaseCatalogSchema.parse(parsed.data);
+	freezeTree(value);
+	return { ok: true, value };
+}