diff --git a/.agents/plans/02-eval-engineering/architecture.md b/.agents/plans/02-eval-engineering/architecture.md index f1eefb5..fd5f83a 100644 --- a/.agents/plans/02-eval-engineering/architecture.md +++ b/.agents/plans/02-eval-engineering/architecture.md @@ -110,8 +110,7 @@ type AttemptRecordV2 = { readonly hostConfigSha256: string; readonly actors: readonly ActorIdentity[]; readonly instructions: readonly InstructionDelivery[]; - readonly transcriptSha256: string; - readonly transcriptArtifact: string; + readonly transcript: { readonly sha256: string; readonly artifact: string } | null; readonly outcome: AttemptOutcome; readonly usage: { readonly durationMs: number; readonly outputTokens: number; readonly costUsd: number | null }; }; @@ -120,6 +119,7 @@ type ScheduledCell = { readonly cellId: string; readonly blockId: string; readonly caseId: string; + readonly caseVersion: number; readonly armToken: string | null; readonly repetition: number; readonly managerModel: ModelIdentity | null; @@ -351,7 +351,10 @@ remain. New code adds evidence semantics and analysis around them. ## Module map -- `evals/report.ts` owns v2 types, strict parsing, integrity checks, and verdicts. +- `evals/report.ts` owns v2 types, strict parsing, integrity checks, and verdicts; + `evals/canonical-json.ts` owns canonical serialization and domain-separated hashes. +- `evals/validated.ts` owns the shared deep-readonly and runtime-freeze boundary. +- `evals/report-pairing.ts` owns paired plan and complete-pair invariants. - `evals/report-store.ts` owns canonical write-once attempt files and finalization. - `evals/provenance.ts` owns source, artifact, evaluator, host, actor, and instruction digests. - `evals/catalog.ts` owns typed case policies and campaign planning. diff --git a/.agents/plans/02-eval-engineering/evidence/phase-1-review.md b/.agents/plans/02-eval-engineering/evidence/phase-1-review.md new file mode 100644 index 0000000..0032df4 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-1-review.md @@ -0,0 +1,60 @@ +# Phase 1 Interrogate review + +## Intent + +Phase 1 must add a fail-closed, schema-derived v2 boundary for frozen campaign +plans, atomic attempts, completion records, and case policy. It must reject +malformed, legacy, summary-only, cross-record, budget, and paired-design drift +without changing the runner or release qualifier. + +## Reviewers + +- `gpt-5.6-terra` reviewed the first implementation and the final corrected diff. +- `gpt-5.6-luna` reviewed the first implementation and the final corrected diff. +- `gpt-5.5` reviewed the first implementation and the final corrected diff. +- `gpt-5.4` reviewed the first implementation and the final corrected diff. + +## Acted on + +- Declared attempt, token, wall-clock, cost, and unknown-cost budgets now constrain + completion state. Wall-clock evidence has timestamp and longest-attempt lower + bounds. +- Complete and stopped states now follow scored outcomes. Failure rows cannot + satisfy fixed targets. +- Paired plans now require complete scored pairs and atomic reserve activation. + Non-paired plans cannot use the paired replacement policy. +- Paired hidden evidence and reviewer fixed-label evidence now agree with their + top-level scored outcome. +- Validated reports and catalogs are deeply readonly and recursively frozen. +- Canonical JSON rejects malformed Unicode, sparse arrays, and non-JSON objects. + Catalog identifiers use the same Unicode scalar boundary. +- Canonicalization, immutable validation, and paired-plan rules moved into small + reusable modules. No changed TypeScript file exceeds 1,000 lines. + +## Deferred by phase boundary + +- Catalog sample floors belong to Phase 2 decision analysis. A structurally valid + under-sampled report must remain analyzable as `INCONCLUSIVE`. +- Expected artifact, evaluator, host, and actor comparison belongs to Phase 3. + Paired campaigns intentionally contain different candidate and baseline + artifacts, so one report-wide artifact equality rule would be wrong. + +## Dismissed + +- Opaque arm tokens are treatment keys and are intentionally reused across + blocks. They must differ within a block, not across the full plan. +- A budget stop may occur below an observed ceiling when the next atomic attempt + cannot safely start. Exact budget exhaustion is not a valid completion + invariant. +- An unsubmitted reviewer result is valid failed evidence, not malformed data. + The explicit nullable verdict and submitted flag preserve it for Phase 2 + qualification. +- Global collection limits do not protect the JSON parse allocation and would + introduce arbitrary campaign caps. Frozen plan budgets constrain executable + work instead. + +## Verdict + +`VERIFIED`. The final four-model recheck found no unresolved blocker after lead +judgment. Focused tests pass with 19 cases. The full repository gate passes with +430 tests, one intentional live-host skip, and zero failures. diff --git a/.agents/plans/02-eval-engineering/phase-1-report-boundary.md b/.agents/plans/02-eval-engineering/phase-1-report-boundary.md index 24e2657..9436abf 100644 --- a/.agents/plans/02-eval-engineering/phase-1-report-boundary.md +++ b/.agents/plans/02-eval-engineering/phase-1-report-boundary.md @@ -12,11 +12,18 @@ finalization record, and strict structural and semantic validation boundary. - `evals/report.ts`. Add v2 report, campaign, cell, attempt, disposition, evidence-payload, completion, provenance, actor, and structured issue types plus strict structural parsing. +- `evals/canonical-json.ts`. Add the reusable RFC 8785-compatible canonical JSON + and domain-separated digest primitive used by frozen plans. +- `evals/validated.ts`. Keep validated boundary values deeply readonly in types + and recursively frozen at runtime. +- `evals/report-pairing.ts`. Isolate complete-pair shape, retry, activation, and + scored-target invariants from the general report boundary. - `evals/catalog.ts`. Add the minimal case identity, evidence class, oracle, sample floor, release disposition, and analysis policy needed for semantic validation. - `tests/eval-report.test.ts`. Cover valid round trips, missing fields, duplicate ids, unknown cases against a supplied catalog, inconsistent pairs, numeric - bounds, finalization causes, non-finite counters, and the summary-only exploit. + bounds, cell-to-case-version drift, finalization causes, non-finite counters, + transcript availability by outcome, and the summary-only exploit. ## Data structures diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index 59c035e..fea2217 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -21,3 +21,8 @@ ts phase decision why evidence result 2026-08-24T22:35:43Z phase-0 reran the full repository gate after review fixes accepted review changes require the same whole-product proof as the original slice bun run check VERIFIED 404 pass, 1 skip, 0 fail 2026-08-24T22:50:31Z phase-0 simplified the probe and tests before commit the first green implementation carried repeated types and fixtures that did not earn their maintenance cost scripts/probe-opencode-eval-metadata.ts; tests/live-opencode-smoke.test.ts VERIFIED 170 lines removed from the test file and shared capability shapes consolidated 2026-08-24T22:50:31Z phase-0 reran the exact paid probe and whole repository gate after simplification the committed observation must be generated by the final code shape .agents/plans/02-eval-engineering/evidence/opencode-metadata-probe.json; bun run check VERIFIED 411 pass, 1 skip, 0 fail +2026-08-24T22:55:54Z phase-1 started from the merged Phase 0 main sequence verifiable units requires a clean verified predecessor before the report cutover begins git status on codex/eval-phase-1 at 26d2aec VERIFIED clean baseline, 411 pass, 1 skip, 0 fail +2026-08-24T22:55:54Z phase-1 bound scheduled cells to exact case versions a cell that names only case id cannot detect semantic drift between case revisions .agents/plans/02-eval-engineering/architecture.md VERIFIED architecture corrected before code +2026-08-24T23:49:00Z phase-1 implemented the strict report and catalog boundary atomic attempts must fail closed on malformed structure and inconsistent plans evals/report.ts; evals/catalog.ts; tests/eval-report.test.ts VERIFIED 19 focused cases +2026-08-24T23:49:00Z phase-1 fixed the multi-model Interrogate findings budgets, terminal state, pair retries, evidence coherence, canonical JSON, and immutable validated values were trust boundaries .agents/plans/02-eval-engineering/evidence/phase-1-review.md VERIFIED no unresolved blocker +2026-08-24T23:49:00Z phase-1 ran Deslop and the whole repository gate the phase must finish in a reviewable state without monolithic growth or product regressions bun run check VERIFIED 430 pass, 1 skip, 0 fail diff --git a/evals/canonical-json.ts b/evals/canonical-json.ts new file mode 100644 index 0000000..8ad22df --- /dev/null +++ b/evals/canonical-json.ts @@ -0,0 +1,54 @@ +import { createHash } from "node:crypto"; + +function canonicalString(value: string): string { + if (!value.isWellFormed()) { + throw new Error("Canonical JSON requires Unicode scalar values."); + } + return JSON.stringify(value); +} + +export function canonicalJson(value: unknown): string { + if (value === null) return "null"; + if (typeof value === "string") return canonicalString(value); + if (typeof value === "boolean") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error("Canonical JSON requires finite numbers."); + } + return JSON.stringify(value); + } + if (Array.isArray(value)) { + if (Object.keys(value).length !== value.length) { + throw new Error("Canonical JSON requires dense arrays."); + } + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) { + throw new Error("Canonical JSON requires dense arrays."); + } + } + return `[${value.map(canonicalJson).join(",")}]`; + } + if (typeof value === "object") { + const prototype = Object.getPrototypeOf(value); + if ( + (prototype !== Object.prototype && prototype !== null) || + Object.getOwnPropertySymbols(value).length > 0 + ) { + throw new Error("Canonical JSON requires plain JSON objects."); + } + const entries = Object.entries(value).sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + ); + return `{${entries + .map(([key, entry]) => `${canonicalString(key)}:${canonicalJson(entry)}`) + .join(",")}}`; + } + throw new Error("Canonical JSON requires JSON values."); +} + +export function canonicalSha256(domain: string, value: unknown): string { + return `sha256:${createHash("sha256") + .update(`${domain}\u0000`) + .update(canonicalJson(value)) + .digest("hex")}`; +} diff --git a/evals/catalog.ts b/evals/catalog.ts new file mode 100644 index 0000000..1238b8f --- /dev/null +++ b/evals/catalog.ts @@ -0,0 +1,174 @@ +import { z } from "zod"; +import { type DeepReadonly, freezeTree } from "./validated.js"; + +const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const CaseIdSchema = z + .string() + .min(1) + .max(256) + .regex(/\S/) + .refine((value) => value.isWellFormed()); +const CaseVersionSchema = z.number().int().safe().positive(); +const CountSchema = z.number().int().safe().nonnegative(); +const RateSchema = z.number().finite().min(0).max(1); + +export const EvidenceClassSchema = z.enum([ + "conformance", + "regression", + "capability", + "compatibility", + "reviewer-only", + "paired-value", +]); + +export const CasePolicySchema = z + .object({ + caseId: CaseIdSchema, + caseVersion: CaseVersionSchema, + evidenceClass: EvidenceClassSchema, + oracle: z.enum([ + "durable-state", + "hidden-executable", + "trajectory", + "fixed-review-label", + ]), + release: z.enum(["required", "report-only"]), + minProviders: CountSchema.positive(), + minScoredAttempts: CountSchema.positive(), + minPassRate: RateSchema.nullable(), + reviewerPromotionRecordSha256: DigestSchema.nullable(), + }) + .strict() + .superRefine((policy, context) => { + const reviewerPolicy = policy.evidenceClass === "reviewer-only"; + if (reviewerPolicy !== (policy.oracle === "fixed-review-label")) { + context.addIssue({ + code: "custom", + path: ["oracle"], + message: + "Fixed-review-label oracles are required exactly for reviewer-only cases.", + }); + } + if ( + policy.evidenceClass === "paired-value" && + policy.oracle !== "hidden-executable" + ) { + context.addIssue({ + code: "custom", + path: ["oracle"], + message: "Paired-value cases require a hidden-executable oracle.", + }); + } + if (policy.release === "required" && policy.minPassRate === null) { + context.addIssue({ + code: "custom", + path: ["minPassRate"], + message: "Required release cases require a minimum pass rate.", + }); + } + if ( + reviewerPolicy && + policy.release === "required" && + policy.reviewerPromotionRecordSha256 === null + ) { + context.addIssue({ + code: "custom", + path: ["reviewerPromotionRecordSha256"], + message: + "Required reviewer-only cases require a promotion record digest.", + }); + } + if (!reviewerPolicy && policy.reviewerPromotionRecordSha256 !== null) { + context.addIssue({ + code: "custom", + path: ["reviewerPromotionRecordSha256"], + message: + "Only reviewer-only cases may declare a promotion record digest.", + }); + } + }); + +export type EvidenceClass = z.infer; +export type CasePolicy = z.infer; + +export type CatalogIssue = { + readonly path: string; + readonly code: "schema" | "missing" | "duplicate" | "policy"; + readonly message: string; +}; + +export const CaseCatalogSchema = z + .array(CasePolicySchema) + .min(1) + .superRefine((catalog, context) => { + const known = new Set(); + for (const [index, policy] of catalog.entries()) { + const key = `${policy.caseId}\u0000${policy.caseVersion}`; + if (known.has(key)) { + context.addIssue({ + code: "custom", + path: [index], + message: "Case id and version must be unique.", + }); + } + known.add(key); + } + }); + +const ValidatedCaseCatalogSchema = + CaseCatalogSchema.brand<"ValidatedCaseCatalog">(); +export type ValidatedCaseCatalog = DeepReadonly< + z.infer +>; + +function pathText(path: readonly PropertyKey[]): string { + return path.length === 0 ? "$" : `$.${path.join(".")}`; +} + +function hasPath(input: unknown, path: readonly PropertyKey[]): boolean { + let current = input; + for (const segment of path) { + if (current === null || typeof current !== "object") return false; + if (!Object.hasOwn(current, segment)) return false; + current = Reflect.get(current, segment); + } + return true; +} + +function catalogIssue(input: unknown, issue: z.core.$ZodIssue): CatalogIssue { + const missing = issue.code === "invalid_type" && !hasPath(input, issue.path); + return { + path: pathText(issue.path), + code: missing + ? "missing" + : issue.message === "Case id and version must be unique." + ? "duplicate" + : issue.code === "custom" + ? "policy" + : "schema", + message: + issue.message === "Case id and version must be unique." || + issue.code === "custom" + ? issue.message + : missing + ? "Missing required value." + : "Invalid catalog value.", + }; +} + +export function parseCaseCatalog( + input: unknown, +): + | { readonly ok: true; readonly value: ValidatedCaseCatalog } + | { readonly ok: false; readonly issues: readonly CatalogIssue[] } { + const parsed = CaseCatalogSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + issues: parsed.error.issues.map((item) => catalogIssue(input, item)), + }; + } + const value = ValidatedCaseCatalogSchema.parse(parsed.data); + freezeTree(value); + return { ok: true, value }; +} diff --git a/evals/report-pairing.ts b/evals/report-pairing.ts new file mode 100644 index 0000000..12193b5 --- /dev/null +++ b/evals/report-pairing.ts @@ -0,0 +1,128 @@ +import { canonicalJson } from "./canonical-json.js"; +import type { EvalReportV2 } from "./report.js"; + +type PairingIssue = { + readonly path: string; + readonly code: "pair" | "policy"; + readonly message: string; +}; + +export function validatePairing( + report: EvalReportV2, + primaryCells: readonly EvalReportV2["plan"]["cells"][number][], + productCells: ReadonlySet, + activatedReserveCells: ReadonlySet, +): { + readonly issues: readonly PairingIssue[]; + readonly scoredOutcomes: number; +} { + const issues: PairingIssue[] = []; + const paired = report.plan.analysis.kind === "paired"; + const add = (path: string, code: PairingIssue["code"], message: string) => + issues.push({ path, code, message }); + + if (paired !== (report.plan.stoppingRule.kind === "fixed-complete-pairs")) { + add( + "$.plan.stoppingRule.kind", + "pair", + "Paired campaigns require fixed-complete-pairs stopping; other campaigns require fixed-attempts.", + ); + } + const primaryBlocks = paired + ? new Set(primaryCells.map((cell) => cell.blockId)).size + : 0; + if ( + report.plan.stoppingRule.kind === "fixed-complete-pairs" && + report.plan.stoppingRule.count !== primaryBlocks + ) { + add( + "$.plan.stoppingRule.count", + "policy", + "Fixed-complete-pairs count must equal the primary paired block count.", + ); + } + if (paired !== (report.allocationCommitmentSha256 !== null)) { + add( + "$.allocationCommitmentSha256", + "pair", + "Allocation commitment is required exactly for paired campaigns.", + ); + } + if (!paired) { + if (report.plan.cells.some((cell) => cell.armToken !== null)) { + add( + "$.plan.cells", + "pair", + "Non-paired campaigns cannot declare arm tokens.", + ); + } + if ( + report.plan.abortPolicy.retry !== "never" || + report.plan.cells.some((cell) => cell.schedule === "replacement-reserve") + ) { + add( + "$.plan.abortPolicy", + "pair", + "Non-paired campaigns cannot declare replacement retries.", + ); + } + return { issues, scoredOutcomes: productCells.size }; + } + + let scoredOutcomes = 0; + const blocks = new Map(); + for (const cell of report.plan.cells) { + const block = blocks.get(cell.blockId) ?? []; + block.push(cell); + blocks.set(cell.blockId, block); + } + for (const [blockId, block] of blocks) { + const first = block[0]; + const second = block[1]; + if ( + block.length !== 2 || + first === undefined || + second === undefined || + first.armToken === null || + second.armToken === null || + first.managerModel === null || + second.managerModel === null || + first.armToken === second.armToken || + first.caseId !== second.caseId || + first.caseVersion !== second.caseVersion || + first.repetition !== second.repetition || + first.schedule !== second.schedule || + canonicalJson(first.managerModel) !== + canonicalJson(second.managerModel) || + canonicalJson(first.reviewerModel) !== canonicalJson(second.reviewerModel) + ) { + add("$.plan.cells", "pair", `Invalid paired block ${blockId}.`); + } + if ( + block.length === 2 && + block.every((cell) => productCells.has(cell.cellId)) + ) { + scoredOutcomes += 1; + } + if ( + first?.schedule === "replacement-reserve" && + second?.schedule === "replacement-reserve" && + activatedReserveCells.has(first.cellId) !== + activatedReserveCells.has(second.cellId) + ) { + add( + "$.completion.activatedReserveCellIds", + "pair", + `Replacement block ${blockId} must be activated as a whole pair.`, + ); + } + } + if (report.plan.abortPolicy.retry !== "whole-pair") { + add( + "$.plan.abortPolicy.retry", + "pair", + "Paired campaigns require whole-pair retries.", + ); + } + return { issues, scoredOutcomes }; +} diff --git a/evals/report.ts b/evals/report.ts new file mode 100644 index 0000000..80c86fc --- /dev/null +++ b/evals/report.ts @@ -0,0 +1,917 @@ +import { z } from "zod"; +import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; +import type { CasePolicy, ValidatedCaseCatalog } from "./catalog.js"; +import { validatePairing } from "./report-pairing.js"; +import { type DeepReadonly, freezeTree } from "./validated.js"; + +const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const TextSchema = z + .string() + .min(1) + .max(4096) + .regex(/\S/) + .refine((value) => value.isWellFormed()); +const CountSchema = z.number().int().safe().nonnegative(); +const PositiveCountSchema = CountSchema.positive(); +const RateSchema = z.number().finite().min(0).max(1); +const TimestampSchema = z.string().datetime({ offset: true }); +const RelativeTranscriptArtifactSchema = TextSchema.superRefine( + (path, context) => { + if ( + path.startsWith("/") || + path.startsWith("\\") || + /^[A-Za-z]:[\\/]/.test(path) || + path.split(/[\\/]+/).includes("..") + ) { + context.addIssue({ + code: "custom", + message: "Invalid transcript artifact.", + }); + } + }, +); + +const ModelIdentitySchema = z + .object({ + routeProvider: TextSchema, + gateway: TextSchema.nullable(), + family: TextSchema, + model: TextSchema, + revision: TextSchema.nullable(), + }) + .strict(); + +const ObservedModelIdentitySchema = z.discriminatedUnion("kind", [ + z + .object({ kind: z.literal("observed"), value: ModelIdentitySchema }) + .strict(), + z.object({ kind: z.literal("unobserved"), reason: TextSchema }).strict(), +]); + +const ArtifactIdentitySchema = z + .object({ + packageVersion: TextSchema, + sourceCommit: TextSchema, + sourceTreeSha256: DigestSchema, + tarballSha256: DigestSchema, + unpackedManifestSha256: DigestSchema, + }) + .strict(); + +const EvaluatorIdentitySchema = z + .object({ + sourceCommit: TextSchema, + caseCatalogSha256: DigestSchema, + policyCatalogSha256: DigestSchema, + graderBundleSha256: DigestSchema, + }) + .strict(); + +const ActorIdentitySchema = z + .object({ + role: z.enum(["manager", "reviewer"]), + requestedModel: ModelIdentitySchema, + actualModel: ObservedModelIdentitySchema, + sessionIds: z.array(TextSchema).min(1), + }) + .strict(); + +const InstructionDeliverySchema = z + .object({ + source: z.enum(["command", "agent", "guidance", "continuation"]), + name: TextSchema, + sequence: CountSchema, + sha256: DigestSchema, + bytes: CountSchema, + }) + .strict(); + +const FactsSchema = z.record( + TextSchema, + z.union([z.boolean(), z.number().finite(), TextSchema]), +); + +const ProductEvidenceSchema = z.discriminatedUnion("kind", [ + z + .object({ + kind: z.literal("conformance"), + falseCompletion: z.boolean(), + unsubmittedReviews: CountSchema, + facts: FactsSchema, + }) + .strict(), + z + .object({ + kind: z.literal("regression"), + falseCompletion: z.boolean(), + unsubmittedReviews: CountSchema, + facts: FactsSchema, + }) + .strict(), + z + .object({ + kind: z.literal("capability"), + falseCompletion: z.boolean(), + unsubmittedReviews: CountSchema, + facts: FactsSchema, + }) + .strict(), + z + .object({ + kind: z.literal("reviewer-only"), + truth: z.enum(["defect", "clean"]), + verdict: z.enum(["passed", "failed"]).nullable(), + findings: z.array(TextSchema), + submitted: z.boolean(), + }) + .strict(), + z + .object({ + kind: z.literal("paired-value"), + hiddenCorrectness: z.boolean(), + claimedComplete: z.boolean(), + falseCompletion: z.boolean(), + }) + .strict(), + z + .object({ + kind: z.literal("compatibility"), + checks: z.record(TextSchema, z.boolean()), + }) + .strict(), +]); + +const AttemptOutcomeSchema = z.discriminatedUnion("kind", [ + z + .object({ + kind: z.literal("product"), + passed: z.boolean(), + endedBy: z.enum(["quiet", "user-escalation"]), + issues: z.array(TextSchema), + evidence: ProductEvidenceSchema, + }) + .strict(), + z + .object({ kind: z.literal("unscored-escalation"), reason: TextSchema }) + .strict(), + z + .object({ + kind: z.literal("failure"), + origin: z.enum(["evaluator", "host", "provider"]), + code: TextSchema, + retryable: z.boolean(), + }) + .strict(), +]); + +const ScheduledCellSchema = z + .object({ + cellId: TextSchema, + blockId: TextSchema, + caseId: TextSchema, + caseVersion: PositiveCountSchema, + armToken: TextSchema.nullable(), + repetition: CountSchema, + managerModel: ModelIdentitySchema.nullable(), + reviewerModel: ModelIdentitySchema.nullable(), + schedule: z.enum(["primary", "replacement-reserve"]), + }) + .strict(); + +const AnalysisPolicySchema = z.discriminatedUnion("kind", [ + z + .object({ + kind: z.literal("rate"), + primaryOutcome: TextSchema, + versionSha256: DigestSchema, + }) + .strict(), + z + .object({ + kind: z.literal("reviewer"), + interval: z.literal("wilson"), + alpha: z.literal(0.05), + versionSha256: DigestSchema, + }) + .strict(), + z + .object({ + kind: z.literal("paired"), + primaryOutcome: z.literal("hidden-correctness"), + estimand: z.literal("candidate-minus-baseline-risk-difference"), + interval: z.literal("task-stratified-paired-bootstrap"), + alpha: z.literal(0.05), + targetPower: RateSchema.positive(), + minimumDetectableEffect: RateSchema.positive(), + tieRule: z.literal("zero-difference"), + bootstrapSeed: TextSchema, + versionSha256: DigestSchema, + }) + .strict(), +]); + +export const CampaignPlanSchema = z + .object({ + schemaVersion: z.literal(1), + planId: TextSchema, + planSha256: DigestSchema, + randomizationSeed: TextSchema, + cells: z.array(ScheduledCellSchema).min(1), + abortPolicy: z + .object({ + retry: z.enum(["whole-pair", "never"]), + maxReplacementBlocks: CountSchema, + }) + .strict(), + stoppingRule: z + .object({ + kind: z.enum(["fixed-attempts", "fixed-complete-pairs"]), + count: PositiveCountSchema, + }) + .strict(), + analysis: AnalysisPolicySchema, + budget: z + .object({ + maxUsd: z.number().finite().nonnegative().nullable(), + unknownCostPolicy: z.enum(["stop", "token-wall-clock-bounds"]), + maxOutputTokens: CountSchema, + maxWallClockMs: CountSchema, + maxAttempts: PositiveCountSchema, + }) + .strict(), + }) + .strict(); + +const AttemptRecordSchema = z + .object({ + schemaVersion: z.literal(2), + attemptId: TextSchema, + cellId: TextSchema, + blockId: TextSchema.nullable(), + caseId: TextSchema, + caseVersion: PositiveCountSchema, + armToken: TextSchema.nullable(), + repetition: CountSchema, + artifact: z.union([ + ArtifactIdentitySchema, + z.object({ kind: z.literal("ordinary-opencode") }).strict(), + ]), + evaluator: EvaluatorIdentitySchema, + hostConfigSha256: DigestSchema, + actors: z.array(ActorIdentitySchema), + instructions: z.array(InstructionDeliverySchema), + transcript: z + .object({ + sha256: DigestSchema, + artifact: RelativeTranscriptArtifactSchema, + }) + .strict() + .nullable(), + outcome: AttemptOutcomeSchema, + usage: z + .object({ + durationMs: CountSchema, + outputTokens: CountSchema, + costUsd: z.number().finite().nonnegative().nullable(), + }) + .strict(), + }) + .strict(); + +const CampaignCompletionSchema = z + .object({ + status: z.enum(["complete", "stopped"]), + cause: z.enum([ + "fixed-target", + "budget", + "provider", + "host", + "evaluator", + "operator", + ]), + startedAt: TimestampSchema, + finishedAt: TimestampSchema, + activatedReserveCellIds: z.array(TextSchema), + observed: z + .object({ + attempts: CountSchema, + outputTokens: CountSchema, + costUsd: z.number().finite().nonnegative().nullable(), + wallClockMs: CountSchema, + }) + .strict(), + }) + .strict(); + +export const EvalReportV2Schema = z + .object({ + schemaVersion: z.literal(2), + reportId: TextSchema, + plan: CampaignPlanSchema, + attempts: z.array(AttemptRecordSchema), + completion: CampaignCompletionSchema, + allocationCommitmentSha256: DigestSchema.nullable(), + }) + .strict(); + +const ValidatedReportSchema = EvalReportV2Schema.brand<"ValidatedReport">(); + +export type ModelIdentity = z.infer; +export type ObservedModelIdentity = z.infer; +export type ArtifactIdentity = z.infer; +export type EvaluatorIdentity = z.infer; +export type ActorIdentity = z.infer; +export type InstructionDelivery = z.infer; +export type ProductEvidence = z.infer; +export type AttemptOutcome = z.infer; +export type ScheduledCell = z.infer; +export type AnalysisPolicy = z.infer; +export type CampaignPlan = z.infer; +export type AttemptRecordV2 = z.infer; +export type CampaignCompletion = z.infer; +export type EvalReportV2 = z.infer; +export type ValidatedReport = DeepReadonly< + z.infer +>; + +export type ReportIssue = { + readonly path: string; + readonly code: + | "schema" + | "missing" + | "duplicate" + | "hash" + | "provenance" + | "pair" + | "policy"; + readonly message: string; +}; + +type SemanticIssue = Omit & { + readonly code: Exclude; +}; + +function pathText(path: readonly PropertyKey[]): string { + return path.length === 0 ? "$" : `$.${path.join(".")}`; +} + +function hasPath(input: unknown, path: readonly PropertyKey[]): boolean { + let current = input; + for (const segment of path) { + if (current === null || typeof current !== "object") return false; + if (!Object.hasOwn(current, segment)) return false; + current = Reflect.get(current, segment); + } + return true; +} + +function schemaIssue(input: unknown, issue: z.core.$ZodIssue): ReportIssue { + const missing = issue.code === "invalid_type" && !hasPath(input, issue.path); + return { + path: pathText(issue.path), + code: missing ? "missing" : "schema", + message: missing ? "Missing required value." : "Invalid report value.", + }; +} + +export function campaignPlanSha256(plan: CampaignPlan): string { + const { planSha256: _planSha256, ...withoutHash } = plan; + return canonicalSha256("flow-campaign-plan-v1", withoutHash); +} +function issue( + issues: SemanticIssue[], + path: string, + code: SemanticIssue["code"], + message: string, +): void { + issues.push({ path, code, message }); +} +function duplicateIssues( + issues: SemanticIssue[], + values: readonly T[], + path: string, + label: string, + id: (value: T) => string, +): void { + const known = new Set(); + for (const [index, value] of values.entries()) { + if (known.has(id(value))) { + issue(issues, `${path}.${index}`, "duplicate", `Duplicate ${label}.`); + } + known.add(id(value)); + } +} +function semanticIssues( + report: EvalReportV2, + catalog: ValidatedCaseCatalog, +): readonly SemanticIssue[] { + const issues: SemanticIssue[] = []; + const cases = new Map(); + for (const policy of catalog) { + cases.set(`${policy.caseId}\u0000${policy.caseVersion}`, policy); + } + const cells = new Map(); + duplicateIssues( + issues, + report.plan.cells, + "$.plan.cells", + "plan cell id", + (cell) => cell.cellId, + ); + duplicateIssues( + issues, + report.attempts, + "$.attempts", + "attempt id", + (attempt) => attempt.attemptId, + ); + duplicateIssues( + issues, + report.completion.activatedReserveCellIds, + "$.completion.activatedReserveCellIds", + "activated reserve cell id", + (cellId) => cellId, + ); + const activatedReserveCells = new Set( + report.completion.activatedReserveCellIds, + ); + const attemptedCells = new Set(); + for (const cell of report.plan.cells) cells.set(cell.cellId, cell); + if (campaignPlanSha256(report.plan) !== report.plan.planSha256) { + issue( + issues, + "$.plan.planSha256", + "hash", + "Plan hash does not match its canonical plan.", + ); + } + for (const [index, cell] of report.plan.cells.entries()) { + if (!cases.has(`${cell.caseId}\u0000${cell.caseVersion}`)) { + issue( + issues, + `$.plan.cells.${index}`, + "policy", + "Unknown case id and version.", + ); + } + } + for (const [index, attempt] of report.attempts.entries()) { + const base = `$.attempts.${index}`; + if (attemptedCells.has(attempt.cellId)) { + issue( + issues, + `${base}.cellId`, + "duplicate", + "Duplicate attempt cell id.", + ); + } + attemptedCells.add(attempt.cellId); + const cell = cells.get(attempt.cellId); + if (!cell) { + issue( + issues, + `${base}.cellId`, + "policy", + "Attempt references an unknown plan cell.", + ); + continue; + } + if ( + cell.schedule === "replacement-reserve" && + !activatedReserveCells.has(cell.cellId) + ) { + issue( + issues, + `${base}.cellId`, + "policy", + "Reserve attempts require explicit activation.", + ); + } + if ( + attempt.caseId !== cell.caseId || + attempt.caseVersion !== cell.caseVersion || + attempt.blockId !== cell.blockId || + attempt.armToken !== cell.armToken || + attempt.repetition !== cell.repetition + ) { + issue( + issues, + base, + "policy", + "Attempt does not match its scheduled cell.", + ); + } + const policy = cases.get(`${attempt.caseId}\u0000${attempt.caseVersion}`); + if (!policy) { + issue(issues, `${base}.caseId`, "policy", "Unknown case id and version."); + } else if (attempt.outcome.kind === "product") { + if (policy.evidenceClass !== attempt.outcome.evidence.kind) { + issue( + issues, + `${base}.outcome.evidence.kind`, + "policy", + "Product evidence kind is incompatible with the case evidence class.", + ); + } + if (attempt.outcome.passed && attempt.outcome.issues.length > 0) { + issue( + issues, + `${base}.outcome.issues`, + "policy", + "Passed product attempts cannot carry issues.", + ); + } + if (!attempt.outcome.passed && attempt.outcome.issues.length === 0) { + issue( + issues, + `${base}.outcome.issues`, + "policy", + "Failed product attempts require at least one issue.", + ); + } + if ( + attempt.outcome.evidence.kind === "reviewer-only" && + attempt.outcome.evidence.submitted !== + (attempt.outcome.evidence.verdict !== null) + ) { + issue( + issues, + `${base}.outcome.evidence.verdict`, + "policy", + "Reviewer submission and verdict must be present together.", + ); + } + if (attempt.outcome.evidence.kind === "reviewer-only") { + const correctVerdict = + attempt.outcome.evidence.verdict !== null && + (attempt.outcome.evidence.truth === "defect" + ? attempt.outcome.evidence.verdict === "failed" + : attempt.outcome.evidence.verdict === "passed"); + if (attempt.outcome.passed !== correctVerdict) { + issue( + issues, + `${base}.outcome.passed`, + "policy", + "Reviewer outcome must agree with the fixed truth label and verdict.", + ); + } + } + if ( + attempt.outcome.evidence.kind === "paired-value" && + (attempt.outcome.passed !== + attempt.outcome.evidence.hiddenCorrectness || + attempt.outcome.evidence.falseCompletion !== + (attempt.outcome.evidence.claimedComplete && + !attempt.outcome.evidence.hiddenCorrectness)) + ) { + issue( + issues, + `${base}.outcome.evidence`, + "policy", + "Paired product outcome must agree with its hidden evidence.", + ); + } + } + const roles = new Set(); + for (const actor of attempt.actors) { + if (roles.has(actor.role)) { + issue( + issues, + `${base}.actors`, + "provenance", + "Actor roles must be unique.", + ); + } + roles.add(actor.role); + const expectedModel = + actor.role === "manager" ? cell.managerModel : cell.reviewerModel; + if ( + expectedModel !== null && + canonicalJson(expectedModel) !== canonicalJson(actor.requestedModel) + ) { + issue( + issues, + `${base}.actors`, + "provenance", + `Requested ${actor.role} model does not match its scheduled cell.`, + ); + } + } + const sequences = new Set(); + for (const instruction of attempt.instructions) { + if (sequences.has(instruction.sequence)) { + issue( + issues, + `${base}.instructions`, + "provenance", + "Instruction sequences must be unique.", + ); + } + sequences.add(instruction.sequence); + } + if (attempt.outcome.kind !== "failure") { + if (attempt.transcript === null) { + issue( + issues, + `${base}.transcript`, + "provenance", + "Product and escalation attempts require a transcript artifact.", + ); + } + if (attempt.actors.length === 0 || attempt.instructions.length === 0) { + issue( + issues, + base, + "provenance", + "Product and unscored attempts require actors and instructions.", + ); + } + const requiredRole = + attempt.outcome.kind === "product" && + policy?.evidenceClass === "reviewer-only" + ? "reviewer" + : "manager"; + if (!roles.has(requiredRole)) { + issue( + issues, + `${base}.actors`, + "provenance", + `Attempt requires a ${requiredRole} actor.`, + ); + } + } + } + for (const [ + index, + cellId, + ] of report.completion.activatedReserveCellIds.entries()) { + const cell = cells.get(cellId); + if (cell?.schedule !== "replacement-reserve") { + issue( + issues, + `$.completion.activatedReserveCellIds.${index}`, + "policy", + "Activated reserve ids must reference replacement reserve cells.", + ); + } + } + if ( + report.completion.status === "complete" && + report.completion.cause !== "fixed-target" + ) { + issue( + issues, + "$.completion.cause", + "policy", + "Complete campaigns require fixed-target cause.", + ); + } + if ( + report.completion.status === "stopped" && + report.completion.cause === "fixed-target" + ) { + issue( + issues, + "$.completion.cause", + "policy", + "Stopped campaigns require a stop cause.", + ); + } + if (report.completion.observed.attempts !== report.attempts.length) { + issue( + issues, + "$.completion.observed.attempts", + "policy", + "Observed attempt count must equal ledger length.", + ); + } + const outputTokens = report.attempts.reduce( + (total, attempt) => total + attempt.usage.outputTokens, + 0, + ); + if (report.completion.observed.outputTokens !== outputTokens) { + issue( + issues, + "$.completion.observed.outputTokens", + "policy", + "Observed output tokens must equal the attempt total.", + ); + } + const costUsd = report.attempts.some( + (attempt) => attempt.usage.costUsd === null, + ) + ? null + : report.attempts.reduce( + (total, attempt) => total + (attempt.usage.costUsd ?? 0), + 0, + ); + const costMatches = + costUsd === null + ? report.completion.observed.costUsd === null + : report.completion.observed.costUsd !== null && + Math.abs(report.completion.observed.costUsd - costUsd) <= 1e-9; + if (!costMatches) { + issue( + issues, + "$.completion.observed.costUsd", + "policy", + "Observed cost must be null for unknown attempt costs or equal their total.", + ); + } + const longestAttemptMs = report.attempts.reduce( + (longest, attempt) => Math.max(longest, attempt.usage.durationMs), + 0, + ); + const startedAtMs = Date.parse(report.completion.startedAt); + const finishedAtMs = Date.parse(report.completion.finishedAt); + const elapsedMs = Math.max(0, finishedAtMs - startedAtMs); + if ( + report.completion.observed.wallClockMs < + Math.max(longestAttemptMs, elapsedMs) + ) { + issue( + issues, + "$.completion.observed.wallClockMs", + "policy", + "Observed wall clock cannot be shorter than elapsed campaign time or its longest attempt.", + ); + } + if (startedAtMs > finishedAtMs) { + issue( + issues, + "$.completion.finishedAt", + "policy", + "Finished time must not precede started time.", + ); + } + const primaryCells = report.plan.cells.filter( + (cell) => cell.schedule === "primary", + ); + const productCells = new Set( + report.attempts + .filter((attempt) => attempt.outcome.kind === "product") + .map((attempt) => attempt.cellId), + ); + if (report.attempts.length > report.plan.budget.maxAttempts) { + issue( + issues, + "$.attempts", + "policy", + "Attempt ledger exceeds the campaign attempt budget.", + ); + } + if (report.completion.status === "complete") { + const requiredCells = [ + ...primaryCells.map((cell) => cell.cellId), + ...activatedReserveCells, + ]; + for (const cellId of requiredCells) { + if (!attemptedCells.has(cellId)) { + issue( + issues, + "$.attempts", + "policy", + `Complete campaign is missing attempt for cell ${cellId}.`, + ); + } + } + } + const exceedsKnownBudget = + report.completion.observed.outputTokens > + report.plan.budget.maxOutputTokens || + report.completion.observed.wallClockMs > + report.plan.budget.maxWallClockMs || + (report.plan.budget.maxUsd !== null && + report.completion.observed.costUsd !== null && + report.completion.observed.costUsd > report.plan.budget.maxUsd); + const unknownCostRequiresStop = + report.plan.budget.maxUsd !== null && + report.completion.observed.costUsd === null && + report.plan.budget.unknownCostPolicy === "stop"; + const budgetRequiresStop = exceedsKnownBudget || unknownCostRequiresStop; + if ( + budgetRequiresStop && + (report.completion.status !== "stopped" || + report.completion.cause !== "budget") + ) { + issue( + issues, + "$.completion", + "policy", + "Exceeded or unverifiable budget requires a budget stop.", + ); + } + const reserveBlocks = new Set( + report.plan.cells + .filter((cell) => cell.schedule === "replacement-reserve") + .map((cell) => cell.blockId), + ); + if (reserveBlocks.size !== report.plan.abortPolicy.maxReplacementBlocks) { + issue( + issues, + "$.plan.abortPolicy.maxReplacementBlocks", + "policy", + "Replacement block limit must equal the preallocated reserve block count.", + ); + } + if ( + report.plan.abortPolicy.retry === "never" && + report.plan.abortPolicy.maxReplacementBlocks !== 0 + ) { + issue( + issues, + "$.plan.abortPolicy", + "policy", + "Never-retry campaigns cannot preallocate replacement blocks.", + ); + } + if (report.plan.budget.maxAttempts < primaryCells.length) { + issue( + issues, + "$.plan.budget.maxAttempts", + "policy", + "Maximum attempts cannot be below the primary cell count.", + ); + } + if ( + report.plan.stoppingRule.kind === "fixed-attempts" && + report.plan.stoppingRule.count !== primaryCells.length + ) { + issue( + issues, + "$.plan.stoppingRule.count", + "policy", + "Fixed-attempt count must equal the primary cell count.", + ); + } + for (const [index, cell] of report.plan.cells.entries()) { + const policy = cases.get(`${cell.caseId}\u0000${cell.caseVersion}`); + if (!policy) continue; + const compatible = + report.plan.analysis.kind === "paired" + ? policy.evidenceClass === "paired-value" + : report.plan.analysis.kind === "reviewer" + ? policy.evidenceClass === "reviewer-only" + : policy.evidenceClass !== "reviewer-only" && + policy.evidenceClass !== "paired-value"; + if (!compatible) { + issue( + issues, + `$.plan.cells.${index}`, + "policy", + "Campaign analysis is incompatible with the case evidence class.", + ); + } + } + const pairing = validatePairing( + report, + primaryCells, + productCells, + activatedReserveCells, + ); + issues.push(...pairing.issues); + const scoredOutcomes = pairing.scoredOutcomes; + if ( + report.completion.status === "complete" && + scoredOutcomes !== report.plan.stoppingRule.count + ) { + issue( + issues, + "$.completion.status", + "policy", + "Complete campaign must reach its fixed scored-outcome target exactly.", + ); + } + if ( + report.completion.status === "stopped" && + scoredOutcomes >= report.plan.stoppingRule.count && + !(report.completion.cause === "budget" && budgetRequiresStop) + ) { + issue( + issues, + "$.completion.status", + "policy", + "Stopped campaign cannot have reached its fixed scored-outcome target.", + ); + } + return issues; +} +export function parseReport( + input: unknown, + catalog: ValidatedCaseCatalog, +): + | { readonly ok: true; readonly value: ValidatedReport } + | { readonly ok: false; readonly issues: readonly ReportIssue[] } { + const parsed = EvalReportV2Schema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + issues: parsed.error.issues.map((item) => schemaIssue(input, item)), + }; + } + const issues = semanticIssues(parsed.data, catalog); + if (issues.length > 0) return { ok: false, issues }; + const value = ValidatedReportSchema.parse(parsed.data); + freezeTree(value); + return { ok: true, value }; +} diff --git a/evals/validated.ts b/evals/validated.ts new file mode 100644 index 0000000..3d8c883 --- /dev/null +++ b/evals/validated.ts @@ -0,0 +1,12 @@ +export type DeepReadonly = T extends readonly (infer Item)[] + ? readonly DeepReadonly[] + : T extends object + ? { readonly [Key in keyof T]: DeepReadonly } + : T; + +export function freezeTree(value: unknown): void { + if (value === null || typeof value !== "object" || Object.isFrozen(value)) + return; + for (const child of Object.values(value)) freezeTree(child); + Object.freeze(value); +} diff --git a/tests/eval-report.test.ts b/tests/eval-report.test.ts new file mode 100644 index 0000000..7b6bad9 --- /dev/null +++ b/tests/eval-report.test.ts @@ -0,0 +1,989 @@ +import { describe, expect, test } from "bun:test"; +import { canonicalJson } from "../evals/canonical-json.js"; +import { parseCaseCatalog } from "../evals/catalog.js"; +import { + campaignPlanSha256, + type EvalReportV2, + parseReport, + type ReportIssue, +} from "../evals/report.js"; + +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; + +function modelIdentity(model = "test") { + return { + routeProvider: "openai", + gateway: null, + family: "gpt", + model, + revision: null, + }; +} + +function caseCatalog() { + const parsed = parseCaseCatalog([ + { + caseId: "durable-plan", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + ]); + if (!parsed.ok) throw new Error("Fixture catalog must be valid."); + return parsed.value; +} + +function pairedCaseCatalog() { + const parsed = parseCaseCatalog([ + { + caseId: "paired-value", + caseVersion: 1, + evidenceClass: "paired-value", + oracle: "hidden-executable", + release: "report-only", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + }, + ]); + if (!parsed.ok) throw new Error("Paired fixture catalog must be valid."); + return parsed.value; +} + +function reviewerCaseCatalog() { + const parsed = parseCaseCatalog([ + { + caseId: "review-case", + caseVersion: 1, + evidenceClass: "reviewer-only", + oracle: "fixed-review-label", + release: "report-only", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + }, + ]); + if (!parsed.ok) throw new Error("Reviewer fixture catalog must be valid."); + return parsed.value; +} + +function report(paired = false): EvalReportV2 { + const cells: EvalReportV2["plan"]["cells"] = paired + ? [ + { + cellId: "cell-a", + blockId: "block-1", + caseId: "paired-value", + caseVersion: 1, + armToken: "opaque-a", + repetition: 0, + managerModel: modelIdentity(), + reviewerModel: null, + schedule: "primary", + }, + { + cellId: "cell-b", + blockId: "block-1", + caseId: "paired-value", + caseVersion: 1, + armToken: "opaque-b", + repetition: 0, + managerModel: modelIdentity(), + reviewerModel: null, + schedule: "primary", + }, + ] + : [ + { + cellId: "cell-a", + blockId: "block-1", + caseId: "durable-plan", + caseVersion: 1, + armToken: null, + repetition: 0, + managerModel: null, + reviewerModel: null, + schedule: "primary", + }, + ]; + const attempt = ( + cell: (typeof cells)[number], + suffix: string, + ): EvalReportV2["attempts"][number] => ({ + schemaVersion: 2, + attemptId: `attempt-${suffix}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: cell.armToken, + repetition: cell.repetition, + artifact: { + packageVersion: "1.0.0", + sourceCommit: "commit", + sourceTreeSha256: digest("a"), + tarballSha256: digest("b"), + unpackedManifestSha256: digest("c"), + }, + evaluator: { + sourceCommit: "evaluator-commit", + caseCatalogSha256: digest("d"), + policyCatalogSha256: digest("e"), + graderBundleSha256: digest("f"), + }, + hostConfigSha256: digest("0"), + actors: [ + { + role: "manager", + requestedModel: modelIdentity(), + actualModel: { + kind: "observed", + value: modelIdentity(), + }, + sessionIds: ["session-1"], + }, + ], + instructions: [ + { + source: "command", + name: "eval", + sequence: 0, + sha256: digest("1"), + bytes: 1, + }, + ], + transcript: { sha256: digest("2"), artifact: "attempt.jsonl" }, + outcome: paired + ? { + kind: "product", + passed: true, + endedBy: "quiet", + issues: [], + evidence: { + kind: "paired-value", + hiddenCorrectness: true, + claimedComplete: true, + falseCompletion: false, + }, + } + : { + kind: "product", + passed: true, + endedBy: "quiet", + issues: [], + evidence: { + kind: "conformance", + falseCompletion: false, + unsubmittedReviews: 0, + facts: { durable: true }, + }, + }, + usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, + }); + const value: EvalReportV2 = { + schemaVersion: 2, + reportId: "report-1", + plan: { + schemaVersion: 1, + planId: "plan-1", + planSha256: digest("3"), + randomizationSeed: "seed", + cells, + abortPolicy: { + retry: paired ? "whole-pair" : "never", + maxReplacementBlocks: 0, + }, + stoppingRule: { + kind: paired ? "fixed-complete-pairs" : "fixed-attempts", + count: 1, + }, + analysis: paired + ? { + kind: "paired", + primaryOutcome: "hidden-correctness", + estimand: "candidate-minus-baseline-risk-difference", + interval: "task-stratified-paired-bootstrap", + alpha: 0.05, + targetPower: 0.8, + minimumDetectableEffect: 0.1, + tieRule: "zero-difference", + bootstrapSeed: "seed", + versionSha256: digest("4"), + } + : { kind: "rate", primaryOutcome: "pass", versionSha256: digest("4") }, + budget: { + maxUsd: 1, + unknownCostPolicy: "stop", + maxOutputTokens: 10, + maxWallClockMs: 2_000, + maxAttempts: paired ? 2 : 1, + }, + }, + attempts: cells.map((cell, index) => attempt(cell, `${index}`)), + completion: { + status: "complete", + cause: "fixed-target", + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + activatedReserveCellIds: [], + observed: { + attempts: cells.length, + outputTokens: cells.length, + costUsd: 0, + wallClockMs: 1_000, + }, + }, + allocationCommitmentSha256: paired ? digest("5") : null, + }; + value.plan.planSha256 = campaignPlanSha256(value.plan); + return value; +} + +function result(input: unknown, catalog = caseCatalog()) { + return parseReport(input, catalog); +} + +function invalidIssues( + input: unknown, + catalog = caseCatalog(), +): readonly ReportIssue[] { + const parsed = result(input, catalog); + expect(parsed.ok).toBe(false); + if (parsed.ok) throw new Error("Expected an invalid report."); + return parsed.issues; +} + +function rehash(value: EvalReportV2): EvalReportV2 { + value.plan.planSha256 = campaignPlanSha256(value.plan); + return value; +} + +function addReplacementPair(value: EvalReportV2) { + const cells = value.plan.cells.map((cell, index) => ({ + ...cell, + cellId: `reserve-${index}`, + blockId: "block-2", + repetition: 1, + schedule: "replacement-reserve" as const, + })); + value.plan.cells.push(...cells); + value.plan.abortPolicy.maxReplacementBlocks = 1; + value.plan.budget.maxAttempts += 2; + return { value: rehash(value), cells }; +} + +function reviewerReport(): EvalReportV2 { + const value = report(); + itemAt(value.plan.cells, 0).caseId = "review-case"; + const attempt = itemAt(value.attempts, 0); + attempt.caseId = "review-case"; + itemAt(attempt.actors, 0).role = "reviewer"; + attempt.outcome = { + kind: "product", + passed: true, + endedBy: "quiet", + issues: [], + evidence: { + kind: "reviewer-only", + truth: "clean", + verdict: "passed", + findings: [], + submitted: true, + }, + }; + value.plan.analysis = { + kind: "reviewer", + interval: "wilson", + alpha: 0.05, + versionSha256: digest("4"), + }; + return rehash(value); +} + +function itemAt(values: readonly T[], index: number): T { + const value = values[index]; + if (value === undefined) throw new Error(`Missing fixture item ${index}.`); + return value; +} + +describe("eval report boundary", () => { + test("accepts a complete ledger and derives a canonical plan hash", () => { + const value = report(); + const parsed = result(value); + expect(parsed.ok).toBe(true); + if (parsed.ok) { + expect(parsed.value.plan.planSha256).toBe(campaignPlanSha256(value.plan)); + expect(Object.isFrozen(parsed.value)).toBe(true); + expect(Object.isFrozen(parsed.value.plan.cells)).toBe(true); + expect(Object.isFrozen(itemAt(parsed.value.plan.cells, 0))).toBe(true); + expect(Reflect.set(parsed.value.plan, "planId", "changed")).toBe(false); + } + }); + + test("accepts a complete paired ledger with paired policy and evidence", () => { + expect(result(report(true), pairedCaseCatalog()).ok).toBe(true); + }); + + test("rejects summary-only qualification and legacy v1 shapes", () => { + const summaryOnly = result({ + schemaVersion: 2, + reportId: "summary", + summary: { passed: true }, + }); + expect(summaryOnly).toEqual({ + ok: false, + issues: expect.arrayContaining([ + { path: "$.plan", code: "missing", message: "Missing required value." }, + ]), + }); + const legacy = result({ schemaVersion: 1, reportId: "legacy", runs: [] }); + expect(legacy.ok).toBe(false); + }); + + test("rejects missing data, unknown cases, duplicate ids, and case version drift", () => { + const { attempts: _attempts, ...missing } = report(); + expect(invalidIssues(missing)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "missing" })]), + ); + const unknown = report(); + itemAt(unknown.plan.cells, 0).caseId = "unknown"; + itemAt(unknown.attempts, 0).caseId = "unknown"; + unknown.plan.planSha256 = campaignPlanSha256(unknown.plan); + expect(invalidIssues(unknown)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: "Unknown case id and version." }), + ]), + ); + const duplicate = report(); + duplicate.attempts.push({ ...itemAt(duplicate.attempts, 0) }); + duplicate.completion.observed.attempts = 2; + expect(invalidIssues(duplicate)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "duplicate" })]), + ); + const duplicateCell = report(); + duplicateCell.plan.cells.push({ ...itemAt(duplicateCell.plan.cells, 0) }); + duplicateCell.plan.planSha256 = campaignPlanSha256(duplicateCell.plan); + expect(invalidIssues(duplicateCell)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "duplicate" })]), + ); + const drift = report(); + itemAt(drift.attempts, 0).caseVersion = 2; + expect(invalidIssues(drift)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: "Attempt does not match its scheduled cell.", + }), + ]), + ); + }); + + test("rejects pair mismatch and a mutated plan hash", () => { + const paired = report(true); + itemAt(paired.attempts, 1).armToken = "opaque-a"; + expect(invalidIssues(paired)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "policy" })]), + ); + const changedPlan = report(); + changedPlan.plan.randomizationSeed = "another-seed"; + expect(invalidIssues(changedPlan)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "hash" })]), + ); + }); + + test("rejects non-finite and out-of-range values", () => { + const nonFinite = report(); + itemAt(nonFinite.attempts, 0).usage.durationMs = Number.NaN; + expect(invalidIssues(nonFinite)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "schema" })]), + ); + const outOfRange = report(true); + if (outOfRange.plan.analysis.kind === "paired") + outOfRange.plan.analysis.targetPower = 1.1; + expect(invalidIssues(outOfRange)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "schema" })]), + ); + const zeroEffect = report(true); + if (zeroEffect.plan.analysis.kind === "paired") + zeroEffect.plan.analysis.minimumDetectableEffect = 0; + expect(invalidIssues(zeroEffect, pairedCaseCatalog())).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "schema" })]), + ); + const reviewerAlpha = reviewerReport(); + if (reviewerAlpha.plan.analysis.kind === "reviewer") + Reflect.set(reviewerAlpha.plan.analysis, "alpha", 0.1); + expect(invalidIssues(reviewerAlpha, reviewerCaseCatalog())).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "schema" })]), + ); + }); + + test("uses canonical Unicode JSON and rejects lone surrogates", () => { + expect( + canonicalJson({ "€": "Euro", "\r": "CR", "1": "One", "😀": "Emoji" }), + ).toBe('{"\\r":"CR","1":"One","€":"Euro","😀":"Emoji"}'); + expect(() => canonicalJson("\ud800")).toThrow( + "Canonical JSON requires Unicode scalar values.", + ); + expect(() => canonicalJson(new Date(0))).toThrow( + "Canonical JSON requires plain JSON objects.", + ); + expect(() => canonicalJson(new Array(1))).toThrow( + "Canonical JSON requires dense arrays.", + ); + const disguisedSparse = new Array(2); + disguisedSparse[0] = 1; + Reflect.set(disguisedSparse, "extra", true); + expect(() => canonicalJson(disguisedSparse)).toThrow( + "Canonical JSON requires dense arrays.", + ); + const value = report(); + value.plan.planId = "\ud800"; + expect(invalidIssues(value)).toEqual( + expect.arrayContaining([expect.objectContaining({ code: "schema" })]), + ); + }); + + test("rejects invalid finalization and incompatible product evidence", () => { + const finalization = report(); + finalization.completion.cause = "provider"; + expect(invalidIssues(finalization)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: "Complete campaigns require fixed-target cause.", + }), + ]), + ); + const evidence = report(); + itemAt(evidence.attempts, 0).outcome = { + kind: "product", + passed: true, + endedBy: "quiet", + issues: [], + evidence: { + kind: "regression", + falseCompletion: false, + unsubmittedReviews: 0, + facts: {}, + }, + }; + expect(invalidIssues(evidence)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: + "Product evidence kind is incompatible with the case evidence class.", + }), + ]), + ); + }); + + test("validates catalog policy and ledger provenance", () => { + const calibration = parseCaseCatalog([ + { + caseId: "review", + caseVersion: 1, + evidenceClass: "reviewer-only", + oracle: "fixed-review-label", + release: "report-only", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + }, + ]); + expect(calibration.ok).toBe(true); + if (calibration.ok) { + expect(Object.isFrozen(calibration.value)).toBe(true); + expect(Object.isFrozen(itemAt(calibration.value, 0))).toBe(true); + } + const provenance = report(); + const firstAttempt = itemAt(provenance.attempts, 0); + firstAttempt.actors.push({ ...itemAt(firstAttempt.actors, 0) }); + firstAttempt.instructions.push({ ...itemAt(firstAttempt.instructions, 0) }); + provenance.completion.activatedReserveCellIds.push("cell-a"); + expect(invalidIssues(provenance)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: "provenance" }), + expect.objectContaining({ + message: + "Activated reserve ids must reference replacement reserve cells.", + }), + ]), + ); + }); + + test("rejects incompatible catalog policies", () => { + const base = { + caseId: "case", + caseVersion: 1, + evidenceClass: "conformance" as const, + oracle: "durable-state" as const, + release: "required" as const, + minProviders: 1, + minScoredAttempts: 1, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }; + for (const input of [ + [{ ...base, minPassRate: null }], + [{ ...base, oracle: "fixed-review-label" }], + [ + { + ...base, + evidenceClass: "paired-value", + oracle: "trajectory", + release: "report-only", + minPassRate: null, + }, + ], + [ + { + ...base, + evidenceClass: "reviewer-only", + oracle: "fixed-review-label", + reviewerPromotionRecordSha256: null, + }, + ], + [base, { ...base }], + [{ ...base, caseId: "\ud800" }], + ]) { + expect(parseCaseCatalog(input).ok).toBe(false); + } + }); + + test("enforces hard report invariants", () => { + const mutations = [ + { + name: "empty product actors", + value: () => { + const value = report(); + itemAt(value.attempts, 0).actors = []; + return value; + }, + message: + "Product and unscored attempts require actors and instructions.", + }, + { + name: "completion tokens", + value: () => { + const value = report(); + value.completion.observed.outputTokens = 0; + return value; + }, + message: "Observed output tokens must equal the attempt total.", + }, + { + name: "completion cost", + value: () => { + const value = report(); + value.completion.observed.costUsd = 1; + return value; + }, + message: + "Observed cost must be null for unknown attempt costs or equal their total.", + }, + { + name: "duplicate reserve activation", + value: () => { + const value = report(); + value.plan.cells.push({ + ...itemAt(value.plan.cells, 0), + cellId: "reserve", + schedule: "replacement-reserve", + }); + value.completion.activatedReserveCellIds = ["reserve", "reserve"]; + return rehash(value); + }, + message: "Duplicate activated reserve cell id.", + }, + { + name: "paired model mismatch", + catalog: pairedCaseCatalog, + value: () => { + const value = report(true); + itemAt(value.plan.cells, 1).managerModel = { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "different", + revision: null, + }; + return rehash(value); + }, + message: "Invalid paired block block-1.", + }, + { + name: "attempt scheduled model mismatch", + catalog: pairedCaseCatalog, + value: () => { + const value = report(true); + const attempt = itemAt(value.attempts, 0); + const manager = attempt.actors.find( + (actor) => actor.role === "manager", + ); + if (manager) manager.requestedModel.model = "different"; + return value; + }, + message: "Requested manager model does not match its scheduled cell.", + }, + { + name: "plan count", + value: () => { + const value = report(); + value.plan.stoppingRule.count = 2; + return rehash(value); + }, + message: "Fixed-attempt count must equal the primary cell count.", + }, + { + name: "plan budget", + value: () => { + const value = report(); + value.plan.cells.push({ + ...itemAt(value.plan.cells, 0), + cellId: "cell-b", + blockId: "block-2", + }); + value.plan.stoppingRule.count = 2; + return rehash(value); + }, + message: "Maximum attempts cannot be below the primary cell count.", + }, + { + name: "output token budget", + value: () => { + const value = report(); + value.plan.budget.maxOutputTokens = 0; + return rehash(value); + }, + message: "Exceeded or unverifiable budget requires a budget stop.", + }, + { + name: "wall clock budget", + value: () => { + const value = report(); + value.plan.budget.maxWallClockMs = 0; + return rehash(value); + }, + message: "Exceeded or unverifiable budget requires a budget stop.", + }, + { + name: "wall clock lower bound", + value: () => { + const value = report(); + itemAt(value.attempts, 0).usage.durationMs = 2_001; + return value; + }, + message: + "Observed wall clock cannot be shorter than elapsed campaign time or its longest attempt.", + }, + { + name: "wall clock timestamp lower bound", + value: () => { + const value = report(); + value.completion.finishedAt = "2026-01-01T01:00:00.000Z"; + return value; + }, + message: + "Observed wall clock cannot be shorter than elapsed campaign time or its longest attempt.", + }, + { + name: "cost budget", + value: () => { + const value = report(); + itemAt(value.attempts, 0).usage.costUsd = 0.02; + value.completion.observed.costUsd = 0.02; + value.plan.budget.maxUsd = 0.01; + return rehash(value); + }, + message: "Exceeded or unverifiable budget requires a budget stop.", + }, + { + name: "unknown cost stop policy", + value: () => { + const value = report(); + itemAt(value.attempts, 0).usage.costUsd = null; + value.completion.observed.costUsd = null; + return value; + }, + message: "Exceeded or unverifiable budget requires a budget stop.", + }, + { + name: "unsafe transcript artifact", + value: () => { + const value = report(); + const attempt = itemAt(value.attempts, 0); + if (attempt.transcript) + attempt.transcript.artifact = "../attempt.jsonl"; + return value; + }, + message: "Invalid report value.", + }, + { + name: "missing product transcript", + value: () => { + const value = report(); + itemAt(value.attempts, 0).transcript = null; + return value; + }, + message: + "Product and escalation attempts require a transcript artifact.", + }, + { + name: "passed issues", + value: () => { + const value = report(); + const attempt = itemAt(value.attempts, 0); + if (attempt.outcome.kind === "product") + attempt.outcome.issues.push("issue"); + return value; + }, + message: "Passed product attempts cannot carry issues.", + }, + { + name: "failed product without issues", + value: () => { + const value = report(); + const attempt = itemAt(value.attempts, 0); + if (attempt.outcome.kind === "product") + attempt.outcome.passed = false; + return value; + }, + message: "Failed product attempts require at least one issue.", + }, + { + name: "unsubmitted reviewer verdict", + catalog: reviewerCaseCatalog, + value: () => { + const value = reviewerReport(); + const attempt = itemAt(value.attempts, 0); + if ( + attempt.outcome.kind === "product" && + attempt.outcome.evidence.kind === "reviewer-only" + ) { + attempt.outcome.evidence.submitted = false; + } + return value; + }, + message: "Reviewer submission and verdict must be present together.", + }, + { + name: "contradictory paired evidence", + catalog: pairedCaseCatalog, + value: () => { + const value = report(true); + const attempt = itemAt(value.attempts, 0); + if ( + attempt.outcome.kind === "product" && + attempt.outcome.evidence.kind === "paired-value" + ) { + attempt.outcome.evidence.hiddenCorrectness = false; + } + return value; + }, + message: "Paired product outcome must agree with its hidden evidence.", + }, + { + name: "contradictory reviewer outcome", + catalog: reviewerCaseCatalog, + value: () => { + const value = reviewerReport(); + const attempt = itemAt(value.attempts, 0); + if ( + attempt.outcome.kind === "product" && + attempt.outcome.evidence.kind === "reviewer-only" + ) { + attempt.outcome.evidence.verdict = "failed"; + } + return value; + }, + message: + "Reviewer outcome must agree with the fixed truth label and verdict.", + }, + { + name: "missing complete primary attempt", + value: () => { + const value = report(); + value.attempts = []; + value.completion.observed = { + attempts: 0, + outputTokens: 0, + costUsd: 0, + wallClockMs: 1, + }; + return value; + }, + message: "Complete campaign is missing attempt for cell cell-a.", + }, + ]; + for (const mutation of mutations) { + expect(invalidIssues(mutation.value(), mutation.catalog?.())).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: mutation.message }), + ]), + ); + } + }); + + test("accepts correct defect detection and rejects non-paired retries", () => { + const detected = reviewerReport(); + const reviewerAttempt = itemAt(detected.attempts, 0); + if ( + reviewerAttempt.outcome.kind === "product" && + reviewerAttempt.outcome.evidence.kind === "reviewer-only" + ) { + reviewerAttempt.outcome.evidence.truth = "defect"; + reviewerAttempt.outcome.evidence.verdict = "failed"; + reviewerAttempt.outcome.evidence.findings = ["known defect"]; + } + expect(result(detected, reviewerCaseCatalog()).ok).toBe(true); + + const retry = report(); + retry.plan.cells.push({ + ...itemAt(retry.plan.cells, 0), + cellId: "reserve", + blockId: "block-2", + schedule: "replacement-reserve", + }); + retry.plan.abortPolicy.retry = "whole-pair"; + retry.plan.abortPolicy.maxReplacementBlocks = 1; + retry.plan.budget.maxAttempts = 2; + expect(invalidIssues(rehash(retry))).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: "Non-paired campaigns cannot declare replacement retries.", + }), + ]), + ); + }); + + test("requires scored targets and whole-pair reserve activation", () => { + const failedPair = report(true); + const failedAttempt = itemAt(failedPair.attempts, 0); + failedAttempt.outcome = { + kind: "failure", + origin: "provider", + code: "unavailable", + retryable: true, + }; + expect(invalidIssues(failedPair, pairedCaseCatalog())).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: + "Complete campaign must reach its fixed scored-outcome target exactly.", + }), + ]), + ); + + const partial = addReplacementPair(report(true)); + partial.value.completion.activatedReserveCellIds = [ + itemAt(partial.cells, 0).cellId, + ]; + expect(invalidIssues(partial.value, pairedCaseCatalog())).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: + "Replacement block block-2 must be activated as a whole pair.", + }), + ]), + ); + }); + + test("accepts a complete whole-pair replacement", () => { + const replacement = addReplacementPair(report(true)); + const templates = replacement.value.attempts.map((attempt) => + structuredClone(attempt), + ); + itemAt(replacement.value.attempts, 0).outcome = { + kind: "failure", + origin: "provider", + code: "unavailable", + retryable: true, + }; + for (const [index, cell] of replacement.cells.entries()) { + const attempt = itemAt(templates, index); + attempt.attemptId = `replacement-${index}`; + attempt.cellId = cell.cellId; + attempt.blockId = cell.blockId; + attempt.repetition = cell.repetition; + replacement.value.attempts.push(attempt); + } + replacement.value.completion.activatedReserveCellIds = + replacement.cells.map((cell) => cell.cellId); + replacement.value.completion.observed.attempts = 4; + replacement.value.completion.observed.outputTokens = 4; + expect(result(replacement.value, pairedCaseCatalog()).ok).toBe(true); + }); + + test("allows empty failure provenance before any model turn", () => { + const value = report(); + const attempt = itemAt(value.attempts, 0); + attempt.actors = []; + attempt.instructions = []; + attempt.transcript = null; + attempt.outcome = { + kind: "failure", + origin: "provider", + code: "unavailable", + retryable: true, + }; + value.completion.status = "stopped"; + value.completion.cause = "provider"; + expect(result(value).ok).toBe(true); + }); + + test("accepts normal floating-point cost accumulation", () => { + const value = report(true); + itemAt(value.attempts, 0).usage.costUsd = 0.1; + itemAt(value.attempts, 1).usage.costUsd = 0.2; + value.completion.observed.costUsd = 0.3; + expect(result(value, pairedCaseCatalog()).ok).toBe(true); + }); + + test("accepts an honestly stopped partial campaign", () => { + const value = report(); + value.attempts = []; + value.completion = { + status: "stopped", + cause: "provider", + startedAt: "2026-01-01T00:00:00.000Z", + finishedAt: "2026-01-01T00:00:01.000Z", + activatedReserveCellIds: [], + observed: { + attempts: 0, + outputTokens: 0, + costUsd: 0, + wallClockMs: 1_000, + }, + }; + expect(result(value).ok).toBe(true); + }); + + test("accepts an honest unknown-cost budget stop", () => { + const value = report(); + itemAt(value.attempts, 0).usage.costUsd = null; + value.completion.observed.costUsd = null; + value.completion.status = "stopped"; + value.completion.cause = "budget"; + expect(result(value).ok).toBe(true); + }); + + test("rejects a stopped campaign after the scored target", () => { + const value = report(); + value.completion.status = "stopped"; + value.completion.cause = "provider"; + expect(invalidIssues(value)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: + "Stopped campaign cannot have reached its fixed scored-outcome target.", + }), + ]), + ); + }); +});