Bind eval results to exact provenance

.agents/plans/02-eval-engineering/evidence/phase-3-pilot.json

@@ -0,0 +1,33 @@
+{
+  "schemaVersion": 1,
+  "recordedAt": "2026-08-25T04:14:14.980Z",
+  "scenario": "happy-path",
+  "modelRoute": "opencode/gpt-5.6-sol",
+  "opencodeVersion": "1.18.6",
+  "passed": true,
+  "artifact": {
+    "packageVersion": "8.1.1",
+    "sourceCommit": "04d3f6ee34bc92373982f9599beb6ce66c08f6c8",
+    "sourceTreeSha256": "sha256:605b10580561bd0bc592614b550eb5516ec9240fd2553dd3e92e3a36a30821d9",
+    "tarballSha256": "sha256:c89f7363248ccc3e3f69728c1aa42044a25938cd533e470d9f73ef08bc64ad24",
+    "unpackedManifestSha256": "sha256:398e6003b3c4d16f5e49dad9dbe734666149ee753e803d020501c863c19f0183"
+  },
+  "evaluator": {
+    "sourceCommit": "04d3f6ee34bc92373982f9599beb6ce66c08f6c8",
+    "caseCatalogSha256": "sha256:b9bd76c55d20db6daca1b40bf4118fae66f829dafec4b9388288d6a603a47be3",
+    "policyCatalogSha256": "sha256:708363968ed83fedf84f0310baa65f2b8b172c8c5ccdc0d090cf50bae2b51c9d",
+    "graderBundleSha256": "sha256:3eba0223d2f005abd4c429e05dd86ddb4c20a495725be69d852ea8d36f09431a"
+  },
+  "hostConfigSha256": "sha256:fc5de04f3e595f7c84d9310369a9ebc3118600c1bbc342be67e65d8d72f5b64e",
+  "actors": [
+    { "role": "manager", "requestedModelId": "opencode/gpt-5.6-sol", "actual": "observed", "sessionCount": 1 },
+    { "role": "reviewer", "requestedModelId": "opencode/gpt-5.6-sol", "actual": "observed", "sessionCount": 1 }
+  ],
+  "instructions": [
+    { "source": "guidance", "name": "flow-plan", "sequence": 0, "sha256": "sha256:f9d2f6dbf60f8be5a53a3959517eb9e5307624e75e7aa25960d76db3c8c71d7a", "bytes": 5685 },
+    { "source": "guidance", "name": "flow-run", "sequence": 1, "sha256": "sha256:9f451a740456a7a59aca02f4485bdcfe5f64ad3fb57a48980fa09cde40c08050", "bytes": 7933 }
+  ],
+  "transcriptSha256": "sha256:875a85c8007daa436be268553319967c35b449ab4ad5d0e663a9a80e8063db0d",
+  "redactionScanPassed": true,
+  "unsupportedClaim": "Observed provider and model fields do not independently establish family, gateway, or revision. Phase 4 must preserve actual v2 model identity as unobserved unless those fields become independently available."
+}

.agents/plans/02-eval-engineering/evidence/phase-3-review.md

@@ -0,0 +1,39 @@
+# Phase 3 Interrogate review
+
+## Intent
+
+Phase 3 must bind real eval output to exact source, packed bytes, evaluator inputs,
+host configuration, requested and observed actors, delivered instructions, and a
+redacted transcript. It must preserve endpoint and identity limitations rather
+than fill missing observations from configuration.
+
+## Acted on
+
+- The runner now consumes provenance helpers and emits the binding on every result.
+- Tarball bytes are checked before inspection and after cache installation.
+- Unpacked manifests bind file and directory type. Other archive entry types are
+  rejected.
+- Reviewer child endpoint failures remain explicit, including partial discovery.
+- Multi-session actor identity fails closed when any session is unobserved.
+- Transcript redaction covers object keys, short sensitive-field values, paths,
+  and credential-shaped strings.
+- Instruction text rejects malformed Unicode and hashes actual UTF-8 delivery.
+- Requested manager and reviewer identities are emitted separately from raw host
+  observations. Reviewer model and step configuration are included in host hashes.
+- Pure actor parsing moved out of the large harness into `host-observation.ts`.
+
+## Lead judgment
+
+The pinned host independently exposes provider and model fields, but not the full
+family, gateway, and revision tuple required by v2 `ModelIdentity`. The legacy
+pilot retains the raw observation. Phase 4 must emit actual identity as
+`unobserved` unless a later host exposes all required fields. Cross-family claims
+remain unavailable otherwise.
+
+## Verdict
+
+`VERIFIED`. Four-model recheck found no unresolved blocker. The final paid packed
+`happy-path` passed with observed manager and reviewer roles, two delivered
+guidance records, exact artifact/evaluator/host hashes, and a clean transcript
+redaction scan. The full repository gate passes 462 tests, one intentional skip,
+and zero failures.

.agents/plans/02-eval-engineering/phase-3-provenance.md

@@ -14,6 +14,13 @@ evidence outside the report and bind each attempt to those observed facts.
   transcript artifacts.
 - `evals/harness.ts`. Capture actor metadata and observed guidance loads through
   the Phase 0 field map.
+- `evals/host-observation.ts`. Keep Phase 0 field-map parsing and lineage rules
+  pure and independently testable.
+- `evals/run.ts`. Attach exact provenance, observed actors, delivered guidance,
+  and redacted transcript evidence to every legacy result before Phase 4 adapts it.
+- `evals/cassette.ts`. Redact sensitive object keys as well as values.
+- `tests/provenance.test.ts`. Cover source, archive, configuration, instruction,
+  and transcript boundaries without paid calls.
 - `tests/eval-reporting.test.ts`. Cover hash swaps, dirty trees, unobserved actor
   fallback, gateway ids, lazy guidance bytes, transcript retention, and redaction.
 
@@ -29,5 +36,8 @@ Static. Focused reporting tests and `bun run check`.
 Runtime. Run one paid `happy-path` attempt, read the actual parent and child model
 fields, and prove a tarball swap invalidates the externally computed binding.
 
+Runtime evidence. [Final paid pilot](evidence/phase-3-pilot.json) and
+[Interrogate review](evidence/phase-3-review.md).
+
 Stop gate. Cross-family reviewer evidence stays unavailable on hosts that cannot
 expose actual child-session identity. Other evidence work may continue.

.audit/eval-engineering.tsv

@@ -31,3 +31,8 @@ ts	phase	decision	why	evidence	result
 2026-08-25T03:43:39Z	phase-2	recovered the implementation after a delegate usage-limit stop	the partial module and tests were inspectable, so the lead could finish without discarding verified work	evals/analysis.ts; tests/atomic-analysis.test.ts; tests/advisory-analysis.test.ts	VERIFIED focused gate green
 2026-08-25T03:43:39Z	phase-2	fixed the multi-model Interrogate findings	report-only scoping, provider fallback, exact provenance sets, reviewer incompleteness, analysis-kind guards, and reserve eligibility affected decision integrity	.agents/plans/02-eval-engineering/evidence/phase-2-review.md	VERIFIED no unresolved blocker
 2026-08-25T03:43:39Z	phase-2	ran Deslop and the whole repository gate	the analysis and tests must remain small, pure, and regression-free	bun run check	VERIFIED 448 pass, 1 skip, 0 fail
+2026-08-25T04:14:31Z	phase-3	started from merged Phase 2 main	exact provenance must build on the atomic analyzer without cutting over v2 emission	git status on codex/eval-phase-3 at 04d3f6e	VERIFIED clean baseline, 448 pass, 1 skip, 0 fail
+2026-08-25T04:14:31Z	phase-3	integrated exact provenance into the real runner	helper-only evidence would not bind paid attempts to the packed bytes and observed actors	evals/provenance.ts; evals/host-observation.ts; evals/harness.ts; evals/run.ts	VERIFIED focused and full gates green
+2026-08-25T04:14:31Z	phase-3	fixed the multi-model Interrogate findings	archive types, endpoint completeness, requested actors, host config, transcript fields, Unicode, and production integration were evidence boundaries	.agents/plans/02-eval-engineering/evidence/phase-3-review.md	VERIFIED no unresolved blocker
+2026-08-25T04:14:31Z	phase-3	reran the final paid packed-host pilot	the committed observation must be generated by the final code and exact tarball	evals/results/2026-08-25T04-14-14-980Z.json; .agents/plans/02-eval-engineering/evidence/phase-3-pilot.json	VERIFIED happy-path pass, observed manager and reviewer, redaction scan clean
+2026-08-25T04:14:31Z	phase-3	ran Deslop and the whole repository gate	the phase must finish reviewable and regression-free	bun run check	VERIFIED 462 pass, 1 skip, 0 fail

evals/cassette.ts

@@ -181,7 +181,7 @@ export function mapStrings(
 	if (value && typeof value === "object") {
 		return Object.fromEntries(
 			Object.entries(value as Record<string, unknown>).map(([key, item]) => [
-				key,
+				map(key),
 				mapStrings(item, map),
 			]),
 		);

evals/harness.ts

@@ -24,6 +24,17 @@ import { createServer } from "node:net";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import packageJson from "../package.json" with { type: "json" };
+import {
+	extractObservedActor,
+	guidanceLoad,
+	isRecord,
+	nonEmptyString,
+	type ObservedActor,
+	type ObservedGuidanceLoad,
+	type ObservedSession,
+	reviewerActorObservation,
+	selectLineageValidatedReviewers,
+} from "./host-observation.js";
 
 const STARTUP_TIMEOUT_MS = 180_000;
 const REQUEST_TIMEOUT_MS = 120_000;
@@ -118,6 +129,10 @@ export type Outcome = {
 	readonly flowCalls: readonly ObservedToolCall[];
 	/** Every tool call, including host tools like bash/edit/task. */
 	readonly allCalls: readonly ObservedToolCall[];
+	/** Actor identity observations from completed, non-error assistant messages. */
+	readonly actors?: readonly ObservedActor[];
+	/** Raw delivered flow_guidance output, with its measured UTF-8 size. */
+	readonly guidanceLoads?: readonly ObservedGuidanceLoad[];
 	/** Parsed `.flow/session.json`, or null when no active session exists. */
 	readonly session: Record<string, unknown> | null;
 	/** Parsed documents under `.flow/history/`. */
@@ -884,6 +899,9 @@ type MessageEntry = {
 	info: {
 		role: string;
 		agent?: string;
+		model?: { providerID?: unknown; modelID?: unknown };
+		providerID?: unknown;
+		modelID?: unknown;
 		time?: { created: number; completed?: number };
 		error?: unknown;
 		cost?: number;
@@ -909,6 +927,10 @@ type MessageEntry = {
 	}[];
 };
 
+type SessionMessages = ObservedSession & {
+	readonly messages: readonly MessageEntry[] | null;
+};
+
 /**
  * One throwaway OpenCode host, over one fixture repository, for one attempt.
  *
@@ -1310,33 +1332,43 @@ export class EvalHost {
 	 * A host that does not expose children yields nothing rather than failing —
 	 * losing the subtask transcript is a smaller loss than losing the run.
 	 */
-	private async descendantSessions(
-		sessionIds: readonly string[],
-	): Promise<string[]> {
+	private async descendantSessions(sessionIds: readonly string[]): Promise<{
+		readonly sessions: readonly ObservedSession[];
+		readonly endpointFailed: boolean;
+	}> {
 		const known = new Set(sessionIds);
-		const found: string[] = [];
+		const found: ObservedSession[] = [];
+		let endpointFailed = false;
 		let frontier = [...sessionIds];
 		while (frontier.length > 0) {
 			const next: string[] = [];
 			for (const parent of frontier) {
-				let children: { id?: string }[];
+				let children: unknown;
 				try {
-					children = (await fetchJson(
+					children = await fetchJson(
 						`${this.baseUrl}/session/${parent}/children`,
-					)) as { id?: string }[];
+					);
 				} catch {
+					endpointFailed = true;
 					continue;
 				}
 				for (const child of Array.isArray(children) ? children : []) {
-					if (typeof child.id !== "string" || known.has(child.id)) continue;
-					known.add(child.id);
-					found.push(child.id);
-					next.push(child.id);
+					if (!isRecord(child)) continue;
+					const id = nonEmptyString(child.id);
+					if (!id || known.has(id)) continue;
+					const childSession: ObservedSession = {
+						id,
+						agent: nonEmptyString(child.agent),
+						parentID: nonEmptyString(child.parentID),
+					};
+					known.add(id);
+					found.push(childSession);
+					next.push(id);
 				}
 			}
 			frontier = next;
 		}
-		return found;
+		return { sessions: found, endpointFailed };
 	}
 
 	/**
@@ -1356,19 +1388,27 @@ export class EvalHost {
 		sessionIds: readonly string[],
 		durationMs: number,
 	): Promise<Outcome> {
-		const ordered = [
-			...sessionIds,
-			...(await this.descendantSessions(sessionIds)),
+		const descendantResult = await this.descendantSessions(sessionIds);
+		const descendants = descendantResult.sessions;
+		const sessionRecords: readonly ObservedSession[] = [
+			...sessionIds.map((id) => ({ id, agent: null, parentID: null })),
+			...descendants,
 		];
+		const ordered = sessionRecords.map((session) => session.id);
 		const messages: { sessionIndex: number; entry: MessageEntry }[] = [];
+		const sessionMessages: SessionMessages[] = [];
 		for (const [sessionIndex, sessionId] of ordered.entries()) {
-			let entries: MessageEntry[];
+			let entries: MessageEntry[] | null;
 			try {
 				entries = (await this.messages(sessionId)) as MessageEntry[];
 			} catch {
-				continue;
+				entries = null;
+			}
+			const session = sessionRecords[sessionIndex];
+			if (session) sessionMessages.push({ ...session, messages: entries });
+			if (entries) {
+				for (const entry of entries) messages.push({ sessionIndex, entry });
 			}
-			for (const entry of entries) messages.push({ sessionIndex, entry });
 		}
 		messages.sort(
 			(left, right) =>
@@ -1388,6 +1428,8 @@ export class EvalHost {
 		let assistantMessages = 0;
 		let hostError: string | null = null;
 		let finalText = "";
+		const guidanceLoads: ObservedGuidanceLoad[] = [];
+		let guidanceSequence = 0;
 
 		for (const { sessionIndex, entry } of messages) {
 			if (entry.info.role === "assistant") {
@@ -1450,12 +1492,55 @@ export class EvalHost {
 					rawOutput: raw,
 					metadata: part.state?.metadata ?? {},
 				});
+				if (part.tool === "flow_guidance") {
+					const input = part.state?.input ?? {};
+					guidanceLoads.push(
+						guidanceLoad({
+							sequence: guidanceSequence,
+							sessionIndex,
+							agent: entry.info.agent ?? "",
+							id: nonEmptyString(input.id),
+							rawOutput: raw,
+						}),
+					);
+					guidanceSequence += 1;
+				}
 			}
 		}
+		const parentSessions = sessionMessages.filter((session) =>
+			sessionIds.includes(session.id),
+		);
+		const reviewerSessions = selectLineageValidatedReviewers(
+			sessionIds,
+			descendants,
+		);
+		const reviewerActor = reviewerActorObservation({
+			childEndpointFailed: descendantResult.endpointFailed,
+			sessions: reviewerSessions.flatMap((session) => {
+				const messagesForSession = sessionMessages.find(
+					(candidate) => candidate.id === session.id,
+				);
+				return messagesForSession
+					? [{ id: session.id, messages: messagesForSession.messages }]
+					: [];
+			}),
+		});
+		const actors: readonly ObservedActor[] = [
+			extractObservedActor({
+				role: "manager",
+				sessions: parentSessions.map((session) => ({
+					id: session.id,
+					messages: session.messages,
+				})),
+			}),
+			reviewerActor,
+		];
 
 		return {
 			allCalls,
 			flowCalls: allCalls.filter((call) => call.tool.startsWith("flow_")),
+			actors,
+			guidanceLoads,
 			session: await this.readJson(join(this.project, ".flow", "session.json")),
 			archives: await this.readArchives(),
 			finalText,