From a5b8a86b15e70506a7cd20ef52181da2da4e7cb0 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 06:15:44 +0200 Subject: [PATCH 1/2] Bind eval results to exact provenance --- evals/cassette.ts | 2 +- evals/harness.ts | 121 ++++++++++++-- evals/host-observation.ts | 195 ++++++++++++++++++++++ evals/provenance.ts | 309 +++++++++++++++++++++++++++++++++++ evals/run.ts | 132 ++++++++++++++- tests/eval-reporting.test.ts | 175 ++++++++++++++++++++ tests/provenance.test.ts | 236 ++++++++++++++++++++++++++ 7 files changed, 1147 insertions(+), 23 deletions(-) create mode 100644 evals/host-observation.ts create mode 100644 evals/provenance.ts create mode 100644 tests/provenance.test.ts diff --git a/evals/cassette.ts b/evals/cassette.ts index b2c1b12..ee2b4dd 100644 --- a/evals/cassette.ts +++ b/evals/cassette.ts @@ -181,7 +181,7 @@ export function mapStrings( if (value && typeof value === "object") { return Object.fromEntries( Object.entries(value as Record).map(([key, item]) => [ - key, + map(key), mapStrings(item, map), ]), ); diff --git a/evals/harness.ts b/evals/harness.ts index 7a648a5..8fbfb3c 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -24,6 +24,17 @@ import { createServer } from "node:net"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import packageJson from "../package.json" with { type: "json" }; +import { + extractObservedActor, + guidanceLoad, + isRecord, + nonEmptyString, + type ObservedActor, + type ObservedGuidanceLoad, + type ObservedSession, + reviewerActorObservation, + selectLineageValidatedReviewers, +} from "./host-observation.js"; const STARTUP_TIMEOUT_MS = 180_000; const REQUEST_TIMEOUT_MS = 120_000; @@ -118,6 +129,10 @@ export type Outcome = { readonly flowCalls: readonly ObservedToolCall[]; /** Every tool call, including host tools like bash/edit/task. */ readonly allCalls: readonly ObservedToolCall[]; + /** Actor identity observations from completed, non-error assistant messages. */ + readonly actors?: readonly ObservedActor[]; + /** Raw delivered flow_guidance output, with its measured UTF-8 size. */ + readonly guidanceLoads?: readonly ObservedGuidanceLoad[]; /** Parsed `.flow/session.json`, or null when no active session exists. */ readonly session: Record | null; /** Parsed documents under `.flow/history/`. */ @@ -884,6 +899,9 @@ type MessageEntry = { info: { role: string; agent?: string; + model?: { providerID?: unknown; modelID?: unknown }; + providerID?: unknown; + modelID?: unknown; time?: { created: number; completed?: number }; error?: unknown; cost?: number; @@ -909,6 +927,10 @@ type MessageEntry = { }[]; }; +type SessionMessages = ObservedSession & { + readonly messages: readonly MessageEntry[] | null; +}; + /** * One throwaway OpenCode host, over one fixture repository, for one attempt. * @@ -1310,33 +1332,43 @@ export class EvalHost { * A host that does not expose children yields nothing rather than failing — * losing the subtask transcript is a smaller loss than losing the run. */ - private async descendantSessions( - sessionIds: readonly string[], - ): Promise { + private async descendantSessions(sessionIds: readonly string[]): Promise<{ + readonly sessions: readonly ObservedSession[]; + readonly endpointFailed: boolean; + }> { const known = new Set(sessionIds); - const found: string[] = []; + const found: ObservedSession[] = []; + let endpointFailed = false; let frontier = [...sessionIds]; while (frontier.length > 0) { const next: string[] = []; for (const parent of frontier) { - let children: { id?: string }[]; + let children: unknown; try { - children = (await fetchJson( + children = await fetchJson( `${this.baseUrl}/session/${parent}/children`, - )) as { id?: string }[]; + ); } catch { + endpointFailed = true; continue; } for (const child of Array.isArray(children) ? children : []) { - if (typeof child.id !== "string" || known.has(child.id)) continue; - known.add(child.id); - found.push(child.id); - next.push(child.id); + if (!isRecord(child)) continue; + const id = nonEmptyString(child.id); + if (!id || known.has(id)) continue; + const childSession: ObservedSession = { + id, + agent: nonEmptyString(child.agent), + parentID: nonEmptyString(child.parentID), + }; + known.add(id); + found.push(childSession); + next.push(id); } } frontier = next; } - return found; + return { sessions: found, endpointFailed }; } /** @@ -1356,19 +1388,27 @@ export class EvalHost { sessionIds: readonly string[], durationMs: number, ): Promise { - const ordered = [ - ...sessionIds, - ...(await this.descendantSessions(sessionIds)), + const descendantResult = await this.descendantSessions(sessionIds); + const descendants = descendantResult.sessions; + const sessionRecords: readonly ObservedSession[] = [ + ...sessionIds.map((id) => ({ id, agent: null, parentID: null })), + ...descendants, ]; + const ordered = sessionRecords.map((session) => session.id); const messages: { sessionIndex: number; entry: MessageEntry }[] = []; + const sessionMessages: SessionMessages[] = []; for (const [sessionIndex, sessionId] of ordered.entries()) { - let entries: MessageEntry[]; + let entries: MessageEntry[] | null; try { entries = (await this.messages(sessionId)) as MessageEntry[]; } catch { - continue; + entries = null; + } + const session = sessionRecords[sessionIndex]; + if (session) sessionMessages.push({ ...session, messages: entries }); + if (entries) { + for (const entry of entries) messages.push({ sessionIndex, entry }); } - for (const entry of entries) messages.push({ sessionIndex, entry }); } messages.sort( (left, right) => @@ -1388,6 +1428,8 @@ export class EvalHost { let assistantMessages = 0; let hostError: string | null = null; let finalText = ""; + const guidanceLoads: ObservedGuidanceLoad[] = []; + let guidanceSequence = 0; for (const { sessionIndex, entry } of messages) { if (entry.info.role === "assistant") { @@ -1450,12 +1492,55 @@ export class EvalHost { rawOutput: raw, metadata: part.state?.metadata ?? {}, }); + if (part.tool === "flow_guidance") { + const input = part.state?.input ?? {}; + guidanceLoads.push( + guidanceLoad({ + sequence: guidanceSequence, + sessionIndex, + agent: entry.info.agent ?? "", + id: nonEmptyString(input.id), + rawOutput: raw, + }), + ); + guidanceSequence += 1; + } } } + const parentSessions = sessionMessages.filter((session) => + sessionIds.includes(session.id), + ); + const reviewerSessions = selectLineageValidatedReviewers( + sessionIds, + descendants, + ); + const reviewerActor = reviewerActorObservation({ + childEndpointFailed: descendantResult.endpointFailed, + sessions: reviewerSessions.flatMap((session) => { + const messagesForSession = sessionMessages.find( + (candidate) => candidate.id === session.id, + ); + return messagesForSession + ? [{ id: session.id, messages: messagesForSession.messages }] + : []; + }), + }); + const actors: readonly ObservedActor[] = [ + extractObservedActor({ + role: "manager", + sessions: parentSessions.map((session) => ({ + id: session.id, + messages: session.messages, + })), + }), + reviewerActor, + ]; return { allCalls, flowCalls: allCalls.filter((call) => call.tool.startsWith("flow_")), + actors, + guidanceLoads, session: await this.readJson(join(this.project, ".flow", "session.json")), archives: await this.readArchives(), finalText, diff --git a/evals/host-observation.ts b/evals/host-observation.ts new file mode 100644 index 0000000..3417b24 --- /dev/null +++ b/evals/host-observation.ts @@ -0,0 +1,195 @@ +export type ObservedModelIdentity = + | { + readonly kind: "observed"; + readonly value: { + readonly providerID: string; + readonly modelID: string; + }; + } + | { + readonly kind: "unobserved"; + readonly reason: + | "endpoint-failure" + | "field-unavailable" + | "no-completed-assistant" + | "conflicting-observations" + | "reviewer-child-not-observed"; + }; + +export type ObservedActor = { + readonly role: "manager" | "reviewer"; + readonly sessionIds: readonly string[]; + readonly actualModel: ObservedModelIdentity; +}; + +export type ObservedGuidanceLoad = { + readonly sequence: number; + readonly sessionIndex: number; + readonly agent: string; + readonly id: string | null; + readonly rawOutput: string; + readonly utf8Bytes: number; +}; + +export type ObservedSession = { + readonly id: string; + readonly agent: string | null; + readonly parentID: string | null; +}; + +export function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export function nonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +function messageModelIdentity(message: unknown): { + readonly providerID: string; + readonly modelID: string; +} | null { + if (!isRecord(message) || !isRecord(message.info)) return null; + const info = message.info; + if (info.role !== "assistant" || "error" in info) return null; + const time = isRecord(info.time) ? info.time : null; + if (typeof time?.completed !== "number") return null; + const model = isRecord(info.model) ? info.model : null; + const providerID = + nonEmptyString(model?.providerID) ?? nonEmptyString(info.providerID); + const modelID = + nonEmptyString(model?.modelID) ?? nonEmptyString(info.modelID); + return providerID && modelID ? { providerID, modelID } : null; +} + +/** Extracts the Phase 0-approved identity fields from a host message response. */ +export function extractObservedModelIdentity( + messages: readonly unknown[] | null, +): ObservedModelIdentity { + if (messages === null) + return { kind: "unobserved", reason: "endpoint-failure" }; + const completed = messages.filter((message) => { + if (!isRecord(message) || !isRecord(message.info)) return false; + if (message.info.role !== "assistant" || "error" in message.info) + return false; + const time = isRecord(message.info.time) ? message.info.time : null; + return typeof time?.completed === "number"; + }); + if (completed.length === 0) + return { kind: "unobserved", reason: "no-completed-assistant" }; + const identities = completed + .map(messageModelIdentity) + .filter( + (value): value is { providerID: string; modelID: string } => + value !== null, + ); + if (identities.length === 0) + return { kind: "unobserved", reason: "field-unavailable" }; + const unique = new Map( + identities.map((identity) => [ + `${identity.providerID}\u0000${identity.modelID}`, + identity, + ]), + ); + if (unique.size > 1) + return { kind: "unobserved", reason: "conflicting-observations" }; + const identity = identities[0]; + return identity + ? { kind: "observed", value: identity } + : { kind: "unobserved", reason: "field-unavailable" }; +} + +/** Only children linked to a known session and named flow-reviewer are reviewers. */ +export function selectLineageValidatedReviewers( + parentSessionIds: readonly string[], + children: readonly ObservedSession[], +): readonly ObservedSession[] { + const parents = new Set(parentSessionIds); + return children.filter( + (child) => + child.agent === "flow-reviewer" && + child.parentID !== null && + parents.has(child.parentID), + ); +} + +export function extractObservedActor(input: { + readonly role: "manager" | "reviewer"; + readonly sessions: readonly { + readonly id: string; + readonly messages: readonly unknown[] | null; + }[]; +}): ObservedActor { + if (input.sessions.length === 0) { + return { + role: input.role, + sessionIds: [], + actualModel: { + kind: "unobserved", + reason: + input.role === "reviewer" + ? "reviewer-child-not-observed" + : "no-completed-assistant", + }, + }; + } + const observations = input.sessions.map((session) => + extractObservedModelIdentity(session.messages), + ); + const observed = observations.filter( + ( + observation, + ): observation is Extract => + observation.kind === "observed", + ); + const unique = new Map( + observed.map((observation) => [ + `${observation.value.providerID}\u0000${observation.value.modelID}`, + observation, + ]), + ); + const unavailable = observations.find( + (observation) => observation.kind === "unobserved", + ); + const actualModel: ObservedModelIdentity = + unique.size > 1 + ? { kind: "unobserved", reason: "conflicting-observations" } + : (unavailable ?? + observed[0] ?? { + kind: "unobserved", + reason: "field-unavailable", + }); + return { + role: input.role, + sessionIds: input.sessions.map((session) => session.id), + actualModel, + }; +} + +export function reviewerActorObservation(input: { + readonly sessions: readonly { + readonly id: string; + readonly messages: readonly unknown[] | null; + }[]; + readonly childEndpointFailed: boolean; +}): ObservedActor { + const actor = extractObservedActor({ + role: "reviewer", + sessions: input.sessions, + }); + return input.childEndpointFailed + ? { + ...actor, + actualModel: { kind: "unobserved", reason: "endpoint-failure" }, + } + : actor; +} + +export function guidanceLoad( + input: Omit, +): ObservedGuidanceLoad { + return { + ...input, + utf8Bytes: new TextEncoder().encode(input.rawOutput).byteLength, + }; +} diff --git a/evals/provenance.ts b/evals/provenance.ts new file mode 100644 index 0000000..53ec8cb --- /dev/null +++ b/evals/provenance.ts @@ -0,0 +1,309 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { isAbsolute, normalize, sep } from "node:path"; +import { promisify } from "node:util"; +import { createFileSourceIdentityProvider } from "../src/infrastructure/fs/source-identity.js"; +import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; +import { normalizeRecorded, REDACTED } from "./cassette.js"; +import type { + ArtifactIdentity, + EvaluatorIdentity, + InstructionDelivery, + ModelIdentity, +} from "./report.js"; + +const exec = promisify(execFile); + +type ArchiveEntry = { + readonly path: string; + readonly kind: "file" | "directory"; + readonly bytes: Buffer; +}; + +export type WorkingSourceIdentity = Pick< + ArtifactIdentity, + "sourceCommit" | "sourceTreeSha256" +>; + +export type RequestedModelInput = { + readonly modelId: string; + readonly gateway: string | null; + readonly family: string; + readonly revision: string | null; +}; + +export type EvaluatorIdentityInput = { + readonly sourceCommit: string; + readonly caseCatalog: unknown; + readonly policyCatalog: unknown; + readonly graderBundle: unknown; +}; + +export type InstructionInput = { + readonly source: InstructionDelivery["source"]; + readonly name: string; + readonly sequence: number; + readonly text: string; +}; + +export type RedactedTranscript = { + readonly text: string; + readonly sha256: string; +}; + +function sha256(bytes: Uint8Array): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +function text(value: Buffer | string): string { + return Buffer.isBuffer(value) ? value.toString("utf8") : value; +} + +async function run(command: string, args: readonly string[]): Promise { + const result = await exec(command, [...args], { + encoding: "buffer", + maxBuffer: 64 * 1024 * 1024, + }); + return Buffer.isBuffer(result.stdout) + ? result.stdout + : Buffer.from(result.stdout); +} + +async function tar(args: readonly string[]): Promise { + let unavailable: unknown = null; + for (const command of ["bsdtar", "tar"]) { + try { + return await run(command, args); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + unavailable = error; + } + } + throw new Error( + "A bsdtar or tar executable is required to inspect eval artifacts.", + { + cause: unavailable, + }, + ); +} + +function archivePath(path: string): string { + const normalized = normalize(path).replaceAll("\\", "/"); + if ( + path.length === 0 || + path.startsWith("-") || + isAbsolute(path) || + path.includes("\\") || + normalized === "." || + normalized === ".." || + normalized.startsWith(`..${sep}`) || + normalized.split("/").includes("..") + ) { + throw new Error(`Unsafe tar archive path: ${JSON.stringify(path)}.`); + } + return normalized.replace(/\/$/, ""); +} + +async function archiveEntries( + tarballPath: string, +): Promise { + const listed = text(await tar(["-tzf", tarballPath])); + const paths = listed.split("\n").filter(Boolean).map(archivePath); + const verbose = text(await tar(["-tvzf", tarballPath])) + .split("\n") + .filter(Boolean); + if (verbose.length !== paths.length) { + throw new Error("Tar archive listing changed while computing provenance."); + } + const known = new Set(); + const entries: ArchiveEntry[] = []; + for (const [index, path] of paths.entries()) { + if (known.has(path)) + throw new Error(`Duplicate tar archive path: ${path}.`); + known.add(path); + const type = verbose[index]?.[0]; + if (type !== "-" && type !== "d") { + throw new Error(`Unsupported tar archive entry type for ${path}.`); + } + entries.push({ + path, + kind: type === "d" ? "directory" : "file", + bytes: + type === "d" + ? Buffer.alloc(0) + : await tar(["-xOzf", tarballPath, path]), + }); + } + return entries.toSorted((left, right) => left.path.localeCompare(right.path)); +} + +function packageVersion(entries: readonly ArchiveEntry[]): string { + const manifest = entries.find( + (entry) => entry.path === "package/package.json", + ); + if (manifest?.kind !== "file") + throw new Error("Packed artifact is missing package/package.json."); + const parsed: unknown = JSON.parse(manifest.bytes.toString("utf8")); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error( + "Packed artifact package.json must contain a string version.", + ); + } + const version = Reflect.get(parsed, "version"); + if (typeof version !== "string") { + throw new Error( + "Packed artifact package.json must contain a string version.", + ); + } + return version; +} + +async function gitCommit(repositoryRoot: string): Promise { + return text( + await run("git", ["-C", repositoryRoot, "rev-parse", "HEAD"]), + ).trim(); +} + +export async function inspectWorkingSource( + repositoryRoot: string, +): Promise { + const sourceCommit = await gitCommit(repositoryRoot); + const sourceTreeSha256 = + await createFileSourceIdentityProvider( + repositoryRoot, + ).computeSourceDigest(); + if ((await gitCommit(repositoryRoot)) !== sourceCommit) { + throw new Error("Git commit changed while computing source identity."); + } + return { sourceCommit, sourceTreeSha256 }; +} + +export async function tarballSha256(tarballPath: string): Promise { + return sha256(await readFile(tarballPath)); +} + +export async function unpackedManifestSha256( + tarballPath: string, +): Promise { + const entries = await archiveEntries(tarballPath); + return canonicalSha256( + "flow-unpacked-tar-manifest-v1", + entries.map((entry) => ({ + path: entry.path, + kind: entry.kind, + sha256: sha256(entry.bytes), + })), + ); +} + +export async function inspectArtifact(input: { + readonly repositoryRoot: string; + readonly tarballPath: string; +}): Promise { + const source = await inspectWorkingSource(input.repositoryRoot); + const tarballDigest = await tarballSha256(input.tarballPath); + const entries = await archiveEntries(input.tarballPath); + if ((await tarballSha256(input.tarballPath)) !== tarballDigest) { + throw new Error("Packed artifact changed while computing provenance."); + } + return { + packageVersion: packageVersion(entries), + ...source, + tarballSha256: tarballDigest, + unpackedManifestSha256: canonicalSha256( + "flow-unpacked-tar-manifest-v1", + entries.map((entry) => ({ + path: entry.path, + kind: entry.kind, + sha256: sha256(entry.bytes), + })), + ), + }; +} + +export function evaluatorIdentity( + input: EvaluatorIdentityInput, +): EvaluatorIdentity { + return { + sourceCommit: input.sourceCommit, + caseCatalogSha256: canonicalSha256( + "flow-evaluator-case-catalog-v1", + input.caseCatalog, + ), + policyCatalogSha256: canonicalSha256( + "flow-evaluator-policy-catalog-v1", + input.policyCatalog, + ), + graderBundleSha256: canonicalSha256( + "flow-evaluator-grader-bundle-v1", + input.graderBundle, + ), + }; +} + +export function hostConfigSha256(config: unknown): string { + return canonicalSha256("flow-eval-host-config-v1", config); +} + +export function normalizeRequestedModel( + input: RequestedModelInput, +): ModelIdentity { + const boundary = input.modelId.indexOf("/"); + if (boundary <= 0 || boundary === input.modelId.length - 1) { + throw new Error( + `Model id ${JSON.stringify(input.modelId)} must be providerID/modelID.`, + ); + } + return { + routeProvider: input.modelId.slice(0, boundary), + gateway: input.gateway, + family: input.family, + model: input.modelId.slice(boundary + 1), + revision: input.revision, + }; +} + +export function instructionDelivery( + input: InstructionInput, +): InstructionDelivery { + if (!input.text.isWellFormed()) { + throw new Error( + "Instruction text must contain only Unicode scalar values.", + ); + } + const bytes = new TextEncoder().encode(input.text); + return { + source: input.source, + name: input.name, + sequence: input.sequence, + sha256: sha256(bytes), + bytes: bytes.byteLength, + }; +} + +const SENSITIVE_FIELD = + /(?:authorization|credential|password|passwd|secret|token|api[_-]?key|client[_-]?secret)/i; + +function redactSensitiveFields(value: unknown): unknown { + if (Array.isArray(value)) return value.map(redactSensitiveFields); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + SENSITIVE_FIELD.test(key) ? REDACTED : redactSensitiveFields(item), + ]), + ); + } + return value; +} + +export function redactTranscript(input: { + readonly value: unknown; + readonly projectPath: string; +}): RedactedTranscript { + const text = canonicalJson( + redactSensitiveFields(normalizeRecorded(input.value, input.projectPath)), + ); + return { text, sha256: sha256(new TextEncoder().encode(text)) }; +} diff --git a/evals/run.ts b/evals/run.ts index 2314802..a1b48ac 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -47,6 +47,21 @@ import { type ReviewerActivity, reviewerActivity, } from "./metrics.js"; +import { + evaluatorIdentity, + hostConfigSha256, + inspectArtifact, + instructionDelivery, + normalizeRequestedModel, + redactTranscript, + tarballSha256, +} from "./provenance.js"; +import type { + ArtifactIdentity, + EvaluatorIdentity, + InstructionDelivery, + ModelIdentity, +} from "./report.js"; import { SCENARIOS } from "./scenarios.js"; const SURFACES: FlowPromptSurfaceName[] = [ @@ -129,6 +144,17 @@ type RunResult = { questions: readonly string[]; durationMs: number; hostError: string | null; + provenance: { + readonly artifact: ArtifactIdentity; + readonly evaluator: EvaluatorIdentity; + readonly hostConfigSha256: string; + readonly actors: readonly (NonNullable[number] & { + readonly requestedModelId: string; + readonly requestedModel: ModelIdentity; + })[]; + readonly instructions: readonly InstructionDelivery[]; + readonly transcript: { readonly sha256: string; readonly text: string }; + }; error?: string; }; @@ -144,6 +170,17 @@ type RunResult = { */ const MAX_CONCURRENCY = 4; +function legacyRequestedModel(modelId: string): ModelIdentity { + const boundary = modelId.indexOf("/"); + const routedModel = boundary >= 0 ? modelId.slice(boundary + 1) : modelId; + return normalizeRequestedModel({ + modelId, + gateway: routedModel.includes("/") ? modelId.slice(0, boundary) : null, + family: routedModel, + revision: null, + }); +} + /** One attempt to run, and the slot its result belongs in. */ type Job = { readonly model: string; @@ -406,10 +443,29 @@ async function main(): Promise { const cassettes: Cassette[] = []; const hostPlatform = normalizeEvidencePlatform(process.platform); try { - const packageCache = await preparePackageCache( - await packPlugin(repositoryRoot, packDir), - packDir, - ); + const tarball = await packPlugin(repositoryRoot, packDir); + const artifact = await inspectArtifact({ + repositoryRoot, + tarballPath: tarball, + }); + const evaluator = evaluatorIdentity({ + sourceCommit: artifact.sourceCommit, + caseCatalog: selected.map((scenario) => ({ + id: scenario.id, + files: Object.keys(scenario.files).sort(), + steps: scenario.steps.map((step) => ({ + command: step.command, + arguments: step.arguments, + freshSession: step.freshSession === true, + })), + })), + policyCatalog: { models, repeat, opencodeVersion }, + graderBundle: { sourceTreeSha256: artifact.sourceTreeSha256 }, + }); + const packageCache = await preparePackageCache(tarball, packDir); + if ((await tarballSha256(tarball)) !== artifact.tarballSha256) { + throw new Error("Packed artifact changed before host installation."); + } await preflight(packageCache, opencodeVersion, models); // One queue per model, run concurrently. The attempts are already independent — // each boots its own OpenCode host on its own free port over its own temp @@ -431,6 +487,15 @@ async function main(): Promise { /** One attempt, start to finish, printing a single line when it lands. */ const runAttempt = async (job: Job): Promise => { const { model, scenario, attempt } = job; + const requestedReviewerModel = + process.env.OPENCODE_FLOW_REVIEWER_MODEL?.trim() || model; + const reviewerStepsText = + process.env.OPENCODE_FLOW_REVIEWER_STEPS?.trim() ?? ""; + const requestedReviewerSteps = + /^[1-9][0-9]*$/.test(reviewerStepsText) && + Number(reviewerStepsText) <= 1000 + ? Number(reviewerStepsText) + : null; const label = `${scenario.id} @ ${model} (${attempt}/${repeat})`; let cassette: Cassette | null = null; const started = Date.now(); @@ -502,6 +567,31 @@ async function main(): Promise { ...(outcome.session ? [outcome.session] : []), ...outcome.archives, ] as MetricSession[]; + const actors = (outcome.actors ?? []).map((actor) => ({ + ...actor, + requestedModelId: + actor.role === "manager" ? model : requestedReviewerModel, + requestedModel: legacyRequestedModel( + actor.role === "manager" ? model : requestedReviewerModel, + ), + })); + const instructions = (outcome.guidanceLoads ?? []).map((load) => + instructionDelivery({ + source: "guidance", + name: load.id ?? "unknown-guidance", + sequence: load.sequence, + text: load.rawOutput, + }), + ); + const transcript = redactTranscript({ + projectPath: host.project, + value: { + actors, + guidanceLoads: outcome.guidanceLoads ?? [], + calls: outcome.allCalls, + finalText: outcome.finalText, + }, + }); const result: RunResult = { scenario: scenario.id, model, @@ -532,6 +622,21 @@ async function main(): Promise { questions: askedQuestions(outcome), durationMs: outcome.durationMs, hostError: outcome.hostError, + provenance: { + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + plugin: `opencode-plugin-flow@${packageJson.version}`, + model, + reviewerModel: requestedReviewerModel, + reviewerSteps: requestedReviewerSteps, + platform: hostPlatform, + }), + actors, + instructions, + transcript, + }, }; const fidelity: FidelityNote[] = []; if (stepError) fidelity.push("run-aborted"); @@ -577,6 +682,10 @@ async function main(): Promise { return { slot: job.slot, result, cassette }; } catch (error) { const message = error instanceof Error ? error.message : String(error); + const transcript = redactTranscript({ + projectPath: host?.project ?? "", + value: { environmentError: message }, + }); console.log(`- ${label} ... ENVIRONMENT (${message.split("\n")[0]})`); // Reaching here means the scenario never got a model turn, with one // exception: a host that answered but rejected every turn is thrown @@ -616,6 +725,21 @@ async function main(): Promise { questions: [], durationMs: Date.now() - started, hostError: null, + provenance: { + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + plugin: `opencode-plugin-flow@${packageJson.version}`, + model, + reviewerModel: requestedReviewerModel, + reviewerSteps: requestedReviewerSteps, + platform: hostPlatform, + }), + actors: [], + instructions: [], + transcript, + }, error: message, }, }; diff --git a/tests/eval-reporting.test.ts b/tests/eval-reporting.test.ts index 13cdcd5..2347e35 100644 --- a/tests/eval-reporting.test.ts +++ b/tests/eval-reporting.test.ts @@ -19,6 +19,13 @@ import { sessionBoundaries, syncProviderCredentialsBack, } from "../evals/harness.js"; +import { + extractObservedActor, + extractObservedModelIdentity, + guidanceLoad, + reviewerActorObservation, + selectLineageValidatedReviewers, +} from "../evals/host-observation.js"; import { aggregateOperationalMetrics, completionHonesty, @@ -178,6 +185,174 @@ describe("eval run classification", () => { }); }); +describe("eval actor and instruction observations", () => { + const assistant = (info: Record) => ({ + info: { role: "assistant", time: { created: 1, completed: 2 }, ...info }, + parts: [], + }); + + test("extracts nested and top-level model identities, including gateway ids", () => { + expect( + extractObservedModelIdentity([ + assistant({ + model: { + providerID: "openrouter", + modelID: "openai/gpt-5.6-sol", + }, + }), + ]), + ).toEqual({ + kind: "observed", + value: { + providerID: "openrouter", + modelID: "openai/gpt-5.6-sol", + }, + }); + expect( + extractObservedModelIdentity([ + assistant({ providerID: "anthropic", modelID: "claude-sonnet" }), + ]), + ).toEqual({ + kind: "observed", + value: { providerID: "anthropic", modelID: "claude-sonnet" }, + }); + }); + + test("refuses incomplete, errored, and conflicting model observations", () => { + expect(extractObservedModelIdentity(null)).toEqual({ + kind: "unobserved", + reason: "endpoint-failure", + }); + expect( + extractObservedModelIdentity([ + { info: { role: "assistant", time: { created: 1 } }, parts: [] }, + ]), + ).toEqual({ kind: "unobserved", reason: "no-completed-assistant" }); + expect( + extractObservedModelIdentity([ + { + info: { + role: "assistant", + time: { created: 1, completed: 2 }, + error: { name: "ProviderError" }, + }, + parts: [], + }, + ]), + ).toEqual({ kind: "unobserved", reason: "no-completed-assistant" }); + expect(extractObservedModelIdentity([assistant({})])).toEqual({ + kind: "unobserved", + reason: "field-unavailable", + }); + expect( + extractObservedModelIdentity([ + assistant({ providerID: "a", modelID: "one" }), + assistant({ providerID: "b", modelID: "two" }), + ]), + ).toEqual({ kind: "unobserved", reason: "conflicting-observations" }); + }); + + test("counts only lineage-validated reviewer children and preserves actor ids", () => { + const children = [ + { id: "worker", agent: "flow-worker", parentID: "parent" }, + { id: "wrong", agent: "flow-reviewer", parentID: "other" }, + { id: "reviewer", agent: "flow-reviewer", parentID: "parent" }, + ] as const; + expect(selectLineageValidatedReviewers(["parent"], children)).toEqual([ + children[2], + ]); + expect(extractObservedActor({ role: "reviewer", sessions: [] })).toEqual({ + role: "reviewer", + sessionIds: [], + actualModel: { + kind: "unobserved", + reason: "reviewer-child-not-observed", + }, + }); + expect( + reviewerActorObservation({ + sessions: [], + childEndpointFailed: true, + }), + ).toEqual({ + role: "reviewer", + sessionIds: [], + actualModel: { kind: "unobserved", reason: "endpoint-failure" }, + }); + expect( + reviewerActorObservation({ + sessions: [ + { + id: "reviewer", + messages: [assistant({ providerID: "a", modelID: "reviewer" })], + }, + ], + childEndpointFailed: true, + }), + ).toMatchObject({ + sessionIds: ["reviewer"], + actualModel: { kind: "unobserved", reason: "endpoint-failure" }, + }); + expect( + extractObservedActor({ + role: "manager", + sessions: [ + { + id: "parent", + messages: [ + assistant({ + model: { providerID: "openrouter", modelID: "x/y" }, + }), + ], + }, + ], + }), + ).toEqual({ + role: "manager", + sessionIds: ["parent"], + actualModel: { + kind: "observed", + value: { providerID: "openrouter", modelID: "x/y" }, + }, + }); + expect( + extractObservedActor({ + role: "manager", + sessions: [ + { + id: "parent", + messages: [assistant({ providerID: "a", modelID: "m" })], + }, + { id: "resume", messages: null }, + ], + }), + ).toEqual({ + role: "manager", + sessionIds: ["parent", "resume"], + actualModel: { kind: "unobserved", reason: "endpoint-failure" }, + }); + }); + + test("measures raw guidance output in UTF-8 bytes", () => { + expect( + guidanceLoad({ + sequence: 3, + sessionIndex: 1, + agent: "", + id: "flow-plan", + rawOutput: "plan café", + }), + ).toEqual({ + sequence: 3, + sessionIndex: 1, + agent: "", + id: "flow-plan", + rawOutput: "plan café", + utf8Bytes: 10, + }); + }); +}); + describe("eval session boundaries", () => { const calls = (indices: number[]) => indices.map((sessionIndex) => ({ sessionIndex })); diff --git a/tests/provenance.test.ts b/tests/provenance.test.ts new file mode 100644 index 0000000..e1121ba --- /dev/null +++ b/tests/provenance.test.ts @@ -0,0 +1,236 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { gzipSync } from "node:zlib"; +import { + evaluatorIdentity, + hostConfigSha256, + inspectArtifact, + inspectWorkingSource, + instructionDelivery, + normalizeRequestedModel, + redactTranscript, + unpackedManifestSha256, +} from "../evals/provenance.js"; + +const exec = promisify(execFile); +const temporary: string[] = []; + +afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +async function directory(): Promise { + const path = await mkdtemp(join(tmpdir(), "flow-provenance-")); + temporary.push(path); + return path; +} + +async function command(cwd: string, args: readonly string[]): Promise { + await exec("git", args, { cwd }); +} + +async function repository(): Promise { + const root = await directory(); + await command(root, ["init", "--initial-branch=main"]); + await command(root, ["config", "user.email", "eval@example.com"]); + await command(root, ["config", "user.name", "Eval"]); + await writeFile(join(root, "source.ts"), "export const source = 1;\n"); + await command(root, ["add", "source.ts"]); + await command(root, ["commit", "-m", "fixture"]); + return root; +} + +async function artifact( + root: string, + name: string, + contents: string, +): Promise { + const packageDirectory = join(root, "package"); + await mkdir(packageDirectory, { recursive: true }); + await writeFile( + join(packageDirectory, "package.json"), + '{"name":"fixture","version":"1.2.3"}\n', + ); + await writeFile(join(packageDirectory, "index.js"), contents); + const path = join(root, name); + await exec("tar", ["-czf", path, "-C", root, "package"]); + return path; +} + +function unsafeTarball(path: string): Promise { + const contents = Buffer.from("unsafe\n"); + const header = Buffer.alloc(512); + header.write("../outside.txt", 0, "utf8"); + header.write("0000644\0", 100, "ascii"); + header.write("0000000\0", 108, "ascii"); + header.write("0000000\0", 116, "ascii"); + header.write(`${contents.byteLength.toString(8).padStart(11, "0")}\0`, 124); + header.write("00000000000\0", 136, "ascii"); + header.fill(" ", 148, 156); + header.write("0", 156, "ascii"); + header.write("ustar\0", 257, "ascii"); + header.write("00", 263, "ascii"); + const checksum = header.reduce((total, byte) => total + byte, 0); + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii"); + const padding = Buffer.alloc((512 - (contents.byteLength % 512)) % 512); + return writeFile( + path, + gzipSync(Buffer.concat([header, contents, padding, Buffer.alloc(1024)])), + ); +} + +describe("eval provenance", () => { + test("binds a commit and dirty working-content digest separately", async () => { + const root = await repository(); + const clean = await inspectWorkingSource(root); + await writeFile(join(root, "source.ts"), "export const source = 2;\n"); + const dirty = await inspectWorkingSource(root); + expect(dirty.sourceCommit).toBe(clean.sourceCommit); + expect(dirty.sourceTreeSha256).not.toBe(clean.sourceTreeSha256); + }); + + test("binds exact tar bytes and an unpacked manifest", async () => { + const root = await repository(); + const first = await artifact( + root, + "first.tgz", + "export const value = 1;\n", + ); + const firstIdentity = await inspectArtifact({ + repositoryRoot: root, + tarballPath: first, + }); + const second = await artifact( + root, + "second.tgz", + "export const value = 2;\n", + ); + const secondIdentity = await inspectArtifact({ + repositoryRoot: root, + tarballPath: second, + }); + expect(firstIdentity.packageVersion).toBe("1.2.3"); + expect(secondIdentity.tarballSha256).not.toBe(firstIdentity.tarballSha256); + expect(secondIdentity.unpackedManifestSha256).not.toBe( + firstIdentity.unpackedManifestSha256, + ); + }); + + test("rejects duplicate archive paths", async () => { + const root = await repository(); + await artifact(root, "package.tgz", "export {};\n"); + const duplicate = join(root, "duplicate.tgz"); + await exec("tar", ["-czf", duplicate, "-C", root, "package", "package"]); + await expect(unpackedManifestSha256(duplicate)).rejects.toThrow( + "Duplicate tar archive path", + ); + }); + + test("rejects unsafe archive paths before extraction", async () => { + const root = await repository(); + const unsafe = join(root, "unsafe.tgz"); + await unsafeTarball(unsafe); + await expect(unpackedManifestSha256(unsafe)).rejects.toThrow( + "Unsafe tar archive path", + ); + }); + + test("preserves gateway model ids after their first slash", () => { + expect( + normalizeRequestedModel({ + modelId: "openrouter/openai/gpt-5.6-sol", + gateway: "openrouter", + family: "gpt-5.6", + revision: null, + }), + ).toEqual({ + routeProvider: "openrouter", + gateway: "openrouter", + family: "gpt-5.6", + model: "openai/gpt-5.6-sol", + revision: null, + }); + }); + + test("hashes actual UTF-8 instruction bytes and canonical evaluator/config inputs", () => { + const instruction = instructionDelivery({ + source: "guidance", + name: "flow-run", + sequence: 3, + text: "€", + }); + expect(instruction.bytes).toBe(3); + expect(instruction.sha256).toMatch(/^sha256:[a-f0-9]{64}$/); + const evaluator = evaluatorIdentity({ + sourceCommit: "commit", + caseCatalog: { b: 2, a: 1 }, + policyCatalog: { version: 1 }, + graderBundle: ["grader"], + }); + expect(evaluator.caseCatalogSha256).toMatch(/^sha256:[a-f0-9]{64}$/); + expect(hostConfigSha256({ b: 2, a: 1 })).toBe( + hostConfigSha256({ a: 1, b: 2 }), + ); + expect( + hostConfigSha256({ reviewerModel: "a/model", reviewerSteps: null }), + ).not.toBe( + hostConfigSha256({ reviewerModel: "b/model", reviewerSteps: 8 }), + ); + }); + + test("rejects malformed Unicode instruction text", () => { + expect(() => + instructionDelivery({ + source: "guidance", + name: "broken", + sequence: 0, + text: "\ud800", + }), + ).toThrow("Unicode scalar values"); + }); + + test("retains only canonical redacted transcript bytes", () => { + const transcript = redactTranscript({ + projectPath: "/private/eval/project", + value: { + output: + "/private/eval/project/src/index.ts api_key=super-secret-value sk-proj-abcdefghijklmnopqr", + }, + }); + expect(transcript.text).toContain("/src/index.ts"); + expect(transcript.text).toContain("[redacted]"); + expect(transcript.text).not.toContain("super-secret-value"); + expect(transcript.text).not.toContain("sk-proj-abcdefghijklmnopqr"); + expect(transcript.sha256).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + + test("redacts transcript object keys as well as values", () => { + const transcript = redactTranscript({ + projectPath: "/private/eval/project", + value: { + "/private/eval/project/src/index.ts": "ok", + "api_key=super-secret-value": "ok", + }, + }); + expect(transcript.text).not.toContain("/private/eval/project"); + expect(transcript.text).not.toContain("super-secret-value"); + }); + + test("redacts short values under sensitive transcript fields", () => { + const transcript = redactTranscript({ + projectPath: "/tmp/project", + value: { token: "short", api_key: "abc", safe: "ok" }, + }); + expect(transcript.text).not.toContain("short"); + expect(transcript.text).not.toContain("abc"); + expect(transcript.text).toContain('"safe":"ok"'); + }); +}); From 84667cbf0c26cd5f491ebcf15c3458e589215fe5 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 06:16:02 +0200 Subject: [PATCH 2/2] Record Phase 3 provenance evidence --- .../evidence/phase-3-pilot.json | 33 ++++++++++++++++ .../evidence/phase-3-review.md | 39 +++++++++++++++++++ .../02-eval-engineering/phase-3-provenance.md | 10 +++++ .audit/eval-engineering.tsv | 5 +++ 4 files changed, 87 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-3-pilot.json create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-3-review.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-3-pilot.json b/.agents/plans/02-eval-engineering/evidence/phase-3-pilot.json new file mode 100644 index 0000000..52f1013 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-3-pilot.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "recordedAt": "2026-08-25T04:14:14.980Z", + "scenario": "happy-path", + "modelRoute": "opencode/gpt-5.6-sol", + "opencodeVersion": "1.18.6", + "passed": true, + "artifact": { + "packageVersion": "8.1.1", + "sourceCommit": "04d3f6ee34bc92373982f9599beb6ce66c08f6c8", + "sourceTreeSha256": "sha256:605b10580561bd0bc592614b550eb5516ec9240fd2553dd3e92e3a36a30821d9", + "tarballSha256": "sha256:c89f7363248ccc3e3f69728c1aa42044a25938cd533e470d9f73ef08bc64ad24", + "unpackedManifestSha256": "sha256:398e6003b3c4d16f5e49dad9dbe734666149ee753e803d020501c863c19f0183" + }, + "evaluator": { + "sourceCommit": "04d3f6ee34bc92373982f9599beb6ce66c08f6c8", + "caseCatalogSha256": "sha256:b9bd76c55d20db6daca1b40bf4118fae66f829dafec4b9388288d6a603a47be3", + "policyCatalogSha256": "sha256:708363968ed83fedf84f0310baa65f2b8b172c8c5ccdc0d090cf50bae2b51c9d", + "graderBundleSha256": "sha256:3eba0223d2f005abd4c429e05dd86ddb4c20a495725be69d852ea8d36f09431a" + }, + "hostConfigSha256": "sha256:fc5de04f3e595f7c84d9310369a9ebc3118600c1bbc342be67e65d8d72f5b64e", + "actors": [ + { "role": "manager", "requestedModelId": "opencode/gpt-5.6-sol", "actual": "observed", "sessionCount": 1 }, + { "role": "reviewer", "requestedModelId": "opencode/gpt-5.6-sol", "actual": "observed", "sessionCount": 1 } + ], + "instructions": [ + { "source": "guidance", "name": "flow-plan", "sequence": 0, "sha256": "sha256:f9d2f6dbf60f8be5a53a3959517eb9e5307624e75e7aa25960d76db3c8c71d7a", "bytes": 5685 }, + { "source": "guidance", "name": "flow-run", "sequence": 1, "sha256": "sha256:9f451a740456a7a59aca02f4485bdcfe5f64ad3fb57a48980fa09cde40c08050", "bytes": 7933 } + ], + "transcriptSha256": "sha256:875a85c8007daa436be268553319967c35b449ab4ad5d0e663a9a80e8063db0d", + "redactionScanPassed": true, + "unsupportedClaim": "Observed provider and model fields do not independently establish family, gateway, or revision. Phase 4 must preserve actual v2 model identity as unobserved unless those fields become independently available." +} diff --git a/.agents/plans/02-eval-engineering/evidence/phase-3-review.md b/.agents/plans/02-eval-engineering/evidence/phase-3-review.md new file mode 100644 index 0000000..3c753bc --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-3-review.md @@ -0,0 +1,39 @@ +# Phase 3 Interrogate review + +## Intent + +Phase 3 must bind real eval output to exact source, packed bytes, evaluator inputs, +host configuration, requested and observed actors, delivered instructions, and a +redacted transcript. It must preserve endpoint and identity limitations rather +than fill missing observations from configuration. + +## Acted on + +- The runner now consumes provenance helpers and emits the binding on every result. +- Tarball bytes are checked before inspection and after cache installation. +- Unpacked manifests bind file and directory type. Other archive entry types are + rejected. +- Reviewer child endpoint failures remain explicit, including partial discovery. +- Multi-session actor identity fails closed when any session is unobserved. +- Transcript redaction covers object keys, short sensitive-field values, paths, + and credential-shaped strings. +- Instruction text rejects malformed Unicode and hashes actual UTF-8 delivery. +- Requested manager and reviewer identities are emitted separately from raw host + observations. Reviewer model and step configuration are included in host hashes. +- Pure actor parsing moved out of the large harness into `host-observation.ts`. + +## Lead judgment + +The pinned host independently exposes provider and model fields, but not the full +family, gateway, and revision tuple required by v2 `ModelIdentity`. The legacy +pilot retains the raw observation. Phase 4 must emit actual identity as +`unobserved` unless a later host exposes all required fields. Cross-family claims +remain unavailable otherwise. + +## Verdict + +`VERIFIED`. Four-model recheck found no unresolved blocker. The final paid packed +`happy-path` passed with observed manager and reviewer roles, two delivered +guidance records, exact artifact/evaluator/host hashes, and a clean transcript +redaction scan. The full repository gate passes 462 tests, one intentional skip, +and zero failures. diff --git a/.agents/plans/02-eval-engineering/phase-3-provenance.md b/.agents/plans/02-eval-engineering/phase-3-provenance.md index 45086b1..ba99339 100644 --- a/.agents/plans/02-eval-engineering/phase-3-provenance.md +++ b/.agents/plans/02-eval-engineering/phase-3-provenance.md @@ -14,6 +14,13 @@ evidence outside the report and bind each attempt to those observed facts. transcript artifacts. - `evals/harness.ts`. Capture actor metadata and observed guidance loads through the Phase 0 field map. +- `evals/host-observation.ts`. Keep Phase 0 field-map parsing and lineage rules + pure and independently testable. +- `evals/run.ts`. Attach exact provenance, observed actors, delivered guidance, + and redacted transcript evidence to every legacy result before Phase 4 adapts it. +- `evals/cassette.ts`. Redact sensitive object keys as well as values. +- `tests/provenance.test.ts`. Cover source, archive, configuration, instruction, + and transcript boundaries without paid calls. - `tests/eval-reporting.test.ts`. Cover hash swaps, dirty trees, unobserved actor fallback, gateway ids, lazy guidance bytes, transcript retention, and redaction. @@ -29,5 +36,8 @@ Static. Focused reporting tests and `bun run check`. Runtime. Run one paid `happy-path` attempt, read the actual parent and child model fields, and prove a tarball swap invalidates the externally computed binding. +Runtime evidence. [Final paid pilot](evidence/phase-3-pilot.json) and +[Interrogate review](evidence/phase-3-review.md). + Stop gate. Cross-family reviewer evidence stays unavailable on hosts that cannot expose actual child-session identity. Other evidence work may continue. diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index f7f107e..43fa53e 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -31,3 +31,8 @@ ts phase decision why evidence result 2026-08-25T03:43:39Z phase-2 recovered the implementation after a delegate usage-limit stop the partial module and tests were inspectable, so the lead could finish without discarding verified work evals/analysis.ts; tests/atomic-analysis.test.ts; tests/advisory-analysis.test.ts VERIFIED focused gate green 2026-08-25T03:43:39Z phase-2 fixed the multi-model Interrogate findings report-only scoping, provider fallback, exact provenance sets, reviewer incompleteness, analysis-kind guards, and reserve eligibility affected decision integrity .agents/plans/02-eval-engineering/evidence/phase-2-review.md VERIFIED no unresolved blocker 2026-08-25T03:43:39Z phase-2 ran Deslop and the whole repository gate the analysis and tests must remain small, pure, and regression-free bun run check VERIFIED 448 pass, 1 skip, 0 fail +2026-08-25T04:14:31Z phase-3 started from merged Phase 2 main exact provenance must build on the atomic analyzer without cutting over v2 emission git status on codex/eval-phase-3 at 04d3f6e VERIFIED clean baseline, 448 pass, 1 skip, 0 fail +2026-08-25T04:14:31Z phase-3 integrated exact provenance into the real runner helper-only evidence would not bind paid attempts to the packed bytes and observed actors evals/provenance.ts; evals/host-observation.ts; evals/harness.ts; evals/run.ts VERIFIED focused and full gates green +2026-08-25T04:14:31Z phase-3 fixed the multi-model Interrogate findings archive types, endpoint completeness, requested actors, host config, transcript fields, Unicode, and production integration were evidence boundaries .agents/plans/02-eval-engineering/evidence/phase-3-review.md VERIFIED no unresolved blocker +2026-08-25T04:14:31Z phase-3 reran the final paid packed-host pilot the committed observation must be generated by the final code and exact tarball evals/results/2026-08-25T04-14-14-980Z.json; .agents/plans/02-eval-engineering/evidence/phase-3-pilot.json VERIFIED happy-path pass, observed manager and reviewer, redaction scan clean +2026-08-25T04:14:31Z phase-3 ran Deslop and the whole repository gate the phase must finish reviewable and regression-free bun run check VERIFIED 462 pass, 1 skip, 0 fail