From 377dbe57b0ff909d8bcf94f5e71f898acff2001f Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:31:52 +0200 Subject: [PATCH 1/4] docs(release): design exact canary gate --- .../evidence/phase-9-architecture.md | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-9-architecture.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-9-architecture.md b/.agents/plans/02-eval-engineering/evidence/phase-9-architecture.md new file mode 100644 index 0000000..3d63cd1 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-9-architecture.md @@ -0,0 +1,72 @@ +# Phase 9 architecture synthesis + +## Arena decision + +Three candidates converged on a separate manual canary gate. The independent judge +selected the exact-artifact, write-once record base and grafted tag-only publication, +ActorIdentity-shaped evidence, checklist expiry, redacted artifact hashes, and an +exact canary digest inside the final release decision input. + +## Operator flow + +```bash +bun run build +bun pm pack --destination .release-artifacts +bun run eval:canary -- prepare \ + --artifact .release-artifacts/opencode-plugin-flow-.tgz \ + --out .release-artifacts/canary- + +# Run the prepared local-plugin fixture in OpenCode and save its session evidence. +bun run eval:canary -- record \ + --prepared .release-artifacts/canary-/prepared.json \ + --status passed --operator \ + --host-config --actors --checks \ + --project-path --session --transcript + +# Reissue the decision with the reviewed canary in its input hash. +bun run qualify -- --report --catalog --artifact \ + --canary evals/canary/.json +``` + +`prepare` is non-claiming. It copies the exact tarball, extracts its validated +`dist/index.js` into a project-local `.opencode/plugins/flow.js`, pins the local +plugin dependencies, creates a small canary workspace, and writes the immutable +checklist/artifact preparation manifest outside Git under `.release-artifacts`. + +`record` redacts JSON-shaped session and transcript evidence, removes workspace and +session identifiers, writes sanitized artifacts, and then publishes exactly one +canonical `evals/canary/.json`. Byte-identical replay succeeds; a changed +record conflicts. Passed, failed, and incomplete attempts are all durable evidence. + +## Record and gate + +The strict canary record binds: + +- full `ArtifactIdentity` and `v` tag; +- exact checklist version, hash, and required check set; +- passed, failed, or incomplete status; +- explicit operator, recorded time, and checklist-derived 72-hour expiry; +- host configuration digest and manager/reviewer `ActorIdentity` observations; +- relative sanitized session/transcript paths, byte counts, and SHA-256 digests; +- its own canonical record hash. + +Passed requires every check true, both sanitized artifacts, at least one actor, and +fresh internally consistent timestamps. Failed requires at least one false check. +Incomplete can preserve partial evidence but never qualifies publication. + +The scheduled evaluation decision remains `canarySha256: null`. After manual canary +recording, qualification validates the exact artifact/tag/fresh passed canary and +writes a distinct canary-bound decision record. `decisionInputSha256` includes +`canarySha256`, so a canary cannot be attached to an older decision after the fact. + +## Workflow + +`release.yml` runs on main and `v*` tags. Main rebuilds and checks the package, +reports missing release decision/canary as `INCONCLUSIVE`, and has no publish job. +Tags rebuild the tarball from the tagged checkout and require an exact VERIFIED +canary-bound decision plus the fresh passed `evals/canary/.json` from that +same checkout before npm or GitHub publication. The temporary Phase 5 stop is +deleted; publication remains tag-only. + +No release is requested in this phase. The manual canary and canary-bound decision +remain pending for the maintainer-run OpenCode session. From b1db76741feea9f600072a6ce9c9048d46736b90 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:31:59 +0200 Subject: [PATCH 2/4] feat(release): add exact artifact canary records --- package.json | 1 + scripts/eval-canary.ts | 690 ++++++++++++++++++++++++++++++++++++++ tests/eval-canary.test.ts | 358 ++++++++++++++++++++ 3 files changed, 1049 insertions(+) create mode 100644 scripts/eval-canary.ts create mode 100644 tests/eval-canary.test.ts diff --git a/package.json b/package.json index f051301..ea18666 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "clean": "bun run scripts/clean-dist.ts", "eval": "bun run evals/run.ts", "eval:smoke": "bun run evals/run.ts -- --repeat 1", + "eval:canary": "bun run scripts/eval-canary.ts", "benchmark": "bun run evals/benchmark-run.ts", "lint": "bunx biome check biome.json src tests scripts evals --files-ignore-unknown=true --vcs-use-ignore-file=true", "release:metadata": "bun run scripts/release-metadata.ts", diff --git a/scripts/eval-canary.ts b/scripts/eval-canary.ts new file mode 100644 index 0000000..cdef16f --- /dev/null +++ b/scripts/eval-canary.ts @@ -0,0 +1,690 @@ +#!/usr/bin/env bun +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + copyFile, + link, + mkdir, + open, + readFile, + stat, + unlink, + writeFile, +} from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { z } from "zod"; +import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; +import { + mapStrings, + normalizeRecorded, + scrubSecrets, +} from "../evals/cassette.js"; +import { inspectArtifact } from "../evals/provenance.js"; +import type { ActorIdentity, ArtifactIdentity } from "../evals/report.js"; + +export const CANARY_CHECKLIST_VERSION = "phase9-canary-v1"; +export const CANARY_CHECK_IDS = [ + "installs-packed-artifact", + "loads-flow-tools", + "saves-plan", + "captures-validation", + "dispatches-reviewer", + "closes-with-delivery", +] as const; +export const CANARY_MAX_AGE_MS = 72 * 60 * 60 * 1_000; + +const checklist = { + version: CANARY_CHECKLIST_VERSION, + checks: CANARY_CHECK_IDS, + maxAgeMs: CANARY_MAX_AGE_MS, +}; +export const CANARY_CHECKLIST_SHA256 = canonicalSha256( + "flow-canary-checklist-v1", + checklist, +); + +const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const TextSchema = z.string().min(1).max(4096).regex(/\S/); +const ModelIdentitySchema = z + .object({ + routeProvider: TextSchema, + gateway: TextSchema.nullable(), + family: TextSchema, + model: TextSchema, + revision: TextSchema.nullable(), + }) + .strict(); +const ObservedModelIdentitySchema = z.discriminatedUnion("kind", [ + z + .object({ kind: z.literal("observed"), value: ModelIdentitySchema }) + .strict(), + z.object({ kind: z.literal("unobserved"), reason: TextSchema }).strict(), +]); +const ActorIdentitySchema = z + .object({ + role: z.enum(["manager", "reviewer"]), + requestedModel: ModelIdentitySchema, + actualModel: ObservedModelIdentitySchema, + sessionIds: z.array(TextSchema), + }) + .strict(); +const RedactedActorIdentitySchema = ActorIdentitySchema.refine( + (actor) => + actor.sessionIds.every((sessionId) => sessionId === ""), + "Canary actor session ids must be redacted.", +); +const ArtifactIdentitySchema = z + .object({ + packageVersion: TextSchema, + sourceCommit: TextSchema, + sourceTreeSha256: DigestSchema, + tarballSha256: DigestSchema, + unpackedManifestSha256: DigestSchema, + }) + .strict(); +const ChecksSchema = z + .object({ + "installs-packed-artifact": z.boolean(), + "loads-flow-tools": z.boolean(), + "saves-plan": z.boolean(), + "captures-validation": z.boolean(), + "dispatches-reviewer": z.boolean(), + "closes-with-delivery": z.boolean(), + }) + .strict(); +const EvidenceRefSchema = z + .object({ + path: TextSchema, + sha256: DigestSchema, + bytes: z.number().int().safe().nonnegative(), + }) + .strict(); + +export const PreparedCanarySchema = z + .object({ + schemaVersion: z.literal(1), + releaseTag: TextSchema, + artifact: ArtifactIdentitySchema, + artifactSha256: DigestSchema, + checklistVersion: z.literal(CANARY_CHECKLIST_VERSION), + checklistSha256: DigestSchema, + preparedAt: z.string().datetime({ offset: true }), + artifactFile: z.literal("artifact.tgz"), + pluginEntrySha256: DigestSchema, + sha256: DigestSchema, + }) + .strict(); +export type PreparedCanary = z.infer; + +export const CanaryRecordSchema = z + .object({ + schemaVersion: z.literal(1), + status: z.enum(["passed", "failed", "incomplete"]), + artifact: ArtifactIdentitySchema, + artifactSha256: DigestSchema, + releaseTag: TextSchema, + operator: TextSchema, + recordedAt: z.string().datetime({ offset: true }), + expiresAt: z.string().datetime({ offset: true }), + checklistVersion: z.literal(CANARY_CHECKLIST_VERSION), + checklistSha256: DigestSchema, + checks: ChecksSchema, + hostConfigSha256: DigestSchema, + actors: z.array(RedactedActorIdentitySchema), + artifacts: z + .object({ + session: EvidenceRefSchema.nullable(), + transcript: EvidenceRefSchema.nullable(), + }) + .strict(), + recordSha256: DigestSchema, + }) + .strict() + .superRefine((record, context) => { + const values = Object.values(record.checks); + if (record.status === "passed" && !values.every(Boolean)) { + context.addIssue({ + code: "custom", + path: ["checks"], + message: "Passed canaries require every check.", + }); + } + if (record.status === "failed" && !values.some((value) => !value)) { + context.addIssue({ + code: "custom", + path: ["checks"], + message: "Failed canaries require a failed check.", + }); + } + if ( + record.status === "passed" && + (record.actors.length === 0 || + record.artifacts.session === null || + record.artifacts.transcript === null) + ) { + context.addIssue({ + code: "custom", + path: ["artifacts"], + message: + "Passed canaries require actors, session, and transcript evidence.", + }); + } + }); +export type CanaryRecord = z.infer; + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +export function artifactIdentitySha256(artifact: ArtifactIdentity): string { + return canonicalSha256("flow-canary-artifact-v1", artifact); +} + +export function canaryRecordSha256( + record: Omit, +): string { + return canonicalSha256("flow-canary-record-v1", record); +} + +export function preparedCanarySha256( + prepared: Omit, +): string { + return canonicalSha256("flow-canary-preparation-v1", prepared); +} + +function parsePreparedCanary(input: unknown): PreparedCanary { + const prepared = PreparedCanarySchema.parse(input); + const { sha256: _sha256, ...withoutHash } = prepared; + if ( + prepared.sha256 !== preparedCanarySha256(withoutHash) || + prepared.releaseTag !== `v${prepared.artifact.packageVersion}` || + prepared.artifactSha256 !== artifactIdentitySha256(prepared.artifact) || + prepared.checklistSha256 !== CANARY_CHECKLIST_SHA256 + ) { + throw new Error("Canary preparation bindings are invalid."); + } + return prepared; +} + +export function parseCanaryRecord( + input: unknown, +): + | { readonly ok: true; readonly value: CanaryRecord } + | { readonly ok: false; readonly issues: readonly string[] } { + const parsed = CanaryRecordSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false, + issues: parsed.error.issues.map((issue) => issue.message), + }; + } + const record = parsed.data; + const { recordSha256: _recordSha256, ...withoutHash } = record; + const issues = [ + ...(record.releaseTag === `v${record.artifact.packageVersion}` + ? [] + : ["Canary tag does not match its artifact version."]), + ...(record.artifactSha256 === artifactIdentitySha256(record.artifact) + ? [] + : ["Canary artifact identity hash is invalid."]), + ...(record.checklistSha256 === CANARY_CHECKLIST_SHA256 + ? [] + : ["Canary checklist hash is invalid."]), + ...(record.recordSha256 === canaryRecordSha256(withoutHash) + ? [] + : ["Canary record hash is invalid."]), + ...(Date.parse(record.expiresAt) - Date.parse(record.recordedAt) === + CANARY_MAX_AGE_MS + ? [] + : ["Canary expiry is not checklist-derived."]), + ]; + return issues.length > 0 + ? { ok: false, issues } + : { ok: true, value: record }; +} + +function syncDirectory(path: string): Promise { + return open(path, "r") + .then(async (handle) => { + try { + await handle.sync(); + } finally { + await handle.close(); + } + }) + .catch(() => {}); +} + +async function writeImmutable(path: string, bytes: Buffer): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + try { + const existing = await readFile(path); + if (existing.equals(bytes)) return; + throw new Error(`Immutable canary artifact conflicts: ${path}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const temporary = `${path}.tmp-${process.pid}-${crypto.randomUUID()}`; + const handle = await open(temporary, "wx", 0o600); + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await link(temporary, path); + await syncDirectory(dirname(path)); + await unlink(temporary); + await syncDirectory(dirname(path)); + } catch (error) { + await unlink(temporary).catch(() => {}); + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + const existing = await readFile(path); + if (existing.equals(bytes)) return; + } + throw error; + } +} + +function extractPluginEntry(artifactPath: string): Buffer { + for (const command of ["bsdtar", "tar"]) { + const result = spawnSync( + command, + ["-xOf", artifactPath, "package/dist/index.js"], + { + encoding: "buffer", + maxBuffer: 32 * 1024 * 1024, + }, + ); + if (result.status === 0 && Buffer.isBuffer(result.stdout)) + return result.stdout; + if ( + result.error && + (result.error as NodeJS.ErrnoException).code === "ENOENT" + ) + continue; + } + throw new Error( + "Canary preparation could not extract package/dist/index.js.", + ); +} + +export async function prepareCanary(input: { + readonly repositoryRoot: string; + readonly artifactPath: string; + readonly outputDirectory: string; + readonly preparedAt?: Date; +}): Promise { + const artifact = await inspectArtifact({ + repositoryRoot: input.repositoryRoot, + tarballPath: input.artifactPath, + }); + const pluginEntry = extractPluginEntry(input.artifactPath); + const fixture = join(input.outputDirectory, "fixture"); + await mkdir(join(fixture, ".opencode", "plugins"), { recursive: true }); + await copyFile( + input.artifactPath, + join(input.outputDirectory, "artifact.tgz"), + ); + await writeFile( + join(fixture, ".opencode", "plugins", "flow.js"), + pluginEntry, + ); + const packageMetadata = JSON.parse( + await readFile(join(input.repositoryRoot, "package.json"), "utf8"), + ) as { + dependencies?: Record; + devDependencies?: Record; + }; + await writeFile( + join(fixture, ".opencode", "package.json"), + `${JSON.stringify( + { + private: true, + dependencies: { + "@opencode-ai/plugin": + packageMetadata.devDependencies?.["@opencode-ai/plugin"], + zod: packageMetadata.dependencies?.zod, + }, + }, + null, + 2, + )}\n`, + ); + await mkdir(join(fixture, "src"), { recursive: true }); + await writeFile( + join(fixture, "src", "canary.ts"), + "export const canary = true;\n", + ); + await writeFile( + join(fixture, "README.md"), + `# Flow exact-artifact canary\n\nArtifact: ${artifact.tarballSha256}\n\nRun every checklist item and export sanitized JSON session and transcript evidence.\n`, + ); + const base: Omit = { + schemaVersion: 1 as const, + releaseTag: `v${artifact.packageVersion}`, + artifact, + artifactSha256: artifactIdentitySha256(artifact), + checklistVersion: CANARY_CHECKLIST_VERSION, + checklistSha256: CANARY_CHECKLIST_SHA256, + preparedAt: (input.preparedAt ?? new Date()).toISOString(), + artifactFile: "artifact.tgz" as const, + pluginEntrySha256: sha256(pluginEntry), + }; + const prepared: PreparedCanary = { + ...base, + sha256: preparedCanarySha256(base), + }; + PreparedCanarySchema.parse(prepared); + await writeImmutable( + join(input.outputDirectory, "prepared.json"), + Buffer.from(canonicalJson(prepared)), + ); + return prepared; +} + +function redactEvidence(value: unknown, projectPath: string): unknown { + const normalized = normalizeRecorded(value, projectPath); + return mapStrings(normalized, (text) => + scrubSecrets(text).replace( + /\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/g, + "", + ), + ); +} + +async function writeEvidence(input: { + readonly repositoryRoot: string; + readonly version: string; + readonly kind: "session" | "transcript"; + readonly value: unknown | null; + readonly projectPath: string; +}): Promise | null> { + if (input.value === null) return null; + const bytes = Buffer.from( + canonicalJson(redactEvidence(input.value, input.projectPath)), + ); + const artifact = `artifacts/${input.version}-${input.kind}.json`; + await writeImmutable( + join(input.repositoryRoot, "evals", "canary", artifact), + bytes, + ); + return { path: artifact, sha256: sha256(bytes), bytes: bytes.byteLength }; +} + +export async function recordCanary(input: { + readonly repositoryRoot: string; + readonly prepared: PreparedCanary; + readonly status: "passed" | "failed" | "incomplete"; + readonly operator: string; + readonly hostConfig: unknown; + readonly actors: readonly ActorIdentity[]; + readonly checks: z.infer; + readonly projectPath: string; + readonly session: unknown | null; + readonly transcript: unknown | null; + readonly recordedAt?: Date; +}): Promise<{ readonly path: string; readonly record: CanaryRecord }> { + const prepared = parsePreparedCanary(input.prepared); + const recordedAt = input.recordedAt ?? new Date(); + const parsedChecks = ChecksSchema.parse(input.checks); + const parsedActors = z.array(ActorIdentitySchema).parse(input.actors); + const checkValues = Object.values(parsedChecks); + if (input.status === "passed" && !checkValues.every(Boolean)) { + throw new Error("Passed canaries require every check."); + } + if (input.status === "failed" && !checkValues.some((value) => !value)) { + throw new Error("Failed canaries require a failed check."); + } + if ( + input.status === "passed" && + (parsedActors.length === 0 || + input.session === null || + input.transcript === null) + ) { + throw new Error( + "Passed canaries require actors, session, and transcript evidence.", + ); + } + const session = await writeEvidence({ + repositoryRoot: input.repositoryRoot, + version: prepared.artifact.packageVersion, + kind: "session", + value: input.session, + projectPath: input.projectPath, + }); + const transcript = await writeEvidence({ + repositoryRoot: input.repositoryRoot, + version: prepared.artifact.packageVersion, + kind: "transcript", + value: input.transcript, + projectPath: input.projectPath, + }); + const base: Omit = { + schemaVersion: 1 as const, + status: input.status, + artifact: prepared.artifact, + artifactSha256: prepared.artifactSha256, + releaseTag: prepared.releaseTag, + operator: input.operator, + recordedAt: recordedAt.toISOString(), + expiresAt: new Date(recordedAt.getTime() + CANARY_MAX_AGE_MS).toISOString(), + checklistVersion: CANARY_CHECKLIST_VERSION, + checklistSha256: CANARY_CHECKLIST_SHA256, + checks: parsedChecks, + hostConfigSha256: canonicalSha256( + "flow-canary-host-config-v1", + input.hostConfig, + ), + actors: parsedActors.map((actor) => ({ + ...actor, + sessionIds: actor.sessionIds.map(() => ""), + })), + artifacts: { session, transcript }, + }; + const record: CanaryRecord = { + ...base, + recordSha256: canaryRecordSha256(base), + }; + const parsed = parseCanaryRecord(record); + if (!parsed.ok) throw new Error(parsed.issues.join("; ")); + const path = join( + input.repositoryRoot, + "evals", + "canary", + `${prepared.artifact.packageVersion}.json`, + ); + await writeImmutable(path, Buffer.from(canonicalJson(record))); + return { path, record }; +} + +function insideCanaryDirectory(directory: string, path: string): string | null { + if (isAbsolute(path)) return null; + const root = resolve(directory); + const target = resolve(join(root, path)); + const within = relative(root, target); + return within && !within.startsWith("..") && !isAbsolute(within) + ? target + : null; +} + +async function evidenceIssue( + directory: string, + ref: z.infer | null, +): Promise { + if (!ref) return "Canary evidence artifact is missing."; + const target = insideCanaryDirectory(directory, ref.path); + if (!target) return "Canary evidence path escapes its directory."; + try { + const bytes = await readFile(target); + const info = await stat(target); + return bytes.byteLength === ref.bytes && + sha256(bytes) === ref.sha256 && + info.isFile() + ? null + : "Canary evidence digest or size does not match."; + } catch { + return "Canary evidence artifact is unreadable."; + } +} + +export async function canaryRecordIssue(input: { + readonly version: string; + readonly record: unknown; + readonly expectedArtifact: ArtifactIdentity; + readonly directory: string; + readonly now?: Date; +}): Promise { + const parsed = parseCanaryRecord(input.record); + if (!parsed.ok) return parsed.issues[0] ?? "Canary record is invalid."; + const record = parsed.value; + if (record.artifact.packageVersion !== input.version) + return "Canary artifact version does not match the release."; + if (canonicalJson(record.artifact) !== canonicalJson(input.expectedArtifact)) + return "Canary artifact does not match the rebuilt artifact."; + if (record.status !== "passed") return `Canary status is ${record.status}.`; + const now = (input.now ?? new Date()).getTime(); + if (Date.parse(record.recordedAt) > now) return "Canary is future-dated."; + if (Date.parse(record.expiresAt) <= now) return "Canary is expired."; + const sessionIssue = await evidenceIssue( + input.directory, + record.artifacts.session, + ); + if (sessionIssue) return sessionIssue; + return evidenceIssue(input.directory, record.artifacts.transcript); +} + +export async function verifyCanary(input: { + readonly repositoryRoot: string; + readonly artifactPath: string; + readonly mode: "dry-run" | "strict"; + readonly now?: Date; +}): Promise<{ + readonly verdict: "VERIFIED" | "NOT VERIFIED" | "INCONCLUSIVE"; + readonly issue: string | null; + readonly record: CanaryRecord | null; +}> { + const artifact = await inspectArtifact({ + repositoryRoot: input.repositoryRoot, + tarballPath: input.artifactPath, + }); + const directory = join(input.repositoryRoot, "evals", "canary"); + let raw: unknown; + try { + raw = JSON.parse( + await readFile( + join(directory, `${artifact.packageVersion}.json`), + "utf8", + ), + ); + } catch { + const issue = "Canary record is missing."; + return { verdict: "INCONCLUSIVE", issue, record: null }; + } + const issue = await canaryRecordIssue({ + version: artifact.packageVersion, + record: raw, + expectedArtifact: artifact, + directory, + ...(input.now ? { now: input.now } : {}), + }); + const parsed = parseCanaryRecord(raw); + const record = parsed.ok ? parsed.value : null; + if (!issue) return { verdict: "VERIFIED", issue: null, record }; + const inconclusive = /missing|incomplete/i.test(issue); + const verdict = inconclusive ? "INCONCLUSIVE" : "NOT VERIFIED"; + if (input.mode === "strict") return { verdict, issue, record }; + return { verdict, issue, record }; +} + +function option(args: readonly string[], name: string): string | undefined { + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +} + +function required(args: readonly string[], name: string): string { + const value = option(args, name); + if (!value || value.startsWith("--")) + throw new Error(`${name} requires a value.`); + return value; +} + +async function json(path: string): Promise { + return JSON.parse(await readFile(path, "utf8")); +} + +async function main(args: readonly string[]): Promise { + const command = args[0]; + const repositoryRoot = join(import.meta.dir, ".."); + if (command === "--help" || command === "-h") { + process.stdout.write( + "Usage: eval:canary [options]\n", + ); + return; + } + if (command === "prepare") { + const prepared = await prepareCanary({ + repositoryRoot, + artifactPath: required(args, "--artifact"), + outputDirectory: required(args, "--out"), + }); + process.stdout.write( + `INCONCLUSIVE: manual canary pending\n${canonicalJson(prepared)}\n`, + ); + return; + } + if (command === "record") { + const status = required(args, "--status"); + if (status !== "passed" && status !== "failed" && status !== "incomplete") + throw new Error("--status must be passed, failed, or incomplete."); + const result = await recordCanary({ + repositoryRoot, + prepared: PreparedCanarySchema.parse( + await json(required(args, "--prepared")), + ), + status, + operator: required(args, "--operator"), + hostConfig: await json(required(args, "--host-config")), + actors: z + .array(ActorIdentitySchema) + .parse(await json(required(args, "--actors"))), + checks: ChecksSchema.parse(await json(required(args, "--checks"))), + projectPath: required(args, "--project-path"), + session: option(args, "--session") + ? await json(required(args, "--session")) + : null, + transcript: option(args, "--transcript") + ? await json(required(args, "--transcript")) + : null, + }); + process.stdout.write(`${result.record.status}: ${result.path}\n`); + return; + } + if (command === "verify") { + const mode = option(args, "--mode") ?? "strict"; + if (mode !== "dry-run" && mode !== "strict") + throw new Error("Invalid --mode."); + const result = await verifyCanary({ + repositoryRoot, + artifactPath: required(args, "--artifact"), + mode, + }); + process.stdout.write( + `${result.verdict}: ${result.issue ?? "exact canary verified"}\n`, + ); + if (mode === "strict" && result.verdict !== "VERIFIED") + process.exitCode = 1; + return; + } + throw new Error("Usage: eval:canary [options]"); +} + +if (import.meta.main) { + main(process.argv.slice(2)).catch((error) => { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + }); +} diff --git a/tests/eval-canary.test.ts b/tests/eval-canary.test.ts new file mode 100644 index 0000000..9aebcfb --- /dev/null +++ b/tests/eval-canary.test.ts @@ -0,0 +1,358 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + artifactIdentitySha256, + CANARY_CHECKLIST_SHA256, + CANARY_CHECKLIST_VERSION, + CANARY_MAX_AGE_MS, + type CanaryRecord, + canaryRecordIssue, + canaryRecordSha256, + type PreparedCanary, + parseCanaryRecord, + prepareCanary, + preparedCanarySha256, + recordCanary, +} from "../scripts/eval-canary.js"; + +const temporary: string[] = []; +afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; +const artifact = { + packageVersion: "1.2.3", + sourceCommit: "commit", + sourceTreeSha256: digest("a"), + tarballSha256: digest("b"), + unpackedManifestSha256: digest("c"), +}; +const checks = { + "installs-packed-artifact": true, + "loads-flow-tools": true, + "saves-plan": true, + "captures-validation": true, + "dispatches-reviewer": true, + "closes-with-delivery": true, +}; +const actor = { + role: "manager" as const, + requestedModel: { + routeProvider: "provider", + gateway: null, + family: "family", + model: "model", + revision: null, + }, + actualModel: { + kind: "unobserved" as const, + reason: "full identity unavailable", + }, + sessionIds: ["ses_secret"], +}; + +function prepared(): PreparedCanary { + const base: Omit = { + schemaVersion: 1 as const, + releaseTag: "v1.2.3", + artifact, + artifactSha256: artifactIdentitySha256(artifact), + checklistVersion: CANARY_CHECKLIST_VERSION, + checklistSha256: CANARY_CHECKLIST_SHA256, + preparedAt: "2026-08-25T00:00:00.000Z", + artifactFile: "artifact.tgz" as const, + pluginEntrySha256: digest("d"), + }; + return { ...base, sha256: preparedCanarySha256(base) }; +} + +function record( + input: { + readonly status?: "passed" | "failed" | "incomplete"; + readonly checks?: typeof checks; + readonly recordedAt?: string; + readonly expiresAt?: string; + readonly artifactValue?: typeof artifact; + } = {}, +): CanaryRecord { + const recordedAt = input.recordedAt ?? "2026-08-25T00:00:00.000Z"; + const artifactValue = input.artifactValue ?? artifact; + const base: Omit = { + schemaVersion: 1 as const, + status: input.status ?? ("passed" as const), + artifact: artifactValue, + artifactSha256: artifactIdentitySha256(artifactValue), + releaseTag: `v${artifactValue.packageVersion}`, + operator: "maintainer", + recordedAt, + expiresAt: + input.expiresAt ?? + new Date(Date.parse(recordedAt) + CANARY_MAX_AGE_MS).toISOString(), + checklistVersion: CANARY_CHECKLIST_VERSION, + checklistSha256: CANARY_CHECKLIST_SHA256, + checks: input.checks ?? checks, + hostConfigSha256: digest("e"), + actors: [{ ...actor, sessionIds: [""] }], + artifacts: { + session: { + path: "artifacts/1.2.3-session.json", + sha256: digest("f"), + bytes: 1, + }, + transcript: { + path: "artifacts/1.2.3-transcript.json", + sha256: digest("9"), + bytes: 1, + }, + }, + }; + return { ...base, recordSha256: canaryRecordSha256(base) }; +} + +describe("canary record boundary", () => { + test("accepts strict passed, failed, and incomplete records", () => { + expect(parseCanaryRecord(record()).ok).toBe(true); + expect( + parseCanaryRecord( + record({ + status: "failed", + checks: { ...checks, "closes-with-delivery": false }, + }), + ).ok, + ).toBe(true); + expect(parseCanaryRecord(record({ status: "incomplete" })).ok).toBe(true); + }); + + test("rejects wrong status/check combinations and unknown checklist keys", () => { + expect( + parseCanaryRecord( + record({ checks: { ...checks, "closes-with-delivery": false } }), + ).ok, + ).toBe(false); + expect(parseCanaryRecord(record({ status: "failed" })).ok).toBe(false); + expect( + parseCanaryRecord({ ...record(), checks: { ...checks, extra: true } }).ok, + ).toBe(false); + }); + + test("rejects tag, artifact, checklist, record hash, and expiry drift", () => { + for (const changed of [ + { ...record(), releaseTag: "v9.9.9" }, + { ...record(), artifactSha256: digest("0") }, + { ...record(), checklistSha256: digest("0") }, + { ...record(), recordSha256: digest("0") }, + record({ expiresAt: "2026-08-29T00:00:00.000Z" }), + ]) { + expect(parseCanaryRecord(changed).ok).toBe(false); + } + }); +}); + +async function evidenceDirectory(value: CanaryRecord): Promise { + const directory = await mkdtemp(join(tmpdir(), "flow-canary-evidence-")); + temporary.push(directory); + await mkdir(join(directory, "artifacts"), { recursive: true }); + await writeFile(join(directory, value.artifacts.session?.path ?? ""), "x"); + await writeFile(join(directory, value.artifacts.transcript?.path ?? ""), "y"); + return directory; +} + +describe("canary release verification", () => { + test("rejects stale, future, failed, incomplete, and artifact-mismatched records", async () => { + const valid = record(); + const directory = await evidenceDirectory(valid); + for (const [value, now, pattern] of [ + [valid, new Date("2026-08-29T00:00:00.000Z"), /expired/], + [valid, new Date("2026-08-24T00:00:00.000Z"), /future/], + [ + record({ status: "incomplete" }), + new Date("2026-08-25T01:00:00.000Z"), + /incomplete/, + ], + [ + record({ + status: "failed", + checks: { ...checks, "saves-plan": false }, + }), + new Date("2026-08-25T01:00:00.000Z"), + /failed/, + ], + ] as const) { + expect( + await canaryRecordIssue({ + version: "1.2.3", + record: value, + expectedArtifact: artifact, + directory, + now, + }), + ).toMatch(pattern); + } + expect( + await canaryRecordIssue({ + version: "1.2.3", + record: valid, + expectedArtifact: { ...artifact, tarballSha256: digest("0") }, + directory, + now: new Date("2026-08-25T01:00:00.000Z"), + }), + ).toMatch(/does not match/); + }); + + test("checks sanitized evidence bytes, sizes, and digests", async () => { + const valid = record(); + const directory = await evidenceDirectory(valid); + expect( + await canaryRecordIssue({ + version: "1.2.3", + record: valid, + expectedArtifact: artifact, + directory, + now: new Date("2026-08-25T01:00:00.000Z"), + }), + ).toMatch(/digest or size/); + }); +}); + +describe("canary recording", () => { + test("redacts evidence and allows only byte-identical replay", async () => { + const root = await mkdtemp(join(tmpdir(), "flow-canary-record-")); + temporary.push(root); + const input = { + repositoryRoot: root, + prepared: prepared(), + status: "passed" as const, + operator: "maintainer", + hostConfig: { opencode: "1.18.6" }, + actors: [actor], + checks, + projectPath: "/secret/project", + session: { + id: "ses_secret", + path: "/secret/project", + apiKey: "sk-proj-1234567890123456", + }, + transcript: { + session: "session:1234-abcd", + text: "Bearer abcdefghijklmnop", + }, + recordedAt: new Date("2026-08-25T00:00:00.000Z"), + }; + const first = await recordCanary(input); + const second = await recordCanary(input); + expect(second.record).toEqual(first.record); + expect( + await canaryRecordIssue({ + version: "1.2.3", + record: first.record, + expectedArtifact: artifact, + directory: join(root, "evals", "canary"), + now: new Date("2026-08-25T01:00:00.000Z"), + }), + ).toBeNull(); + const stored = [ + await readFile( + join(root, "evals/canary/artifacts/1.2.3-session.json"), + "utf8", + ), + await readFile( + join(root, "evals/canary/artifacts/1.2.3-transcript.json"), + "utf8", + ), + await readFile(first.path, "utf8"), + ].join("\n"); + expect(stored).not.toContain("/secret/project"); + expect(stored).not.toContain("ses_secret"); + expect(stored).not.toContain("1234-abcd"); + expect(stored).not.toContain("sk-proj-"); + expect(stored).not.toContain("Bearer abcdef"); + await expect(recordCanary({ ...input, operator: "other" })).rejects.toThrow( + "conflicts", + ); + }); + + test("passed recording requires actors and both artifacts", async () => { + const root = await mkdtemp(join(tmpdir(), "flow-canary-record-")); + temporary.push(root); + await expect( + recordCanary({ + repositoryRoot: root, + prepared: prepared(), + status: "passed", + operator: "maintainer", + hostConfig: {}, + actors: [], + checks, + projectPath: root, + session: null, + transcript: null, + }), + ).rejects.toThrow(/require actors/); + }); +}); + +async function run(command: readonly string[], cwd: string): Promise { + const child = Bun.spawn([...command], { + cwd, + stdout: "ignore", + stderr: "pipe", + }); + if ((await child.exited) !== 0) + throw new Error(await new Response(child.stderr).text()); +} + +describe("canary preparation", () => { + test("builds an exact local-plugin fixture from the tarball", async () => { + const root = await mkdtemp(join(tmpdir(), "flow-canary-package-")); + const output = await mkdtemp(join(tmpdir(), "flow-canary-output-")); + temporary.push(root, output); + await mkdir(join(root, "dist"), { recursive: true }); + await writeFile( + join(root, "dist/index.js"), + "export default async () => ({});\n", + ); + await writeFile( + join(root, "package.json"), + JSON.stringify({ + name: "opencode-plugin-flow", + version: "1.2.3", + files: ["dist/index.js"], + dependencies: { zod: "4.4.3" }, + devDependencies: { "@opencode-ai/plugin": "1.18.6" }, + }), + ); + for (const args of [ + ["git", "init", "--initial-branch=main"], + ["git", "config", "user.email", "canary@example.com"], + ["git", "config", "user.name", "Canary"], + ["git", "add", "-A"], + ["git", "commit", "-m", "fixture"], + ]) + await run(args, root); + await run(["bun", "pm", "pack", "--destination", output], root); + const artifactPath = join(output, "opencode-plugin-flow-1.2.3.tgz"); + const prepared = await prepareCanary({ + repositoryRoot: root, + artifactPath, + outputDirectory: join(output, "prepared"), + preparedAt: new Date("2026-08-25T00:00:00.000Z"), + }); + expect(prepared.releaseTag).toBe("v1.2.3"); + expect( + await readFile( + join(output, "prepared/fixture/.opencode/plugins/flow.js"), + "utf8", + ), + ).toContain("export default"); + expect(await readFile(join(output, "prepared/artifact.tgz"))).toEqual( + await readFile(artifactPath), + ); + }); +}); From 8208c53b47282b857b90cd4dc99b5412de167686 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:32:06 +0200 Subject: [PATCH 3/4] feat(release): require canary-bound decisions --- .github/workflows/release.yml | 72 +++++++-- scripts/qualify-release.ts | 61 +++++++- scripts/release-metadata.ts | 221 +++++++++++++++++++++++++-- tests/documentation-contract.test.ts | 15 +- tests/release-metadata.test.ts | 197 ++++++++++++++++++++++-- tests/release-qualification.test.ts | 7 + 6 files changed, 526 insertions(+), 47 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 669406c..0d5363f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,12 +2,13 @@ name: Release on: push: + branches: + - main tags: - 'v*' permissions: - contents: write - id-token: write + contents: read jobs: decide: @@ -28,16 +29,58 @@ jobs: tag="v${version}" echo "tag=${tag}" >> "$GITHUB_OUTPUT" - if [[ "${GITHUB_REF_NAME}" != "${tag}" ]]; then - echo "::error::Release tag/version mismatch: tag=${GITHUB_REF_NAME}, package.json=${version}." - exit 1 + if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${tag}" ]]; then + echo "::error::Release tag/version mismatch: tag=${GITHUB_REF_NAME}, package.json=${version}." + exit 1 + fi + echo "publish=true" >> "$GITHUB_OUTPUT" + else + echo "publish=false" >> "$GITHUB_OUTPUT" fi - echo "publish=true" >> "$GITHUB_OUTPUT" - release: + verify-main-and-tag: needs: decide + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + package-manager-cache: false + + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Run deterministic checks + run: bun run check + + - name: Rebuild release candidate + run: bun pm pack --destination . + + - name: Report release evidence readiness without publishing + shell: bash + run: | + set -euo pipefail + tarball="$(ls opencode-plugin-flow-*.tgz)" + bun run release:metadata -- --artifact "$tarball" + bun run eval:canary -- verify --artifact "$tarball" --mode dry-run + + release: + needs: [decide, verify-main-and-tag] if: needs.decide.outputs.publish == 'true' runs-on: ubuntu-latest + permissions: + contents: write + id-token: write steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -78,20 +121,19 @@ jobs: set -euo pipefail bun pm pack --destination . - - name: Verify exact VERIFIED V2 artifact decision + - name: Verify exact VERIFIED V2 artifact decision and fresh canary shell: bash run: | set -euo pipefail + version="$(node -p "require('./package.json').version")" tarball="$(ls opencode-plugin-flow-*.tgz)" - bun run release:metadata -- --tag "${{ needs.decide.outputs.tag }}" --notes-file release-notes.md --artifact "$tarball" + bun run release:metadata -- \ + --tag "${{ needs.decide.outputs.tag }}" \ + --notes-file release-notes.md \ + --artifact "$tarball" \ + --canary "evals/canary/${version}.json" shasum -a 256 "$tarball" > "${tarball}.sha256" - # Temporary Phase 5 stop: the exact manual OpenCode canary is owned by Phase 9. - - name: Require Phase 9 canary - run: | - echo "::error::Release blocked: canary-not-enabled until Phase 9 release alignment is implemented." - exit 1 - - name: Publish to npm shell: bash run: | diff --git a/scripts/qualify-release.ts b/scripts/qualify-release.ts index 68d0a5a..bd06bec 100644 --- a/scripts/qualify-release.ts +++ b/scripts/qualify-release.ts @@ -17,7 +17,7 @@ // and the reasoning. import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { deriveReleaseDecision, type ExpectedActorProvenance, @@ -35,7 +35,11 @@ import { parseReport, type ValidatedReport, } from "../evals/report.js"; -import { isMajorRelease } from "./release-metadata.js"; +import { + type CanaryRecord, + canaryRecordIssue as verifyCanaryRecord, +} from "./eval-canary.js"; +import { canaryRecordIssue, isMajorRelease } from "./release-metadata.js"; export type DecisionRecord = { readonly schemaVersion: 1; @@ -50,6 +54,7 @@ export type DecisionRecord = { readonly analyzerSha256: string; readonly expectedProvenanceSha256: string; readonly decisionInputSha256: string; + readonly canarySha256: string | null; readonly artifact: ArtifactIdentity; readonly reasons: readonly string[]; }; @@ -517,11 +522,13 @@ export function qualifyV2(input: { readonly reportInput: unknown; readonly catalogInput: unknown; readonly artifact: ArtifactIdentity; + readonly canary?: CanaryRecord | null; }): { readonly report: ValidatedReport; readonly catalog: ValidatedCaseCatalog; readonly decision: ReleaseDecision; readonly expected: ReleaseExpectedProvenance; + readonly canary: CanaryRecord | null; } { const catalog = parseCaseCatalog(input.catalogInput); if (!catalog.ok) { @@ -540,6 +547,14 @@ export function qualifyV2(input: { ); } const expected = expectedProvenanceFor(parsed.value, input.artifact); + if (input.canary) { + const canaryIssue = canaryRecordIssue( + input.artifact.packageVersion, + input.canary, + input.artifact, + ); + if (canaryIssue) throw new Error(canaryIssue); + } return { report: parsed.value, catalog: catalog.value, @@ -549,6 +564,7 @@ export function qualifyV2(input: { catalog: catalog.value, expected, }), + canary: input.canary ?? null, }; } @@ -557,6 +573,7 @@ export function decisionRecordFor(input: { readonly catalog: ValidatedCaseCatalog; readonly expected: ReleaseExpectedProvenance; readonly decision: ReleaseDecision; + readonly canarySha256?: string | null; }): DecisionRecord { const reportSha256 = canonicalSha256("flow-decision-report-v1", input.report); const artifactSha256 = canonicalSha256( @@ -599,6 +616,7 @@ export function decisionRecordFor(input: { actorSha256, analyzerSha256, expectedProvenanceSha256, + canarySha256: input.canarySha256 ?? null, }); return { schemaVersion: 1, @@ -612,6 +630,7 @@ export function decisionRecordFor(input: { actorSha256, analyzerSha256, expectedProvenanceSha256, + canarySha256: input.canarySha256 ?? null, decisionInputSha256, artifact: input.expected.artifact, reasons: input.decision.reasons.map((reason) => reason.message), @@ -623,8 +642,19 @@ export async function writeDecisionRecord(input: { readonly directory: string; }): Promise { await mkdir(input.directory, { recursive: true }); - const path = join(input.directory, `${input.record.reportId}.json`); - await writeFile(path, canonicalJson(input.record), "utf8"); + const suffix = input.record.canarySha256 + ? `-canary-${input.record.canarySha256.slice("sha256:".length, "sha256:".length + 12)}` + : ""; + const path = join(input.directory, `${input.record.reportId}${suffix}.json`); + const bytes = canonicalJson(input.record); + try { + await writeFile(path, bytes, { encoding: "utf8", flag: "wx" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + if ((await readFile(path, "utf8")) !== bytes) { + throw new Error(`Immutable decision record conflicts: ${path}`); + } + } return path; } @@ -635,7 +665,7 @@ function requiredOption( const value = options[name]; if (!value) { throw new Error( - "Usage: bun run qualify -- --report --catalog --artifact [--decisions-dir ]", + "Usage: bun run qualify -- --report --catalog --artifact [--canary ] [--decisions-dir ]", ); } return value; @@ -650,10 +680,11 @@ async function main(): Promise { option !== "--report" && option !== "--catalog" && option !== "--artifact" && + option !== "--canary" && option !== "--decisions-dir" ) { throw new Error( - "Usage: bun run qualify -- --report --catalog --artifact [--decisions-dir ]", + "Usage: bun run qualify -- --report --catalog --artifact [--canary ] [--decisions-dir ]", ); } const value = args[index + 1]; @@ -669,12 +700,28 @@ async function main(): Promise { repositoryRoot: join(import.meta.dir, ".."), tarballPath: artifactPath, }); + const canary = options["--canary"] + ? (JSON.parse(await readFile(options["--canary"], "utf8")) as CanaryRecord) + : null; + if (canary && options["--canary"]) { + const canaryIssue = await verifyCanaryRecord({ + version: artifact.packageVersion, + record: canary, + expectedArtifact: artifact, + directory: dirname(options["--canary"]), + }); + if (canaryIssue) throw new Error(canaryIssue); + } const result = qualifyV2({ reportInput: JSON.parse(await readFile(reportPath, "utf8")), catalogInput: JSON.parse(await readFile(catalogPath, "utf8")), artifact, + canary, + }); + const record = decisionRecordFor({ + ...result, + canarySha256: result.canary?.recordSha256 ?? null, }); - const record = decisionRecordFor(result); const path = await writeDecisionRecord({ record, directory: diff --git a/scripts/release-metadata.ts b/scripts/release-metadata.ts index abee8e7..c8666e9 100644 --- a/scripts/release-metadata.ts +++ b/scripts/release-metadata.ts @@ -1,8 +1,84 @@ import { readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; -import { canonicalJson } from "../evals/canonical-json.js"; +import { dirname, join } from "node:path"; +import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; import { inspectArtifact } from "../evals/provenance.js"; import type { ArtifactIdentity } from "../evals/report.js"; +import { + artifactIdentitySha256, + CANARY_CHECKLIST_SHA256, + CANARY_CHECKLIST_VERSION, + type CanaryRecord, + canaryRecordSha256, + parseCanaryRecord, + canaryRecordIssue as verifyCanaryRecord, +} from "./eval-canary.js"; + +export function canaryRecordIssue( + version: string, + record: unknown, + expectedArtifact: ArtifactIdentity, + expectedTag = `v${version}`, + now = new Date(), +): string | null { + if (!record || typeof record !== "object" || Array.isArray(record)) + return `no canary record exists for ${version}`; + const entry = record as Partial; + if (entry.schemaVersion !== 1 || entry.status !== "passed") + return `the canary for ${version} is not a passed v1 record`; + if (entry.releaseTag !== expectedTag) + return `the canary tag ${String(entry.releaseTag)} does not match ${expectedTag}`; + if (canonicalJson(entry.artifact) !== canonicalJson(expectedArtifact)) + return `the canary artifact does not match the rebuilt artifact for ${version}`; + const parsed = parseCanaryRecord(record); + if (!parsed.ok) return `the canary record for ${version} is invalid`; + const { recordSha256: _recordSha256, ...recordWithoutHash } = parsed.value; + if (parsed.value.recordSha256 !== canaryRecordSha256(recordWithoutHash)) + return `the canary record for ${version} has an invalid digest`; + if ( + entry.artifactSha256 !== artifactIdentitySha256(expectedArtifact) || + entry.checklistVersion !== CANARY_CHECKLIST_VERSION || + entry.checklistSha256 !== CANARY_CHECKLIST_SHA256 || + Object.values(entry.checks ?? {}).length === 0 || + Object.values(entry.checks ?? {}).some((passed) => !passed) + ) + return `the canary checklist for ${version} is incomplete or failed`; + if ( + typeof entry.operator !== "string" || + entry.operator.trim() === "" || + typeof entry.hostConfigSha256 !== "string" || + !/^sha256:[a-f0-9]{64}$/.test(entry.hostConfigSha256) || + !Array.isArray(entry.actors) || + entry.actors.length === 0 + ) + return `the canary for ${version} is missing operator, host, or actor evidence`; + const recorded = Date.parse(entry.recordedAt ?? ""); + const expires = Date.parse(entry.expiresAt ?? ""); + if ( + !Number.isFinite(recorded) || + !Number.isFinite(expires) || + recorded >= expires + ) + return `the canary timestamps for ${version} are invalid`; + if (recorded > now.getTime()) + return `the canary for ${version} is future-dated`; + if (expires <= now.getTime()) return `the canary for ${version} is stale`; + for (const artifact of [ + entry.artifacts?.session, + entry.artifacts?.transcript, + ]) { + if ( + !artifact || + typeof artifact.path !== "string" || + artifact.path.startsWith("/") || + artifact.path.split("/").includes("..") || + !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256) || + !Number.isSafeInteger(artifact.bytes) || + artifact.bytes < 0 + ) + return `the canary redacted artifacts for ${version} are invalid`; + } + return null; +} export type ReleaseMetadataInput = { packageVersion: string; @@ -73,6 +149,7 @@ export function qualificationRecordIssue( version: string, record: unknown, expectedArtifact?: ArtifactIdentity, + expectedCanarySha256?: string, ): string | null { if (!record || typeof record !== "object" || Array.isArray(record)) { return `no qualification record exists for ${version}`; @@ -90,6 +167,7 @@ export function qualificationRecordIssue( actorSha256?: unknown; analyzerSha256?: unknown; expectedProvenanceSha256?: unknown; + canarySha256?: unknown; decisionInputSha256?: unknown; }; if (entry.schemaVersion !== 1 || typeof entry.reportId !== "string") { @@ -126,6 +204,23 @@ export function qualificationRecordIssue( ) { return `the qualification artifact does not match the rebuilt artifact for ${version}`; } + if (expectedCanarySha256 !== undefined) { + if (entry.canarySha256 !== expectedCanarySha256) + return `the qualification record for ${version} is not bound to the exact canary`; + const expectedDecisionInput = canonicalSha256("flow-decision-input-v1", { + reportSha256: entry.reportSha256, + artifactSha256: entry.artifactSha256, + evaluatorSha256: entry.evaluatorSha256, + catalogSha256: entry.catalogSha256, + policySha256: entry.policySha256, + actorSha256: entry.actorSha256, + analyzerSha256: entry.analyzerSha256, + expectedProvenanceSha256: entry.expectedProvenanceSha256, + canarySha256: entry.canarySha256, + }); + if (entry.decisionInputSha256 !== expectedDecisionInput) + return `the qualification decision input does not bind the exact canary for ${version}`; + } return null; } @@ -134,7 +229,6 @@ export async function assertQualificationRecord( directory = join("evals", "decisions"), expectedArtifact?: ArtifactIdentity, ): Promise { - if (!isMajorRelease(version)) return; let records: unknown[] = []; try { const { readdir } = await import("node:fs/promises"); @@ -155,9 +249,100 @@ export async function assertQualificationRecord( ) ) { throw new Error( - `Major release ${version} cannot proceed: no exact VERIFIED v2 decision record exists. Run \`bun run qualify -- --report --catalog --artifact \` and commit the decision.`, + `Release ${version} cannot proceed: no exact VERIFIED v2 decision record exists. Run \`bun run qualify -- --report --catalog --artifact \` and commit the decision.`, + ); + } +} + +export async function assertStrictReleaseEvidence(input: { + readonly version: string; + readonly decisionsDirectory?: string; + readonly canaryPath: string; + readonly expectedArtifact: ArtifactIdentity; + readonly tag?: string; + readonly now?: Date; +}): Promise { + const tag = input.tag ?? `v${input.version}`; + const canary = JSON.parse( + await readFile(input.canaryPath, "utf8"), + ) as unknown; + const canaryIssue = canaryRecordIssue( + input.version, + canary, + input.expectedArtifact, + tag, + input.now, + ); + if (canaryIssue) throw new Error(canaryIssue); + const evidenceIssue = await verifyCanaryRecord({ + version: input.version, + record: canary, + expectedArtifact: input.expectedArtifact, + directory: dirname(input.canaryPath), + ...(input.now ? { now: input.now } : {}), + }); + if (evidenceIssue) throw new Error(evidenceIssue); + const canaryHash = (canary as CanaryRecord).recordSha256; + let records: unknown[] = []; + try { + const { readdir } = await import("node:fs/promises"); + records = await Promise.all( + (await readdir(input.decisionsDirectory ?? join("evals", "decisions"))) + .filter((name) => name.endsWith(".json")) + .map(async (name) => + JSON.parse( + await readFile( + join( + input.decisionsDirectory ?? join("evals", "decisions"), + name, + ), + "utf8", + ), + ), + ), ); + } catch { + records = []; } + const match = records.find((record) => { + const entry = record as { + readonly reportSha256?: unknown; + readonly artifactSha256?: unknown; + readonly evaluatorSha256?: unknown; + readonly catalogSha256?: unknown; + readonly policySha256?: unknown; + readonly actorSha256?: unknown; + readonly analyzerSha256?: unknown; + readonly expectedProvenanceSha256?: unknown; + readonly canarySha256?: unknown; + readonly decisionInputSha256?: unknown; + }; + const decisionInputSha256 = canonicalSha256("flow-decision-input-v1", { + reportSha256: entry.reportSha256, + artifactSha256: entry.artifactSha256, + evaluatorSha256: entry.evaluatorSha256, + catalogSha256: entry.catalogSha256, + policySha256: entry.policySha256, + actorSha256: entry.actorSha256, + analyzerSha256: entry.analyzerSha256, + expectedProvenanceSha256: entry.expectedProvenanceSha256, + canarySha256: canaryHash, + }); + return ( + entry.canarySha256 === canaryHash && + entry.decisionInputSha256 === decisionInputSha256 && + qualificationRecordIssue( + input.version, + record, + input.expectedArtifact, + canaryHash, + ) === null + ); + }); + if (!match) + throw new Error( + `Release ${input.version} cannot proceed: no exact VERIFIED canary-bound v2 decision record exists.`, + ); } function optionValue( @@ -176,6 +361,7 @@ async function main(args: readonly string[]): Promise { let tag: string | undefined; let notesFile: string | undefined; let artifactPath: string | undefined; + let canaryPath: string | undefined; for (let index = 0; index < args.length; index += 1) { const argument = args[index]; switch (argument) { @@ -191,6 +377,10 @@ async function main(args: readonly string[]): Promise { artifactPath = optionValue(args, index, argument); index += 1; break; + case "--canary": + canaryPath = optionValue(args, index, argument); + index += 1; + break; default: throw new Error(`Unknown option: ${argument}`); } @@ -214,11 +404,24 @@ async function main(args: readonly string[]): Promise { tarballPath: artifactPath, }) : undefined; - await assertQualificationRecord( - packageMetadata.version, - undefined, - expectedArtifact, - ); + if (artifactPath && tag) { + if (!canaryPath) + throw new Error("Strict tag release metadata requires --canary."); + if (!expectedArtifact) + throw new Error("Strict tag release metadata requires an artifact."); + await assertStrictReleaseEvidence({ + version: packageMetadata.version, + tag, + canaryPath, + expectedArtifact, + }); + } else if (artifactPath) { + process.stdout.write( + `INCONCLUSIVE: rebuilt artifact ${packageMetadata.version} has no strict tag evidence.\n`, + ); + if (notesFile) await writeFile(notesFile, result.releaseNotes, "utf8"); + return; + } if (notesFile) await writeFile(notesFile, result.releaseNotes, "utf8"); process.stdout.write( `Release metadata matches ${packageMetadata.version}.\n`, diff --git a/tests/documentation-contract.test.ts b/tests/documentation-contract.test.ts index 008c557..34edb24 100644 --- a/tests/documentation-contract.test.ts +++ b/tests/documentation-contract.test.ts @@ -504,12 +504,17 @@ describe("Flow documentation contract", () => { expect(combined).toContain("tests/workspace-persistence.test.ts"); expect(combined).toContain("npm publish"); const release = await readFile(".github/workflows/release.yml", "utf8"); - expect(release).toMatch(/^ {2}push:\n {4}tags:/m); - expect(release).not.toContain("branches:"); + expect(release).toMatch(/^ {2}push:\n {4}branches:/m); + expect(release).toContain("tags:"); expect(release).toContain('tag="v${version}"'); expect(release).toContain('--target "${GITHUB_SHA}"'); - expect(release).toContain("Verify exact VERIFIED V2 artifact decision"); - expect(release).toContain("canary-not-enabled"); + expect(release).toContain( + "Verify exact VERIFIED V2 artifact decision and fresh canary", + ); + expect(release).toContain("bun run eval:canary -- verify"); + expect(release).toContain("--mode dry-run"); + expect(release).toContain("evals/canary/${version}.json"); + expect(release).not.toContain("canary-not-enabled"); // Model-driven evals need credentials and cost real money, so they run on a // schedule and never on a pull request. `evals.yml` is the one workflow allowed @@ -537,7 +542,7 @@ describe("Flow documentation contract", () => { await readFile(join(".github/workflows", gate), "utf8"), gate, ).not.toMatch( - /harness|lifecycle-soak|cross-version|replay-report|prompt:model-eval|bun run eval/i, + /harness|lifecycle-soak|cross-version|replay-report|prompt:model-eval|bun run eval(?:\s|$)/i, ); } }); diff --git a/tests/release-metadata.test.ts b/tests/release-metadata.test.ts index 53da24d..fdeb7d7 100644 --- a/tests/release-metadata.test.ts +++ b/tests/release-metadata.test.ts @@ -1,9 +1,20 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { canonicalSha256 } from "../evals/canonical-json.js"; +import type { CanaryRecord } from "../scripts/eval-canary.js"; +import { + artifactIdentitySha256, + CANARY_CHECKLIST_SHA256, + CANARY_CHECKLIST_VERSION, + canaryRecordSha256, +} from "../scripts/eval-canary.js"; import { assertQualificationRecord, + assertStrictReleaseEvidence, + canaryRecordIssue, isMajorRelease, qualificationRecordIssue, releaseNotesForVersion, @@ -28,6 +39,8 @@ async function recordDirectory(): Promise { const VERSION = "6.0.0"; const digest = (letter: string) => `sha256:${letter.repeat(64)}`; +const bytesDigest = (value: string) => + `sha256:${createHash("sha256").update(value).digest("hex")}`; const artifact = (packageVersion: string) => ({ packageVersion, sourceCommit: "commit", @@ -49,7 +62,92 @@ const decisionRecord = (packageVersion: string, verdict = "VERIFIED") => ({ analyzerSha256: digest("2"), expectedProvenanceSha256: digest("3"), decisionInputSha256: digest("4"), + canarySha256: null, }); +function canaryBoundDecision(packageVersion: string, canarySha256: string) { + const base = decisionRecord(packageVersion); + return { + ...base, + canarySha256, + decisionInputSha256: canonicalSha256("flow-decision-input-v1", { + reportSha256: base.reportSha256, + artifactSha256: base.artifactSha256, + evaluatorSha256: base.evaluatorSha256, + catalogSha256: base.catalogSha256, + policySha256: base.policySha256, + actorSha256: base.actorSha256, + analyzerSha256: base.analyzerSha256, + expectedProvenanceSha256: base.expectedProvenanceSha256, + canarySha256, + }), + }; +} +function canaryRecord( + packageVersion: string, + overrides: Record = {}, +) { + const base: Omit = { + schemaVersion: 1 as const, + releaseTag: `v${packageVersion}`, + status: "passed" as const, + artifact: artifact(packageVersion), + checklistVersion: CANARY_CHECKLIST_VERSION, + checklistSha256: CANARY_CHECKLIST_SHA256, + artifactSha256: artifactIdentitySha256(artifact(packageVersion)), + checks: { + "installs-packed-artifact": true, + "loads-flow-tools": true, + "saves-plan": true, + "captures-validation": true, + "dispatches-reviewer": true, + "closes-with-delivery": true, + }, + operator: "maintainer@example.com", + recordedAt: "2026-08-25T00:00:00.000Z", + expiresAt: "2026-08-28T00:00:00.000Z", + hostConfigSha256: digest("6"), + actors: [ + { + role: "manager" as const, + requestedModel: { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "test", + revision: null, + }, + actualModel: { + kind: "observed" as const, + value: { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "test", + revision: null, + }, + }, + sessionIds: [""], + }, + ], + artifacts: { + session: { + path: "artifacts/session.json", + sha256: bytesDigest("session"), + bytes: 7, + }, + transcript: { + path: "artifacts/transcript.json", + sha256: bytesDigest("transcript"), + bytes: 10, + }, + }, + ...overrides, + }; + return { + ...base, + recordSha256: canaryRecordSha256(base), + }; +} const exactChangelog = [ "# Changelog", "", @@ -103,17 +201,13 @@ describe("release metadata", () => { expect(isMajorRelease("7.0.1")).toBe(false); }); - test("refuses a major release with no qualification record", async () => { + test("refuses every release with no qualification record", async () => { const directory = await recordDirectory(); - await expect(assertQualificationRecord("7.0.0", directory)).rejects.toThrow( - /no exact VERIFIED v2 decision record exists/, - ); - await expect( - assertQualificationRecord("7.1.0", directory), - ).resolves.toBeUndefined(); - await expect( - assertQualificationRecord("7.0.1", directory), - ).resolves.toBeUndefined(); + for (const version of ["7.0.0", "7.1.0", "7.0.1"]) { + await expect( + assertQualificationRecord(version, directory), + ).rejects.toThrow(/no exact VERIFIED v2 decision record exists/); + } }); test("accepts only an exact VERIFIED v2 record and refuses mismatches", async () => { @@ -157,4 +251,85 @@ describe("release metadata", () => { }), ).toMatch(/missing v2 decision digests/); }); + + test("requires a fresh passed exact-artifact canary for non-major strict evidence", async () => { + const decisions = await recordDirectory(); + const canaries = await recordDirectory(); + const version = "8.1.1"; + const expected = artifact(version); + const canary = canaryRecord(version); + await mkdir(join(canaries, "artifacts"), { recursive: true }); + await writeFile(join(canaries, "artifacts", "session.json"), "session"); + await writeFile( + join(canaries, "artifacts", "transcript.json"), + "transcript", + ); + await writeFile(join(canaries, `${version}.json`), JSON.stringify(canary)); + await writeFile( + join(decisions, "report-canary.json"), + JSON.stringify({ + ...canaryBoundDecision(version, canary.recordSha256), + reportId: "report-canary", + artifact: expected, + }), + ); + await expect( + assertStrictReleaseEvidence({ + version, + decisionsDirectory: decisions, + canaryPath: join(canaries, `${version}.json`), + expectedArtifact: expected, + }), + ).resolves.toBeUndefined(); + }); + + test("rejects stale, failed, incomplete, and artifact-mismatched canaries", () => { + const expected = artifact("8.1.1"); + for (const record of [ + canaryRecord("8.1.1", { status: "failed" }), + canaryRecord("8.1.1", { status: "incomplete" }), + canaryRecord("8.1.1", { + recordedAt: "2026-08-20T00:00:00.000Z", + expiresAt: "2026-08-21T00:00:00.000Z", + }), + canaryRecord("8.1.1", { artifact: artifact("8.1.2") }), + ]) { + expect(canaryRecordIssue("8.1.1", record, expected)).not.toBeNull(); + } + }); + + test("does not accept a null or grafted canary decision", async () => { + const decisions = await recordDirectory(); + const canaries = await recordDirectory(); + const version = "8.1.1"; + const expected = artifact(version); + const canary = canaryRecord(version); + await writeFile(join(canaries, `${version}.json`), JSON.stringify(canary)); + await mkdir(join(canaries, "artifacts"), { recursive: true }); + await writeFile(join(canaries, "artifacts", "session.json"), "session"); + await writeFile( + join(canaries, "artifacts", "transcript.json"), + "transcript", + ); + await writeFile( + join(decisions, "null.json"), + JSON.stringify(decisionRecord(version)), + ); + await writeFile( + join(decisions, "grafted.json"), + JSON.stringify({ + ...decisionRecord(version), + reportId: "grafted", + canarySha256: canary.recordSha256, + }), + ); + await expect( + assertStrictReleaseEvidence({ + version, + decisionsDirectory: decisions, + canaryPath: join(canaries, `${version}.json`), + expectedArtifact: expected, + }), + ).rejects.toThrow(/canary-bound/); + }); }); diff --git a/tests/release-qualification.test.ts b/tests/release-qualification.test.ts index 9830762..b364aae 100644 --- a/tests/release-qualification.test.ts +++ b/tests/release-qualification.test.ts @@ -636,7 +636,14 @@ describe("v2 qualification cutover", () => { analyzerSha256: expect.stringMatching(/^sha256:/), expectedProvenanceSha256: expect.stringMatching(/^sha256:/), decisionInputSha256: expect.stringMatching(/^sha256:/), + canarySha256: null, }); + const canaryBound = decisionRecordFor({ + ...verified, + canarySha256: digest("9"), + }); + expect(canaryBound.canarySha256).toBe(digest("9")); + expect(canaryBound.decisionInputSha256).not.toBe(first.decisionInputSha256); const notVerified = qualifyV2({ reportInput: v2Report(), From 0702650f18be702087a4e516b18f332fd485b4a0 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 11:32:13 +0200 Subject: [PATCH 4/4] docs(release): record phase 9 handoff --- .../evidence/phase-9-preparation.json | 30 +++++++++++++++ .../evidence/phase-9-review.md | 38 +++++++++++++++++++ .../phase-9-release-alignment.md | 10 +++++ .audit/eval-engineering.tsv | 5 +++ 4 files changed, 83 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-9-preparation.json create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-9-review.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-9-preparation.json b/.agents/plans/02-eval-engineering/evidence/phase-9-preparation.json new file mode 100644 index 0000000..51ca746 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-9-preparation.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "status": "incomplete", + "reason": "maintainer-run-opencode-canary-pending", + "releaseTag": "v8.1.1", + "artifact": { + "packageVersion": "8.1.1", + "sourceCommit": "647729c87d98fd773bc57355b31b2ac6ea7ab958", + "sourceTreeSha256": "sha256:cfbda87999aeffcfaaf1571d44ada3155e4ab5812e64c6bc7ac5129d8a3c9dcb", + "tarballSha256": "sha256:ea83bd80ae830781cc63983eb3b7138ce4d9e3f8fa21d94a753d16fcb85fcc24", + "unpackedManifestSha256": "sha256:60ae7a2de693c735bbace6cc331f556210673ec0e83fbc17d01fe6c752bd430a" + }, + "artifactSha256": "sha256:3e5c3dca41ddeb6424a5f876005db56a2c5a1bb5466b99c148f539f77901d304", + "checklistVersion": "phase9-canary-v1", + "checklistSha256": "sha256:bcaa925277568ce9b67c50f0785c3bfbadfa5bb40a6f36b39fad63c21a605110", + "pluginEntrySha256": "sha256:43ff17acb2d51279e28b806fdb7022b2f5d065bd8ee4ba2ae121542937420dfc", + "preparedSha256": "sha256:8fad620fe38ff76a109a11baa5d71d8c045046e8aa840e70fa186b2ff455a844", + "fixtureHost": "opencode-1.18.6", + "fixtureAutoLoadVerified": true, + "fixtureObservedSurfaces": [ + "flow-reviewer", + "flow-auto", + "flow-status" + ], + "dryRunVerdict": "INCONCLUSIVE", + "strictTagGateBlocked": true, + "expectedCanaryPath": "evals/canary/8.1.1.json", + "expectedDecisionState": "VERIFIED with matching non-null canarySha256", + "releaseRequested": false +} diff --git a/.agents/plans/02-eval-engineering/evidence/phase-9-review.md b/.agents/plans/02-eval-engineering/evidence/phase-9-review.md new file mode 100644 index 0000000..e7574a5 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-9-review.md @@ -0,0 +1,38 @@ +# Phase 9 Interrogate review and canary handoff + +Phase 9 replaces the temporary unconditional release stop with an exact manual +canary protocol. Preparation validates and copies the tarball, extracts its bundled +entry into a project-local OpenCode plugin fixture, pins dependencies, and writes a +versioned checklist manifest outside Git. Direct OpenCode 1.18.6 inspection proved +that the fixture auto-loads the exact local plugin and exposes Flow's reviewer and +commands. + +Recording accepts passed, failed, or incomplete outcomes. It requires the exact +check set, explicit operator, host digest, ActorIdentity-shaped manager/reviewer +evidence, and checklist-derived 72-hour expiry. Session and transcript JSON are +scrubbed for credentials, workspace paths, and runtime IDs before immutable +artifact and record publication. Byte-identical replay succeeds and changed bytes +conflict. + +Release decisions now carry `canarySha256`, and `decisionInputSha256` includes it. +The scheduled canary-null decision remains separate; after the manual run the +maintainer reruns qualification to create a hash-suffixed canary-bound decision. +Strict tag verification recomputes that input hash, checks the full rebuilt +ArtifactIdentity, reads the fresh passed canary and sanitized artifacts from the +tagged checkout, and stops before any publish step on mismatch. + +The release workflow now verifies on main and tags. Main has read-only contents +permission, reports missing release evidence as `INCONCLUSIVE`, and has no publish +job path. The tag-only release job has scoped write/id-token permissions and depends +on deterministic verification before npm or GitHub publication. The temporary +`canary-not-enabled` step is removed. + +The non-model runtime proof packed the current bytes, generated the fixture, +observed Flow surfaces on the pinned host, returned `INCONCLUSIVE` in dry-run, and +failed strict verification as required. No canary record, canary-bound decision, +tag, or release was fabricated. The final phase remains `INCONCLUSIVE` until the +maintainer completes the prepared OpenCode checklist. + +The full repository gate passes 520 tests with one intentional live-smoke skip. +The final four-model review found no unresolved blocker in the implemented +infrastructure; the human canary stop remains open by design. diff --git a/.agents/plans/02-eval-engineering/phase-9-release-alignment.md b/.agents/plans/02-eval-engineering/phase-9-release-alignment.md index 0ce82f4..9e8278a 100644 --- a/.agents/plans/02-eval-engineering/phase-9-release-alignment.md +++ b/.agents/plans/02-eval-engineering/phase-9-release-alignment.md @@ -35,3 +35,13 @@ rebuild the same tarball hash and preserve deterministic checks in the publish j Stop gate. This final phase remains `INCONCLUSIVE` until the maintainer completes the canary. It does not block Phases 0 through 8. + +## Outcome + +Infrastructure implemented and verified. Exact-artifact preparation, strict +passed/failed/incomplete records, sanitized evidence, expiry, canary-bound decision +hashes, main dry-run verification, and tag-only strict publication are live. The +prepared fixture loads Flow on OpenCode 1.18.6. The phase remains `INCONCLUSIVE` +because the maintainer-run canary and resulting canary-bound decision are pending; +no release was requested. See `evidence/phase-9-architecture.md`, +`evidence/phase-9-review.md`, and `evidence/phase-9-preparation.json`. diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index 3bb7e32..4853a0d 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -58,3 +58,8 @@ ts phase decision why evidence result 2026-08-25T08:55:41Z phase-8 expanded mutation-tested hidden coverage new tasks need executable controls and explicit contamination boundaries before producing useful evidence evals/benchmarks.ts; tests/benchmark-reporting.test.ts VERIFIED 5 cases, 12 rejected mutations, known-good implementations pass 2026-08-25T08:55:41Z phase-8 kept coverage promotion closed uncalibrated cases cannot silently become release regressions evals/benchmark-run.ts catalog policy; .agents/plans/02-eval-engineering/evidence/phase-8-review.md VERIFIED every benchmark case remains report-only; no legacy backfill 2026-08-25T08:55:41Z phase-8 ran Deslop, four-model Interrogate, and full repository gate trend and coverage changes must remain reviewable and regression-free bun run check VERIFIED 509 pass, 1 skip, 0 fail; no unresolved blocker +2026-08-25T09:30:36Z phase-9 selected exact canary and release binding through Architect and Arena tag publication needs a human stop gate bound into the final decision input without making main a publisher .agents/plans/02-eval-engineering/evidence/phase-9-architecture.md VERIFIED ActorIdentity evidence, 72h expiry, canarySha256 decision binding, tag-only publish +2026-08-25T09:30:36Z phase-9 implemented exact-artifact prepare, record, and verify the maintainer needs a rerunnable fixture and write-once sanitized evidence instead of a prose attestation scripts/eval-canary.ts; tests/eval-canary.test.ts VERIFIED strict record boundary, redaction, immutable replay/conflict, local plugin fixture +2026-08-25T09:30:36Z phase-9 bound release decisions and workflow to the fresh canary a canary cannot be grafted onto an older decision and main must have no publication authority scripts/qualify-release.ts; scripts/release-metadata.ts; .github/workflows/release.yml VERIFIED decision input recomputation, main dry-run, strict tag gate, scoped permissions +2026-08-25T09:30:36Z phase-9 prepared and drove the exact fixture without fabricating the human result the infrastructure must be proven while the manual stop remains honest .agents/plans/02-eval-engineering/evidence/phase-9-preparation.json INCONCLUSIVE exact artifact loads on OpenCode 1.18.6; canary and canary-bound decision pending +2026-08-25T09:30:36Z phase-9 ran Deslop, four-model Interrogate, workflow checks, and full repository gate the final release boundary must be reviewable and regression-free bun run check; actionlint; strict canary dry-run VERIFIED 520 pass, 1 skip, 0 fail; strict tag gate blocks pending canary