From 04a7eba6c21aafc076c8269979d4e60f7ed25831 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 06:55:04 +0200 Subject: [PATCH 1/2] Cut release qualification over to v2 evidence --- .github/workflows/evals.yml | 68 +++++++- .github/workflows/release.yml | 23 +-- evals/analysis.ts | 4 +- evals/release-policy.ts | 12 ++ evals/report-store.ts | 22 +++ evals/run.ts | 36 ++-- scripts/qualify-release.ts | 238 +++++++++++++++++++++++++++ scripts/release-metadata.ts | 113 ++++++++++--- tests/advisory-analysis.test.ts | 4 +- tests/atomic-analysis.test.ts | 19 ++- tests/documentation-contract.test.ts | 12 +- tests/release-metadata.test.ts | 56 +++++-- tests/release-qualification.test.ts | 221 +++++++++++++++++++++++++ 13 files changed, 759 insertions(+), 69 deletions(-) create mode 100644 evals/release-policy.ts diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index e5246f4..567a08c 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -83,6 +83,7 @@ jobs: fi - name: Run the eval matrix + id: run if: steps.matrix.outputs.models != '' env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -96,19 +97,74 @@ jobs: # A failing scenario must still publish its report: the durable documents are # the only way to read why, and paying for the pass twice is the alternative. continue-on-error: true - run: bun run eval -- --repeat "$REPEAT" + run: | + set -o pipefail + set +e + bun run eval -- --repeat "$REPEAT" 2>&1 | tee eval-v2.log + status=${PIPESTATUS[0]} + set -e + report="$(sed -n 's/^V2 report: //p' eval-v2.log | tail -n 1)" + if [ -z "$report" ] || [ ! -f "$report" ]; then + echo "::error::The evaluator did not print an existing V2 report path." + exit 1 + fi + campaign_dir="$(dirname "$report")" + catalog="$campaign_dir/catalog.json" + artifact="$campaign_dir/artifact.tgz" + if [ ! -f "$catalog" ]; then + echo "::error::V2 campaign plan/catalog was not persisted at $catalog." + exit 1 + fi + if [ ! -f "$artifact" ]; then + echo "::error::Measured artifact was not persisted at $artifact." + exit 1 + fi + { + printf 'report=%s\n' "$report" + printf 'campaign=%s\n' "$campaign_dir" + printf 'catalog=%s\n' "$catalog" + printf 'artifact=%s\n' "$artifact" + } >> "$GITHUB_OUTPUT" + exit "$status" # Before qualification, so a NOT QUALIFIED verdict still leaves the report that # explains it. Publishing after the gate meant the one run worth reading was the # one whose artifact was skipped. - name: Publish the report - if: steps.matrix.outputs.models != '' + if: always() && steps.matrix.outputs.models != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: eval-report - path: evals/results/ + name: eval-v2-report + path: | + ${{ steps.run.outputs.report }} + ${{ steps.run.outputs.campaign }} + eval-v2.log retention-days: 90 - name: Apply release-qualification thresholds - if: steps.matrix.outputs.models != '' - run: bun run qualify + id: qualify + if: always() && steps.matrix.outputs.models != '' && steps.run.outputs.report != '' + continue-on-error: true + run: | + set +e + mkdir -p evals/decisions + bun run qualify -- --report "${{ steps.run.outputs.report }}" --catalog "${{ steps.run.outputs.catalog }}" --artifact "${{ steps.run.outputs.artifact }}" --decisions-dir evals/decisions 2>&1 | tee qualification.log + status=${PIPESTATUS[0]} + decision_path="$(sed -n -E 's/^(VERIFIED|NOT VERIFIED|INCONCLUSIVE): (.*)$/\2/p' qualification.log | tail -n 1)" + if [ -z "$decision_path" ] || [ ! -f "$decision_path" ]; then + echo "::error::Qualifier did not print an existing decision path." + exit 1 + fi + printf 'decision=%s\n' "$decision_path" >> "$GITHUB_OUTPUT" + exit "$status" + + - name: Publish the decision artifacts + if: always() && steps.matrix.outputs.models != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eval-v2-decision + path: | + qualification.log + ${{ steps.qualify.outputs.decision }} + if-no-files-found: warn + retention-days: 90 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5b3046..669406c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,6 @@ on: push: tags: - 'v*' - branches: - - main permissions: contents: write @@ -30,17 +28,10 @@ jobs: tag="v${version}" echo "tag=${tag}" >> "$GITHUB_OUTPUT" - if [[ "${GITHUB_REF_TYPE}" == "tag" && "${GITHUB_REF_NAME}" != "${tag}" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${tag}" ]]; then echo "::error::Release tag/version mismatch: tag=${GITHUB_REF_NAME}, package.json=${version}." exit 1 fi - - if [[ "${GITHUB_REF_TYPE}" == "branch" ]] && git ls-remote --exit-code --tags origin "refs/tags/${tag}" >/dev/null; then - echo "Tag ${tag} already exists; skipping release on main." - echo "publish=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "publish=true" >> "$GITHUB_OUTPUT" release: @@ -86,9 +77,21 @@ jobs: run: | set -euo pipefail bun pm pack --destination . + + - name: Verify exact VERIFIED V2 artifact decision + shell: bash + run: | + set -euo pipefail tarball="$(ls opencode-plugin-flow-*.tgz)" + bun run release:metadata -- --tag "${{ needs.decide.outputs.tag }}" --notes-file release-notes.md --artifact "$tarball" shasum -a 256 "$tarball" > "${tarball}.sha256" + # Temporary Phase 5 stop: the exact manual OpenCode canary is owned by Phase 9. + - name: Require Phase 9 canary + run: | + echo "::error::Release blocked: canary-not-enabled until Phase 9 release alignment is implemented." + exit 1 + - name: Publish to npm shell: bash run: | diff --git a/evals/analysis.ts b/evals/analysis.ts index 0a807a1..73003d3 100644 --- a/evals/analysis.ts +++ b/evals/analysis.ts @@ -29,13 +29,13 @@ export type ExpectedActorProvenance = { export type ExpectedAttemptProvenance = { readonly cellId: string; + readonly hostConfigSha256: string; readonly actors: readonly ExpectedActorProvenance[]; readonly instructions: readonly InstructionDelivery[]; }; type CommonExpectedProvenance = { readonly evaluator: EvaluatorIdentity; - readonly hostConfigSha256: string; readonly attempts: readonly ExpectedAttemptProvenance[]; }; @@ -249,7 +249,7 @@ function compareCommonProvenance( "Evaluator identity does not match expected provenance.", ); } - if (attempt.hostConfigSha256 !== expected.hostConfigSha256) { + if (attempt.hostConfigSha256 !== expectedAttempt.hostConfigSha256) { mismatch( mismatches, attempt.attemptId, diff --git a/evals/release-policy.ts b/evals/release-policy.ts new file mode 100644 index 0000000..2c9bb8d --- /dev/null +++ b/evals/release-policy.ts @@ -0,0 +1,12 @@ +export const RELEASE_PASS_RATES: Readonly> = { + "happy-path": 1, + "plan-only-stops": 1, + "goal-change-refused": 1, + "failing-gate-blocks": 0.9, + "resumes-after-interruption": 1, + "unprovable-claim-refused": 0.9, + "continuation-accepted": 1, +}; + +export const RELEASE_MIN_PROVIDERS = 2; +export const RELEASE_MIN_SCORED_ATTEMPTS = 3; diff --git a/evals/report-store.ts b/evals/report-store.ts index 6f69d67..053e9fb 100644 --- a/evals/report-store.ts +++ b/evals/report-store.ts @@ -138,6 +138,8 @@ async function readJson(path: string): Promise { export class ReportStore { private readonly attemptsDirectory: string; private readonly transcriptsDirectory: string; + private readonly catalogPath: string; + private readonly artifactPath: string; private readonly planPath: string; private readonly completionPath: string; private readonly reportPath: string; @@ -153,6 +155,8 @@ export class ReportStore { this.hooks = hooks; this.attemptsDirectory = join(directory, "attempts"); this.transcriptsDirectory = join(directory, "transcripts"); + this.catalogPath = join(directory, "catalog.json"); + this.artifactPath = join(directory, "artifact.tgz"); this.planPath = join(directory, "plan.json"); this.completionPath = join(directory, "completion.json"); this.reportPath = join(directory, "report.json"); @@ -167,6 +171,24 @@ export class ReportStore { ); } + async writeCatalog( + catalog: ValidatedCaseCatalog, + ): Promise<"written" | "replayed"> { + return writeImmutable( + this.catalogPath, + Buffer.from(canonicalJson(catalog)), + this.hooks, + ); + } + + async writeArtifact(artifactPath: string): Promise<"written" | "replayed"> { + return writeImmutable( + this.artifactPath, + await readFile(artifactPath), + this.hooks, + ); + } + private async plan(): Promise { const parsed = CampaignPlanSchema.safeParse(await readJson(this.planPath)); if (!parsed.success) fail("Stored campaign plan is invalid."); diff --git a/evals/run.ts b/evals/run.ts index 09a581e..0b3fb50 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -58,6 +58,11 @@ import { redactTranscript, tarballSha256, } from "./provenance.js"; +import { + RELEASE_MIN_PROVIDERS, + RELEASE_MIN_SCORED_ATTEMPTS, + RELEASE_PASS_RATES, +} from "./release-policy.js"; import type { ActorIdentity, ArtifactIdentity, @@ -198,17 +203,24 @@ function caseCatalogFor( scenarios: readonly (typeof SCENARIOS)[number][], ): ValidatedCaseCatalog { const parsed = parseCaseCatalog( - scenarios.map((scenario) => ({ - caseId: scenario.id, - caseVersion: 1, - evidenceClass: "conformance" as const, - oracle: "durable-state" as const, - release: "report-only" as const, - minProviders: 1, - minScoredAttempts: 1, - minPassRate: 1, - reviewerPromotionRecordSha256: null, - })), + scenarios.map((scenario) => { + const minPassRate = RELEASE_PASS_RATES[scenario.id] ?? null; + return { + caseId: scenario.id, + caseVersion: 1, + evidenceClass: "conformance" as const, + oracle: "durable-state" as const, + release: + minPassRate === null + ? ("report-only" as const) + : ("required" as const), + minProviders: minPassRate === null ? 1 : RELEASE_MIN_PROVIDERS, + minScoredAttempts: + minPassRate === null ? 1 : RELEASE_MIN_SCORED_ATTEMPTS, + minPassRate, + reviewerPromotionRecordSha256: null, + }; + }), ); if (!parsed.ok) { throw new Error( @@ -616,6 +628,7 @@ async function main(): Promise { catalog: v2Catalog, }); await reportStore.initialize(v2Plan); + await reportStore.writeCatalog(v2Catalog); const campaignStartedAt = new Date().toISOString(); const campaignCells = v2Plan.cells; const v2Attempts: AttemptRecordV2[] = []; @@ -630,6 +643,7 @@ async function main(): Promise { repositoryRoot, tarballPath: tarball, }); + await reportStore.writeArtifact(tarball); const evaluator = evaluatorIdentity({ sourceCommit: artifact.sourceCommit, caseCatalog: selected.map((scenario) => ({ diff --git a/scripts/qualify-release.ts b/scripts/qualify-release.ts index 747b998..68d0a5a 100644 --- a/scripts/qualify-release.ts +++ b/scripts/qualify-release.ts @@ -18,8 +18,42 @@ import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { + deriveReleaseDecision, + type ExpectedActorProvenance, + type ReleaseDecision, + type ReleaseExpectedProvenance, +} from "../evals/analysis.js"; +import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; +import { + parseCaseCatalog, + type ValidatedCaseCatalog, +} from "../evals/catalog.js"; +import { inspectArtifact } from "../evals/provenance.js"; +import { + type ArtifactIdentity, + parseReport, + type ValidatedReport, +} from "../evals/report.js"; import { isMajorRelease } from "./release-metadata.js"; +export type DecisionRecord = { + readonly schemaVersion: 1; + readonly reportId: string; + readonly verdict: "VERIFIED" | "NOT VERIFIED" | "INCONCLUSIVE"; + readonly reportSha256: string; + readonly artifactSha256: string; + readonly evaluatorSha256: string; + readonly catalogSha256: string; + readonly policySha256: string; + readonly actorSha256: string; + readonly analyzerSha256: string; + readonly expectedProvenanceSha256: string; + readonly decisionInputSha256: string; + readonly artifact: ArtifactIdentity; + readonly reasons: readonly string[]; +}; + /** * Minimum share of scored attempts a scenario must pass, per scenario. * @@ -447,7 +481,211 @@ export function qualificationFailures(report: Report): string[] { return failures; } +export function expectedProvenanceFor( + report: ValidatedReport, + artifact: ArtifactIdentity, +): ReleaseExpectedProvenance { + const first = report.attempts[0]; + if (!first) throw new Error("A v2 qualification report requires an attempt."); + return { + kind: "release", + artifact, + evaluator: first.evaluator, + attempts: report.attempts.map((attempt) => ({ + cellId: attempt.cellId, + hostConfigSha256: attempt.hostConfigSha256, + actors: attempt.actors.map( + (actor): ExpectedActorProvenance => ({ + role: actor.role, + requestedModel: actor.requestedModel, + actualModel: + actor.actualModel.kind === "observed" + ? { kind: "observed", value: actor.actualModel.value } + : { + kind: "allow-unobserved", + value: actor.requestedModel, + reason: actor.actualModel.reason, + }, + }), + ), + instructions: attempt.instructions, + })), + }; +} + +export function qualifyV2(input: { + readonly reportInput: unknown; + readonly catalogInput: unknown; + readonly artifact: ArtifactIdentity; +}): { + readonly report: ValidatedReport; + readonly catalog: ValidatedCaseCatalog; + readonly decision: ReleaseDecision; + readonly expected: ReleaseExpectedProvenance; +} { + const catalog = parseCaseCatalog(input.catalogInput); + if (!catalog.ok) { + throw new Error( + `Invalid v2 catalog: ${catalog.issues + .map((issue) => `${issue.path} ${issue.message}`) + .join("; ")}`, + ); + } + const parsed = parseReport(input.reportInput, catalog.value); + if (!parsed.ok) { + throw new Error( + `Invalid v2 report: ${parsed.issues + .map((issue) => `${issue.path} ${issue.message}`) + .join("; ")}`, + ); + } + const expected = expectedProvenanceFor(parsed.value, input.artifact); + return { + report: parsed.value, + catalog: catalog.value, + expected, + decision: deriveReleaseDecision({ + report: parsed.value, + catalog: catalog.value, + expected, + }), + }; +} + +export function decisionRecordFor(input: { + readonly report: ValidatedReport; + readonly catalog: ValidatedCaseCatalog; + readonly expected: ReleaseExpectedProvenance; + readonly decision: ReleaseDecision; +}): DecisionRecord { + const reportSha256 = canonicalSha256("flow-decision-report-v1", input.report); + const artifactSha256 = canonicalSha256( + "flow-decision-artifact-v1", + input.expected.artifact, + ); + const evaluatorSha256 = canonicalSha256( + "flow-decision-evaluator-v1", + input.expected.evaluator, + ); + const catalogSha256 = canonicalSha256( + "flow-decision-catalog-v1", + input.catalog, + ); + const policySha256 = canonicalSha256("flow-decision-policy-v1", { + plan: input.report.plan, + catalogSha256, + }); + const actorSha256 = canonicalSha256( + "flow-decision-actors-v1", + input.expected.attempts.map((attempt) => ({ + cellId: attempt.cellId, + actors: attempt.actors, + })), + ); + const analyzerSha256 = canonicalSha256("flow-decision-analyzer-v1", { + module: "evals/analysis.ts", + decisionSchemaVersion: 1, + }); + const expectedProvenanceSha256 = canonicalSha256( + "flow-decision-expected-provenance-v1", + input.expected, + ); + const decisionInputSha256 = canonicalSha256("flow-decision-input-v1", { + reportSha256, + artifactSha256, + evaluatorSha256, + catalogSha256, + policySha256, + actorSha256, + analyzerSha256, + expectedProvenanceSha256, + }); + return { + schemaVersion: 1, + reportId: input.report.reportId, + verdict: input.decision.verdict, + reportSha256, + artifactSha256, + evaluatorSha256, + catalogSha256, + policySha256, + actorSha256, + analyzerSha256, + expectedProvenanceSha256, + decisionInputSha256, + artifact: input.expected.artifact, + reasons: input.decision.reasons.map((reason) => reason.message), + }; +} + +export async function writeDecisionRecord(input: { + readonly record: DecisionRecord; + readonly directory: string; +}): Promise { + await mkdir(input.directory, { recursive: true }); + const path = join(input.directory, `${input.record.reportId}.json`); + await writeFile(path, canonicalJson(input.record), "utf8"); + return path; +} + +function requiredOption( + options: Readonly>, + name: string, +): string { + const value = options[name]; + if (!value) { + throw new Error( + "Usage: bun run qualify -- --report --catalog --artifact [--decisions-dir ]", + ); + } + return value; +} + async function main(): Promise { + const args = process.argv.slice(2); + const options: Record = {}; + for (let index = 0; index < args.length; index += 1) { + const option = args[index]; + if ( + option !== "--report" && + option !== "--catalog" && + option !== "--artifact" && + option !== "--decisions-dir" + ) { + throw new Error( + "Usage: bun run qualify -- --report --catalog --artifact [--decisions-dir ]", + ); + } + const value = args[index + 1]; + if (!value || value.startsWith("--")) + throw new Error(`${option} requires a value.`); + options[option] = value; + index += 1; + } + const reportPath = requiredOption(options, "--report"); + const catalogPath = requiredOption(options, "--catalog"); + const artifactPath = requiredOption(options, "--artifact"); + const artifact = await inspectArtifact({ + repositoryRoot: join(import.meta.dir, ".."), + tarballPath: artifactPath, + }); + const result = qualifyV2({ + reportInput: JSON.parse(await readFile(reportPath, "utf8")), + catalogInput: JSON.parse(await readFile(catalogPath, "utf8")), + artifact, + }); + const record = decisionRecordFor(result); + const path = await writeDecisionRecord({ + record, + directory: + options["--decisions-dir"] ?? + join(import.meta.dir, "..", "evals", "decisions"), + }); + process.stdout.write(`${record.verdict}: ${path}\n`); + if (record.verdict !== "VERIFIED") process.exitCode = 1; +} + +export async function historicalSummaryQualification(): Promise { const args = process.argv.slice(2); let recordVersion: string | undefined; const paths: string[] = []; diff --git a/scripts/release-metadata.ts b/scripts/release-metadata.ts index 1ae7c5a..abee8e7 100644 --- a/scripts/release-metadata.ts +++ b/scripts/release-metadata.ts @@ -1,5 +1,8 @@ import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { canonicalJson } from "../evals/canonical-json.js"; +import { inspectArtifact } from "../evals/provenance.js"; +import type { ArtifactIdentity } from "../evals/report.js"; export type ReleaseMetadataInput = { packageVersion: string; @@ -62,50 +65,97 @@ export function isMajorRelease(version: string): boolean { } /** - * Why a qualification record does not qualify this version, or null when it - * does. The record is a checklist with a filename, not a proof: a human can - * write one by hand. The point is that a major tag is refused without one, so - * skipping the qualification run has to show up in the release diff. - * - * `flowVersion` is not checked here. Qualification runs against the current - * build; the bump commit then cuts the release, so the record's flowVersion is - * the pre-bump version by design. `writeQualificationRecord` binds the report - * to that build at record time. + * Why a v2 decision record does not qualify this version, or null when it does. + * The release path accepts only a parsed, three-valued decision whose measured + * artifact package version is exact and whose verdict is VERIFIED. */ export function qualificationRecordIssue( version: string, record: unknown, + expectedArtifact?: ArtifactIdentity, ): string | null { - const entry = record as { version?: unknown; verdict?: unknown } | null; - if (!entry || typeof entry !== "object") { + if (!record || typeof record !== "object" || Array.isArray(record)) { return `no qualification record exists for ${version}`; } - if (entry.version !== version) { - return `the qualification record names ${String(entry.version)}, not ${version}`; + const entry = record as { + schemaVersion?: unknown; + reportId?: unknown; + verdict?: unknown; + artifact?: ArtifactIdentity | null; + reportSha256?: unknown; + artifactSha256?: unknown; + evaluatorSha256?: unknown; + catalogSha256?: unknown; + policySha256?: unknown; + actorSha256?: unknown; + analyzerSha256?: unknown; + expectedProvenanceSha256?: unknown; + decisionInputSha256?: unknown; + }; + if (entry.schemaVersion !== 1 || typeof entry.reportId !== "string") { + return `the qualification record for ${version} is not a v2 decision record`; + } + if (entry.artifact?.packageVersion !== version) { + return `the qualification artifact names ${String(entry.artifact?.packageVersion)}, not ${version}`; + } + if (entry.verdict !== "VERIFIED") { + return `the qualification record for ${version} has verdict ${String(entry.verdict)}, not VERIFIED`; + } + const digests = [ + entry.reportSha256, + entry.artifactSha256, + entry.evaluatorSha256, + entry.catalogSha256, + entry.policySha256, + entry.actorSha256, + entry.analyzerSha256, + entry.expectedProvenanceSha256, + entry.decisionInputSha256, + ]; + if ( + !digests.every( + (digest) => + typeof digest === "string" && /^sha256:[a-f0-9]{64}$/.test(digest), + ) + ) { + return `the qualification record for ${version} is missing v2 decision digests`; } - if (entry.verdict !== "QUALIFIED") { - return `the qualification record for ${version} has verdict ${String(entry.verdict)}, not QUALIFIED`; + if ( + expectedArtifact && + canonicalJson(entry.artifact) !== canonicalJson(expectedArtifact) + ) { + return `the qualification artifact does not match the rebuilt artifact for ${version}`; } return null; } export async function assertQualificationRecord( version: string, - directory = join("evals", "qualification"), + directory = join("evals", "decisions"), + expectedArtifact?: ArtifactIdentity, ): Promise { if (!isMajorRelease(version)) return; - let record: unknown = null; + let records: unknown[] = []; try { - record = JSON.parse( - await readFile(join(directory, `${version}.json`), "utf8"), + const { readdir } = await import("node:fs/promises"); + records = await Promise.all( + (await readdir(directory)) + .filter((name) => name.endsWith(".json")) + .map(async (name) => + JSON.parse(await readFile(join(directory, name), "utf8")), + ), ); } catch { - record = null; + records = []; } - const issue = qualificationRecordIssue(version, record); - if (issue) { + if ( + !records.some( + (record) => + qualificationRecordIssue(version, record, expectedArtifact) === null, + ) + ) { throw new Error( - `Major release ${version} cannot proceed: ${issue}. Run \`bun run qualify -- --record ${version}\` against a qualifying report and commit the record.`, + `Major release ${version} cannot proceed: no exact VERIFIED v2 decision record exists. Run \`bun run qualify -- --report --catalog --artifact \` and commit the decision.`, ); } } @@ -125,6 +175,7 @@ function optionValue( async function main(args: readonly string[]): Promise { let tag: string | undefined; let notesFile: string | undefined; + let artifactPath: string | undefined; for (let index = 0; index < args.length; index += 1) { const argument = args[index]; switch (argument) { @@ -136,6 +187,10 @@ async function main(args: readonly string[]): Promise { notesFile = optionValue(args, index, argument); index += 1; break; + case "--artifact": + artifactPath = optionValue(args, index, argument); + index += 1; + break; default: throw new Error(`Unknown option: ${argument}`); } @@ -153,7 +208,17 @@ async function main(args: readonly string[]): Promise { ...(tag ? { tag } : {}), changelog: await readFile("CHANGELOG.md", "utf8"), }); - await assertQualificationRecord(packageMetadata.version); + const expectedArtifact = artifactPath + ? await inspectArtifact({ + repositoryRoot: join(import.meta.dir, ".."), + tarballPath: artifactPath, + }) + : undefined; + await assertQualificationRecord( + packageMetadata.version, + undefined, + expectedArtifact, + ); if (notesFile) await writeFile(notesFile, result.releaseNotes, "utf8"); process.stdout.write( `Release metadata matches ${packageMetadata.version}.\n`, diff --git a/tests/advisory-analysis.test.ts b/tests/advisory-analysis.test.ts index 9760d01..b484d07 100644 --- a/tests/advisory-analysis.test.ts +++ b/tests/advisory-analysis.test.ts @@ -109,9 +109,9 @@ function releaseExpected(report: ValidatedReport): ReleaseExpectedProvenance { kind: "release", artifact: first.artifact, evaluator: first.evaluator, - hostConfigSha256: first.hostConfigSha256, attempts: report.attempts.map((attempt) => ({ cellId: attempt.cellId, + hostConfigSha256: attempt.hostConfigSha256, actors: attempt.actors.map(expectedActor), instructions: attempt.instructions, })), @@ -452,9 +452,9 @@ function pairedExpected(report: ValidatedReport): PairedExpectedProvenance { kind: "paired", artifacts: [V2_ARTIFACT_A, V2_ARTIFACT_B], evaluator: first.evaluator, - hostConfigSha256: first.hostConfigSha256, attempts: report.attempts.map((attempt) => ({ cellId: attempt.cellId, + hostConfigSha256: attempt.hostConfigSha256, actors: attempt.actors.map(expectedActor), instructions: attempt.instructions, })), diff --git a/tests/atomic-analysis.test.ts b/tests/atomic-analysis.test.ts index a6d9a6c..eae4ff3 100644 --- a/tests/atomic-analysis.test.ts +++ b/tests/atomic-analysis.test.ts @@ -278,9 +278,9 @@ function releaseExpected(report: ValidatedReport): ReleaseExpectedProvenance { kind: "release", artifact: first.artifact, evaluator: first.evaluator, - hostConfigSha256: first.hostConfigSha256, attempts: report.attempts.map((attempt) => ({ cellId: attempt.cellId, + hostConfigSha256: attempt.hostConfigSha256, actors: attempt.actors.map(expectedActor), instructions: attempt.instructions, })), @@ -364,6 +364,23 @@ describe("v2 atomic release decisions", () => { ); }); + test("accepts distinct frozen host configurations for different model cells", () => { + const raw = buildRateReport(); + const second = raw.attempts[1]; + if (second === undefined) + throw new Error("Expected second provider attempt."); + second.hostConfigSha256 = DIGEST_A; + const catalog = mustCatalog(rateCatalog()); + const report = mustReport(raw, catalog); + expect( + deriveReleaseDecision({ + report, + catalog, + expected: releaseExpected(report), + }).verdict, + ).toBe("VERIFIED"); + }); + test("keeps report-only stops and provenance out of the release verdict", () => { const raw = buildRateReport(); const cell = structuredClone(raw.plan.cells[0]); diff --git a/tests/documentation-contract.test.ts b/tests/documentation-contract.test.ts index f447c65..008c557 100644 --- a/tests/documentation-contract.test.ts +++ b/tests/documentation-contract.test.ts @@ -504,9 +504,12 @@ describe("Flow documentation contract", () => { expect(combined).toContain("tests/workspace-persistence.test.ts"); expect(combined).toContain("npm publish"); const release = await readFile(".github/workflows/release.yml", "utf8"); - expect(release).toMatch(/^ {4}branches:\n {6}- main$/m); + expect(release).toMatch(/^ {2}push:\n {4}tags:/m); + expect(release).not.toContain("branches:"); expect(release).toContain('tag="v${version}"'); expect(release).toContain('--target "${GITHUB_SHA}"'); + expect(release).toContain("Verify exact VERIFIED V2 artifact decision"); + expect(release).toContain("canary-not-enabled"); // Model-driven evals need credentials and cost real money, so they run on a // schedule and never on a pull request. `evals.yml` is the one workflow allowed @@ -516,6 +519,13 @@ describe("Flow documentation contract", () => { // beyond one provider (docs/adr/0010-declared-canonical-gate.md). const evals = await readFile(".github/workflows/evals.yml", "utf8"); expect(evals).toContain("bun run eval"); + expect(evals).toContain("V2 report:"); + expect(evals).toContain("bun run qualify -- --report"); + expect(evals).toContain("steps.run.outputs.catalog"); + expect(evals).toContain("steps.run.outputs.artifact"); + expect(evals).toContain("--decisions-dir"); + expect(evals).toContain("eval-v2-decision"); + expect(evals).toContain("if: always()"); expect(evals).toContain("schedule:"); expect(evals).not.toMatch(/^on:[\s\S]*?^\s{2}(?:pull_request|push):/m); for (const gate of [ diff --git a/tests/release-metadata.test.ts b/tests/release-metadata.test.ts index 4be8794..53da24d 100644 --- a/tests/release-metadata.test.ts +++ b/tests/release-metadata.test.ts @@ -27,6 +27,29 @@ async function recordDirectory(): Promise { } const VERSION = "6.0.0"; +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; +const artifact = (packageVersion: string) => ({ + packageVersion, + sourceCommit: "commit", + sourceTreeSha256: digest("a"), + tarballSha256: digest("b"), + unpackedManifestSha256: digest("c"), +}); +const decisionRecord = (packageVersion: string, verdict = "VERIFIED") => ({ + schemaVersion: 1, + reportId: "report", + verdict, + artifact: artifact(packageVersion), + reportSha256: digest("d"), + artifactSha256: digest("e"), + evaluatorSha256: digest("f"), + catalogSha256: digest("9"), + policySha256: digest("0"), + actorSha256: digest("1"), + analyzerSha256: digest("2"), + expectedProvenanceSha256: digest("3"), + decisionInputSha256: digest("4"), +}); const exactChangelog = [ "# Changelog", "", @@ -83,7 +106,7 @@ describe("release metadata", () => { test("refuses a major release with no qualification record", async () => { const directory = await recordDirectory(); await expect(assertQualificationRecord("7.0.0", directory)).rejects.toThrow( - /no qualification record exists for 7\.0\.0/, + /no exact VERIFIED v2 decision record exists/, ); await expect( assertQualificationRecord("7.1.0", directory), @@ -93,11 +116,11 @@ describe("release metadata", () => { ).resolves.toBeUndefined(); }); - test("accepts a matching QUALIFIED record and refuses mismatches", async () => { + test("accepts only an exact VERIFIED v2 record and refuses mismatches", async () => { const directory = await recordDirectory(); await writeFile( - join(directory, "7.0.0.json"), - JSON.stringify({ version: "7.0.0", verdict: "QUALIFIED" }), + join(directory, "report.json"), + JSON.stringify(decisionRecord("7.0.0")), ); await expect( assertQualificationRecord("7.0.0", directory), @@ -108,21 +131,30 @@ describe("release metadata", () => { ); expect( qualificationRecordIssue("8.0.0", { - version: "7.0.0", - verdict: "QUALIFIED", + ...decisionRecord("7.0.0"), }), - ).toMatch(/names 7\.0\.0, not 8\.0\.0/); + ).toMatch(/artifact names 7\.0\.0, not 8\.0\.0/); expect( qualificationRecordIssue("8.0.0", { - version: "8.0.0", - verdict: "NOT QUALIFIED", + ...decisionRecord("8.0.0", "NOT VERIFIED"), }), - ).toMatch(/not QUALIFIED/); + ).toMatch(/not VERIFIED/); expect( qualificationRecordIssue("8.0.0", { - version: "8.0.0", - verdict: "QUALIFIED", + ...decisionRecord("8.0.0"), }), ).toBeNull(); + expect( + qualificationRecordIssue("8.0.0", decisionRecord("8.0.0"), { + ...artifact("8.0.0"), + tarballSha256: digest("9"), + }), + ).toMatch(/does not match the rebuilt artifact/); + expect( + qualificationRecordIssue("8.0.0", { + ...decisionRecord("8.0.0"), + analyzerSha256: "sha256:short", + }), + ).toMatch(/missing v2 decision digests/); }); }); diff --git a/tests/release-qualification.test.ts b/tests/release-qualification.test.ts index ee54568..9830762 100644 --- a/tests/release-qualification.test.ts +++ b/tests/release-qualification.test.ts @@ -2,11 +2,15 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { RELEASE_PASS_RATES } from "../evals/release-policy.js"; +import { campaignPlanSha256 } from "../evals/report.js"; import { SCENARIOS } from "../evals/scenarios.js"; import { + decisionRecordFor, mergeReports, providers, qualificationFailures, + qualifyV2, writeQualificationRecord, } from "../scripts/qualify-release.js"; @@ -32,6 +36,157 @@ const GATED = [ "unprovable-claim-refused", ]; +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; + +const V2_CATALOG = [ + { + caseId: "v2-case", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 1, + minScoredAttempts: 1, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, +]; + +const V2_ARTIFACT = { + packageVersion: "8.1.1", + sourceCommit: "commit", + sourceTreeSha256: digest("a"), + tarballSha256: digest("b"), + unpackedManifestSha256: digest("c"), +}; + +function v2Report(stopped = false) { + const model = { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "test", + revision: null, + }; + const plan = { + schemaVersion: 1 as const, + planId: "v2-plan", + planSha256: digest("d"), + randomizationSeed: "seed", + cells: [ + { + cellId: "cell", + blockId: "block", + caseId: "v2-case", + caseVersion: 1, + armToken: null, + repetition: 0, + managerModel: model, + reviewerModel: null, + schedule: "primary" as const, + }, + ], + abortPolicy: { retry: "never" as const, maxReplacementBlocks: 0 }, + stoppingRule: { kind: "fixed-attempts" as const, count: 1 }, + analysis: { + kind: "rate" as const, + primaryOutcome: "pass", + versionSha256: digest("e"), + }, + budget: { + maxUsd: 1, + unknownCostPolicy: "stop" as const, + maxOutputTokens: 10, + maxWallClockMs: 10_000, + maxAttempts: 1, + }, + }; + plan.planSha256 = campaignPlanSha256(plan); + return { + schemaVersion: 2 as const, + reportId: stopped ? "v2-stopped" : "v2-verified", + plan, + attempts: [ + { + schemaVersion: 2 as const, + attemptId: "attempt", + cellId: "cell", + blockId: "block", + caseId: "v2-case", + caseVersion: 1, + armToken: null, + repetition: 0, + artifact: V2_ARTIFACT, + evaluator: { + sourceCommit: "evaluator", + caseCatalogSha256: digest("f"), + policyCatalogSha256: digest("0"), + graderBundleSha256: digest("1"), + }, + hostConfigSha256: digest("2"), + actors: stopped + ? [] + : [ + { + role: "manager" as const, + requestedModel: model, + actualModel: { kind: "observed" as const, value: model }, + sessionIds: ["session"], + }, + ], + instructions: stopped + ? [] + : [ + { + source: "command" as const, + name: "eval", + sequence: 0, + sha256: digest("3"), + bytes: 1, + }, + ], + transcript: stopped + ? null + : { sha256: digest("4"), artifact: "attempt.json" }, + outcome: stopped + ? { + kind: "failure" as const, + origin: "host" as const, + code: "down", + retryable: true, + } + : { + kind: "product" as const, + passed: true, + endedBy: "quiet" as const, + issues: [], + evidence: { + kind: "conformance" as const, + falseCompletion: false, + unsubmittedReviews: 0, + facts: {}, + }, + }, + usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, + }, + ], + completion: { + status: stopped ? ("stopped" as const) : ("complete" as const), + cause: stopped ? ("host" as const) : ("fixed-target" as const), + startedAt: "2026-08-25T00:00:00.000Z", + finishedAt: "2026-08-25T00:00:01.000Z", + activatedReserveCellIds: [], + observed: { + attempts: 1, + outputTokens: 1, + costUsd: 0, + wallClockMs: 1_000, + }, + }, + allocationCommitmentSha256: null, + }; +} + function report(overrides: { models?: string[]; scenarios?: string[]; @@ -80,6 +235,9 @@ function report(overrides: { } describe("release qualification", () => { + test("shares the v2 required-case policy with the live runner", () => { + expect(Object.keys(RELEASE_PASS_RATES).sort()).toEqual([...GATED].sort()); + }); test("qualifies a clean two-provider report", () => { expect(qualificationFailures(report({}))).toEqual([]); }); @@ -457,3 +615,66 @@ describe("qualification records", () => { ).rejects.toThrow(/requires the report's flowVersion/); }); }); + +describe("v2 qualification cutover", () => { + test("derives and records all three decision verdicts from explicit atomic inputs", () => { + const verified = qualifyV2({ + reportInput: v2Report(), + catalogInput: V2_CATALOG, + artifact: V2_ARTIFACT, + }); + expect(verified.decision.verdict).toBe("VERIFIED"); + const first = decisionRecordFor(verified); + expect(first.verdict).toBe("VERIFIED"); + expect(first).toEqual(decisionRecordFor(verified)); + expect(first).toMatchObject({ + reportSha256: expect.stringMatching(/^sha256:/), + artifactSha256: expect.stringMatching(/^sha256:/), + evaluatorSha256: expect.stringMatching(/^sha256:/), + policySha256: expect.stringMatching(/^sha256:/), + actorSha256: expect.stringMatching(/^sha256:/), + analyzerSha256: expect.stringMatching(/^sha256:/), + expectedProvenanceSha256: expect.stringMatching(/^sha256:/), + decisionInputSha256: expect.stringMatching(/^sha256:/), + }); + + const notVerified = qualifyV2({ + reportInput: v2Report(), + catalogInput: V2_CATALOG, + artifact: { ...V2_ARTIFACT, tarballSha256: digest("9") }, + }); + expect(notVerified.decision.verdict).toBe("NOT VERIFIED"); + + const inconclusive = qualifyV2({ + reportInput: v2Report(true), + catalogInput: V2_CATALOG, + artifact: V2_ARTIFACT, + }); + expect(inconclusive.decision.verdict).toBe("INCONCLUSIVE"); + }); + + test("rejects legacy summary-only input rather than converting it", () => { + expect(() => + qualifyV2({ + reportInput: report({}), + catalogInput: V2_CATALOG, + artifact: V2_ARTIFACT, + }), + ).toThrow("Invalid v2 report"); + }); + + test("requires explicit report, catalog, and artifact paths in the CLI", async () => { + const process = Bun.spawn(["bun", "run", "scripts/qualify-release.ts"], { + cwd: new URL("..", import.meta.url).pathname, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(process.stdout).text(), + new Response(process.stderr).text(), + process.exited, + ]); + expect(exitCode).not.toBe(0); + expect(`${stdout}${stderr}`).toContain("Usage: bun run qualify"); + }); +}); From fc62d8927477702cea9acb8badfdeb8f42e99191 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 06:55:10 +0200 Subject: [PATCH 2/2] Record Phase 5 cutover evidence --- .../evidence/phase-5-review.md | 23 +++++++++++++++++++ .../phase-5-qualification-cutover.md | 2 ++ .audit/eval-engineering.tsv | 4 ++++ 3 files changed, 29 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-5-review.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-5-review.md b/.agents/plans/02-eval-engineering/evidence/phase-5-review.md new file mode 100644 index 0000000..b54807d --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-5-review.md @@ -0,0 +1,23 @@ +# Phase 5 review and throughput checkpoint + +Phase 5 replaces release authority with explicit v2 report, catalog, and measured +artifact inputs. Every verdict writes a deterministic report-ID decision record. +Release scans committed records, validates the full digest set, rebuilds and +rechecks the exact artifact, and remains intentionally blocked by +`canary-not-enabled` until Phase 9. + +Interrogate fixed workflow flag/catalog/artifact mismatches, report-ID record +handoff, full artifact comparison, per-attempt host configuration, catalog hashing, +checksum ordering, and the missing required-case promotion. The final four-model +recheck found no blocker. + +Throughput checkpoint: + +- The live v2 catalog promotes exactly seven current release cases and leaves + uncalibrated cases report-only. +- A new scenario needs its scenario definition, release-policy entry when promoted, + and one policy/test update. It does not require a qualifier branch. +- The historical summary path remains as 39 symbol references across the legacy + helper tests and 743-line transitional qualifier module. It has no CLI or release + authority and can be deleted after migration evidence no longer needs comparison. +- The vertical slice passes the full gate. Later phases may proceed. diff --git a/.agents/plans/02-eval-engineering/phase-5-qualification-cutover.md b/.agents/plans/02-eval-engineering/phase-5-qualification-cutover.md index c2be927..c3bac3a 100644 --- a/.agents/plans/02-eval-engineering/phase-5-qualification-cutover.md +++ b/.agents/plans/02-eval-engineering/phase-5-qualification-cutover.md @@ -38,3 +38,5 @@ The old implicit newest-report command must fail with a usage error. Stop gate. Complete the throughput checkpoint in the overview before adding new evidence families. Publication must already be inside the new decision system, even though it remains disabled until Phase 9 adds canary evidence. + +Evidence. [Interrogate and throughput checkpoint](evidence/phase-5-review.md). diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index 093bb0b..3545026 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -41,3 +41,7 @@ ts phase decision why evidence result 2026-08-25T04:38:50Z phase-4 fixed the multi-model Interrogate findings transcript binding, cell-level publication, temporary cleanup, and terminal cause affected evidence integrity .agents/plans/02-eval-engineering/evidence/phase-4-review.md VERIFIED no unresolved blocker 2026-08-25T04:38:50Z phase-4 emitted and parsed a live v2 report the cutover cannot proceed on synthetic storage evidence alone .agents/plans/02-eval-engineering/evidence/phase-4-pilot.json VERIFIED one packed happy-path product attempt 2026-08-25T04:38:50Z phase-4 ran Deslop and the whole repository gate the phase must finish reviewable and regression-free bun run check VERIFIED 468 pass, 1 skip, 0 fail +2026-08-25T04:54:04Z phase-5 cut release authority over to explicit v2 inputs summary-only and implicit-newest reports cannot remain publication inputs scripts/qualify-release.ts; scripts/release-metadata.ts; .github/workflows/evals.yml; .github/workflows/release.yml VERIFIED explicit report, catalog, artifact, and decision paths +2026-08-25T04:54:04Z phase-5 fixed the multi-model Interrogate findings workflow contracts, record handoff, full artifact identity, catalog hashing, per-attempt host config, and required policies affected release integrity .agents/plans/02-eval-engineering/evidence/phase-5-review.md VERIFIED no unresolved blocker +2026-08-25T04:54:04Z phase-5 completed the throughput checkpoint the vertical slice must reduce maintenance load before new evidence families .agents/plans/02-eval-engineering/evidence/phase-5-review.md VERIFIED new scenario touches scenario, policy, and one test; legacy path has no authority +2026-08-25T04:54:04Z phase-5 ran Deslop, workflow lint, and the full repository gate the cutover must land atomically and regression-free actionlint; bun run check VERIFIED 473 pass, 1 skip, 0 fail