Cut release qualification over to v2 evidence

.agents/plans/02-eval-engineering/evidence/phase-5-review.md

@@ -0,0 +1,23 @@
+# Phase 5 review and throughput checkpoint
+
+Phase 5 replaces release authority with explicit v2 report, catalog, and measured
+artifact inputs. Every verdict writes a deterministic report-ID decision record.
+Release scans committed records, validates the full digest set, rebuilds and
+rechecks the exact artifact, and remains intentionally blocked by
+`canary-not-enabled` until Phase 9.
+
+Interrogate fixed workflow flag/catalog/artifact mismatches, report-ID record
+handoff, full artifact comparison, per-attempt host configuration, catalog hashing,
+checksum ordering, and the missing required-case promotion. The final four-model
+recheck found no blocker.
+
+Throughput checkpoint:
+
+- The live v2 catalog promotes exactly seven current release cases and leaves
+  uncalibrated cases report-only.
+- A new scenario needs its scenario definition, release-policy entry when promoted,
+  and one policy/test update. It does not require a qualifier branch.
+- The historical summary path remains as 39 symbol references across the legacy
+  helper tests and 743-line transitional qualifier module. It has no CLI or release
+  authority and can be deleted after migration evidence no longer needs comparison.
+- The vertical slice passes the full gate. Later phases may proceed.

.agents/plans/02-eval-engineering/phase-5-qualification-cutover.md

@@ -38,3 +38,5 @@ The old implicit newest-report command must fail with a usage error.
 Stop gate. Complete the throughput checkpoint in the overview before adding new
 evidence families. Publication must already be inside the new decision system,
 even though it remains disabled until Phase 9 adds canary evidence.
+
+Evidence. [Interrogate and throughput checkpoint](evidence/phase-5-review.md).

.audit/eval-engineering.tsv

@@ -41,3 +41,7 @@ ts	phase	decision	why	evidence	result
 2026-08-25T04:38:50Z	phase-4	fixed the multi-model Interrogate findings	transcript binding, cell-level publication, temporary cleanup, and terminal cause affected evidence integrity	.agents/plans/02-eval-engineering/evidence/phase-4-review.md	VERIFIED no unresolved blocker
 2026-08-25T04:38:50Z	phase-4	emitted and parsed a live v2 report	the cutover cannot proceed on synthetic storage evidence alone	.agents/plans/02-eval-engineering/evidence/phase-4-pilot.json	VERIFIED one packed happy-path product attempt
 2026-08-25T04:38:50Z	phase-4	ran Deslop and the whole repository gate	the phase must finish reviewable and regression-free	bun run check	VERIFIED 468 pass, 1 skip, 0 fail
+2026-08-25T04:54:04Z	phase-5	cut release authority over to explicit v2 inputs	summary-only and implicit-newest reports cannot remain publication inputs	scripts/qualify-release.ts; scripts/release-metadata.ts; .github/workflows/evals.yml; .github/workflows/release.yml	VERIFIED explicit report, catalog, artifact, and decision paths
+2026-08-25T04:54:04Z	phase-5	fixed the multi-model Interrogate findings	workflow contracts, record handoff, full artifact identity, catalog hashing, per-attempt host config, and required policies affected release integrity	.agents/plans/02-eval-engineering/evidence/phase-5-review.md	VERIFIED no unresolved blocker
+2026-08-25T04:54:04Z	phase-5	completed the throughput checkpoint	the vertical slice must reduce maintenance load before new evidence families	.agents/plans/02-eval-engineering/evidence/phase-5-review.md	VERIFIED new scenario touches scenario, policy, and one test; legacy path has no authority
+2026-08-25T04:54:04Z	phase-5	ran Deslop, workflow lint, and the full repository gate	the cutover must land atomically and regression-free	actionlint; bun run check	VERIFIED 473 pass, 1 skip, 0 fail

.github/workflows/evals.yml

@@ -83,6 +83,7 @@ jobs:
           fi
 
       - name: Run the eval matrix
+        id: run
         if: steps.matrix.outputs.models != ''
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -96,19 +97,74 @@ jobs:
         # A failing scenario must still publish its report: the durable documents are
         # the only way to read why, and paying for the pass twice is the alternative.
         continue-on-error: true
-        run: bun run eval -- --repeat "$REPEAT"
+        run: |
+          set -o pipefail
+          set +e
+          bun run eval -- --repeat "$REPEAT" 2>&1 | tee eval-v2.log
+          status=${PIPESTATUS[0]}
+          set -e
+          report="$(sed -n 's/^V2 report: //p' eval-v2.log | tail -n 1)"
+          if [ -z "$report" ] || [ ! -f "$report" ]; then
+            echo "::error::The evaluator did not print an existing V2 report path."
+            exit 1
+          fi
+          campaign_dir="$(dirname "$report")"
+          catalog="$campaign_dir/catalog.json"
+          artifact="$campaign_dir/artifact.tgz"
+          if [ ! -f "$catalog" ]; then
+            echo "::error::V2 campaign plan/catalog was not persisted at $catalog."
+            exit 1
+          fi
+          if [ ! -f "$artifact" ]; then
+            echo "::error::Measured artifact was not persisted at $artifact."
+            exit 1
+          fi
+          {
+            printf 'report=%s\n' "$report"
+            printf 'campaign=%s\n' "$campaign_dir"
+            printf 'catalog=%s\n' "$catalog"
+            printf 'artifact=%s\n' "$artifact"
+          } >> "$GITHUB_OUTPUT"
+          exit "$status"
 
       # Before qualification, so a NOT QUALIFIED verdict still leaves the report that
       # explains it. Publishing after the gate meant the one run worth reading was the
       # one whose artifact was skipped.
       - name: Publish the report
-        if: steps.matrix.outputs.models != ''
+        if: always() && steps.matrix.outputs.models != ''
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: eval-report
-          path: evals/results/
+          name: eval-v2-report
+          path: |
+            ${{ steps.run.outputs.report }}
+            ${{ steps.run.outputs.campaign }}
+            eval-v2.log
           retention-days: 90
 
       - name: Apply release-qualification thresholds
-        if: steps.matrix.outputs.models != ''
-        run: bun run qualify
+        id: qualify
+        if: always() && steps.matrix.outputs.models != '' && steps.run.outputs.report != ''
+        continue-on-error: true
+        run: |
+          set +e
+          mkdir -p evals/decisions
+          bun run qualify -- --report "${{ steps.run.outputs.report }}" --catalog "${{ steps.run.outputs.catalog }}" --artifact "${{ steps.run.outputs.artifact }}" --decisions-dir evals/decisions 2>&1 | tee qualification.log
+          status=${PIPESTATUS[0]}
+          decision_path="$(sed -n -E 's/^(VERIFIED|NOT VERIFIED|INCONCLUSIVE): (.*)$/\2/p' qualification.log | tail -n 1)"
+          if [ -z "$decision_path" ] || [ ! -f "$decision_path" ]; then
+            echo "::error::Qualifier did not print an existing decision path."
+            exit 1
+          fi
+          printf 'decision=%s\n' "$decision_path" >> "$GITHUB_OUTPUT"
+          exit "$status"
+
+      - name: Publish the decision artifacts
+        if: always() && steps.matrix.outputs.models != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: eval-v2-decision
+          path: |
+            qualification.log
+            ${{ steps.qualify.outputs.decision }}
+          if-no-files-found: warn
+          retention-days: 90

.github/workflows/release.yml

@@ -4,8 +4,6 @@ on:
   push:
     tags:
       - 'v*'
-    branches:
-      - main
 
 permissions:
   contents: write
@@ -30,17 +28,10 @@ jobs:
           tag="v${version}"
           echo "tag=${tag}" >> "$GITHUB_OUTPUT"
 
-          if [[ "${GITHUB_REF_TYPE}" == "tag" && "${GITHUB_REF_NAME}" != "${tag}" ]]; then
+          if [[ "${GITHUB_REF_NAME}" != "${tag}" ]]; then
             echo "::error::Release tag/version mismatch: tag=${GITHUB_REF_NAME}, package.json=${version}."
             exit 1
           fi
-
-          if [[ "${GITHUB_REF_TYPE}" == "branch" ]] && git ls-remote --exit-code --tags origin "refs/tags/${tag}" >/dev/null; then
-            echo "Tag ${tag} already exists; skipping release on main."
-            echo "publish=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
           echo "publish=true" >> "$GITHUB_OUTPUT"
 
   release:
@@ -86,9 +77,21 @@ jobs:
         run: |
           set -euo pipefail
           bun pm pack --destination .
+
+      - name: Verify exact VERIFIED V2 artifact decision
+        shell: bash
+        run: |
+          set -euo pipefail
           tarball="$(ls opencode-plugin-flow-*.tgz)"
+          bun run release:metadata -- --tag "${{ needs.decide.outputs.tag }}" --notes-file release-notes.md --artifact "$tarball"
           shasum -a 256 "$tarball" > "${tarball}.sha256"
 
+      # Temporary Phase 5 stop: the exact manual OpenCode canary is owned by Phase 9.
+      - name: Require Phase 9 canary
+        run: |
+          echo "::error::Release blocked: canary-not-enabled until Phase 9 release alignment is implemented."
+          exit 1
+
       - name: Publish to npm
         shell: bash
         run: |

evals/analysis.ts

@@ -29,13 +29,13 @@ export type ExpectedActorProvenance = {
 
 export type ExpectedAttemptProvenance = {
 	readonly cellId: string;
+	readonly hostConfigSha256: string;
 	readonly actors: readonly ExpectedActorProvenance[];
 	readonly instructions: readonly InstructionDelivery[];
 };
 
 type CommonExpectedProvenance = {
 	readonly evaluator: EvaluatorIdentity;
-	readonly hostConfigSha256: string;
 	readonly attempts: readonly ExpectedAttemptProvenance[];
 };
 
@@ -249,7 +249,7 @@ function compareCommonProvenance(
 				"Evaluator identity does not match expected provenance.",
 			);
 		}
-		if (attempt.hostConfigSha256 !== expected.hostConfigSha256) {
+		if (attempt.hostConfigSha256 !== expectedAttempt.hostConfigSha256) {
 			mismatch(
 				mismatches,
 				attempt.attemptId,

evals/release-policy.ts

@@ -0,0 +1,12 @@
+export const RELEASE_PASS_RATES: Readonly<Record<string, number>> = {
+	"happy-path": 1,
+	"plan-only-stops": 1,
+	"goal-change-refused": 1,
+	"failing-gate-blocks": 0.9,
+	"resumes-after-interruption": 1,
+	"unprovable-claim-refused": 0.9,
+	"continuation-accepted": 1,
+};
+
+export const RELEASE_MIN_PROVIDERS = 2;
+export const RELEASE_MIN_SCORED_ATTEMPTS = 3;

evals/report-store.ts

@@ -138,6 +138,8 @@ async function readJson(path: string): Promise<unknown> {
 export class ReportStore {
 	private readonly attemptsDirectory: string;
 	private readonly transcriptsDirectory: string;
+	private readonly catalogPath: string;
+	private readonly artifactPath: string;
 	private readonly planPath: string;
 	private readonly completionPath: string;
 	private readonly reportPath: string;
@@ -153,6 +155,8 @@ export class ReportStore {
 		this.hooks = hooks;
 		this.attemptsDirectory = join(directory, "attempts");
 		this.transcriptsDirectory = join(directory, "transcripts");
+		this.catalogPath = join(directory, "catalog.json");
+		this.artifactPath = join(directory, "artifact.tgz");
 		this.planPath = join(directory, "plan.json");
 		this.completionPath = join(directory, "completion.json");
 		this.reportPath = join(directory, "report.json");
@@ -167,6 +171,24 @@ export class ReportStore {
 		);
 	}
 
+	async writeCatalog(
+		catalog: ValidatedCaseCatalog,
+	): Promise<"written" | "replayed"> {
+		return writeImmutable(
+			this.catalogPath,
+			Buffer.from(canonicalJson(catalog)),
+			this.hooks,
+		);
+	}
+
+	async writeArtifact(artifactPath: string): Promise<"written" | "replayed"> {
+		return writeImmutable(
+			this.artifactPath,
+			await readFile(artifactPath),
+			this.hooks,
+		);
+	}
+
 	private async plan(): Promise<CampaignPlan> {
 		const parsed = CampaignPlanSchema.safeParse(await readJson(this.planPath));
 		if (!parsed.success) fail("Stored campaign plan is invalid.");

evals/run.ts

@@ -58,6 +58,11 @@ import {
 	redactTranscript,
 	tarballSha256,
 } from "./provenance.js";
+import {
+	RELEASE_MIN_PROVIDERS,
+	RELEASE_MIN_SCORED_ATTEMPTS,
+	RELEASE_PASS_RATES,
+} from "./release-policy.js";
 import type {
 	ActorIdentity,
 	ArtifactIdentity,
@@ -198,17 +203,24 @@ function caseCatalogFor(
 	scenarios: readonly (typeof SCENARIOS)[number][],
 ): ValidatedCaseCatalog {
 	const parsed = parseCaseCatalog(
-		scenarios.map((scenario) => ({
-			caseId: scenario.id,
-			caseVersion: 1,
-			evidenceClass: "conformance" as const,
-			oracle: "durable-state" as const,
-			release: "report-only" as const,
-			minProviders: 1,
-			minScoredAttempts: 1,
-			minPassRate: 1,
-			reviewerPromotionRecordSha256: null,
-		})),
+		scenarios.map((scenario) => {
+			const minPassRate = RELEASE_PASS_RATES[scenario.id] ?? null;
+			return {
+				caseId: scenario.id,
+				caseVersion: 1,
+				evidenceClass: "conformance" as const,
+				oracle: "durable-state" as const,
+				release:
+					minPassRate === null
+						? ("report-only" as const)
+						: ("required" as const),
+				minProviders: minPassRate === null ? 1 : RELEASE_MIN_PROVIDERS,
+				minScoredAttempts:
+					minPassRate === null ? 1 : RELEASE_MIN_SCORED_ATTEMPTS,
+				minPassRate,
+				reviewerPromotionRecordSha256: null,
+			};
+		}),
 	);
 	if (!parsed.ok) {
 		throw new Error(
@@ -616,6 +628,7 @@ async function main(): Promise<void> {
 		catalog: v2Catalog,
 	});
 	await reportStore.initialize(v2Plan);
+	await reportStore.writeCatalog(v2Catalog);
 	const campaignStartedAt = new Date().toISOString();
 	const campaignCells = v2Plan.cells;
 	const v2Attempts: AttemptRecordV2[] = [];
@@ -630,6 +643,7 @@ async function main(): Promise<void> {
 			repositoryRoot,
 			tarballPath: tarball,
 		});
+		await reportStore.writeArtifact(tarball);
 		const evaluator = evaluatorIdentity({
 			sourceCommit: artifact.sourceCommit,
 			caseCatalog: selected.map((scenario) => ({