From f1a8517f256c976349a7cadc607ec1cba5d9969f Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 09:16:08 +0200 Subject: [PATCH 1/2] feat(evals): add reviewer calibration campaign --- evals/canonical-json.ts | 5 +- evals/harness.ts | 5 + evals/reviewer-assignment.ts | 255 +++++++++++++++++ evals/reviewer-calibration.ts | 496 ++++++++++++++++++++++++++++++++++ evals/reviewer-cases.ts | 84 ++++++ evals/reviewer-run.ts | 414 ++++++++++++++++++++++++++++ tests/reviewer-eval.test.ts | 404 +++++++++++++++++++++++++++ tests/reviewer-run.test.ts | 157 +++++++++++ 8 files changed, 1819 insertions(+), 1 deletion(-) create mode 100644 evals/reviewer-assignment.ts create mode 100644 evals/reviewer-calibration.ts create mode 100644 evals/reviewer-cases.ts create mode 100644 evals/reviewer-run.ts create mode 100644 tests/reviewer-eval.test.ts create mode 100644 tests/reviewer-run.test.ts diff --git a/evals/canonical-json.ts b/evals/canonical-json.ts index 8ad22df..bce0d14 100644 --- a/evals/canonical-json.ts +++ b/evals/canonical-json.ts @@ -46,7 +46,10 @@ export function canonicalJson(value: unknown): string { throw new Error("Canonical JSON requires JSON values."); } -export function canonicalSha256(domain: string, value: unknown): string { +export function canonicalSha256( + domain: string, + value: unknown, +): `sha256:${string}` { return `sha256:${createHash("sha256") .update(`${domain}\u0000`) .update(canonicalJson(value)) diff --git a/evals/harness.ts b/evals/harness.ts index 8fbfb3c..130a4cb 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -982,6 +982,8 @@ export class EvalHost { packageCache: string; opencodeVersion: string; files: Readonly>; + /** Pins the hidden reviewer child independently from the command model. */ + reviewerModel?: string; /** False creates the paired benchmark's ordinary OpenCode control host. */ withFlow?: boolean; }): Promise { @@ -1055,6 +1057,9 @@ export class EvalHost { cwd: project, env: { ...process.env, + ...(options.reviewerModel + ? { OPENCODE_FLOW_REVIEWER_MODEL: options.reviewerModel } + : {}), HOME: childHome, XDG_CACHE_HOME: childCache, XDG_CONFIG_HOME: join(childHome, ".config"), diff --git a/evals/reviewer-assignment.ts b/evals/reviewer-assignment.ts new file mode 100644 index 0000000..54e135e --- /dev/null +++ b/evals/reviewer-assignment.ts @@ -0,0 +1,255 @@ +import type { ReviewFinding, Session } from "../src/domain/session.js"; +import { observeAssertions } from "../src/domain/test-results.js"; +import { normalizeEvidencePlatform } from "../src/domain/validation.js"; +import { createFileSessionRepository } from "../src/infrastructure/fs/session-repository.js"; +import { + flowPlanApprove, + flowPlanSave, + flowReviewStart, + flowRunStart, +} from "../src/infrastructure/fs/workspace-flow-service.js"; +import { + persistWorkspaceValidation, + prepareWorkspaceValidation, + readWorkspaceTestReport, +} from "../src/infrastructure/fs/workspace-validation.js"; +import { canonicalSha256 } from "./canonical-json.js"; +import { + assertReviewerCaseTruth, + type ReviewerCase, +} from "./reviewer-cases.js"; + +const FEATURE_ID = "review-target"; +const RESULTS_PATH = ".flow/reviewer-results.xml"; +const VALIDATION_COMMAND = `bun test --reporter=junit --reporter-outfile=${RESULTS_PATH}`; + +export type SeededReviewerAssignment = { + readonly flowSessionId: string; + readonly featureId: string; + readonly runId: string; + readonly assignmentId: string; +}; + +export type DurableReviewerSubmission = + | { readonly kind: "unsubmitted" } + | { + readonly kind: "submitted"; + readonly verdict: "passed" | "failed"; + readonly findings: readonly ReviewFinding[]; + }; + +function requireFlowSuccess(response: { + readonly status: "ok" | "error"; + readonly summary: string; +}): void { + if (response.status === "error") throw new Error(response.summary); +} + +async function currentSession(workspace: string): Promise { + const session = await createFileSessionRepository(workspace).read(); + if (!session) throw new Error("Reviewer assignment setup lost Flow state."); + return session; +} + +async function verifySeedValidation(workspace: string): Promise<{ + readonly outputDigest: `sha256:${string}`; + readonly report: string; +}> { + const process = Bun.spawn( + ["bun", "test", "--reporter=junit", `--reporter-outfile=${RESULTS_PATH}`], + { + cwd: workspace, + stdout: "pipe", + stderr: "pipe", + }, + ); + const [exitCode, stdout, stderr] = await Promise.all([ + process.exited, + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]); + if (exitCode !== 0) { + throw new Error(`Reviewer seed validation failed: ${stderr || stdout}`); + } + const report = await readWorkspaceTestReport(workspace, RESULTS_PATH); + if (!report) + throw new Error("Reviewer seed validation wrote no JUnit report."); + return { + outputDigest: canonicalSha256("flow-reviewer-seed-validation-v1", { + command: VALIDATION_COMMAND, + exitCode, + stdout, + stderr, + report: report.text, + }), + report: report.text, + }; +} + +/** Creates a real pending Flow assignment without involving a manager model. */ +export async function seedReviewerAssignment(input: { + readonly workspace: string; + readonly fixture: ReviewerCase; +}): Promise { + assertReviewerCaseTruth(input.fixture); + const operationSuffix = `${input.fixture.caseId}-v${input.fixture.caseVersion}`; + requireFlowSuccess( + await flowPlanSave(input.workspace, { + request: { + operationId: `reviewer-plan-save-${operationSuffix}`, + expectedRevision: 0, + goal: "Verify the current value implementation.", + plan: { + summary: "Verify the value implementation.", + overview: + "Inspect the implementation independently before completion.", + requirements: [ + "For every safe-integer input below Number.MAX_SAFE_INTEGER, value returns input plus one.", + ], + decisions: [ + "Use the exported value function as the public boundary.", + ], + evidence: [ + { + scope: "gate", + requirement: "Behavioral correctness", + environment: "isolated evaluator workspace", + command: VALIDATION_COMMAND, + platform: normalizeEvidencePlatform(process.platform), + assertions: ["value returns the next safe integer"], + }, + ], + features: [ + { + id: FEATURE_ID, + kind: "inspect", + title: "Review the value implementation", + summary: + "Verify the value function against the approved requirement.", + targets: Object.keys(input.fixture.files).sort(), + validation: [VALIDATION_COMMAND], + dependsOn: [], + }, + ], + }, + }, + }), + ); + requireFlowSuccess( + await flowPlanApprove(input.workspace, { + request: { + operationId: `reviewer-plan-approve-${operationSuffix}`, + expectedRevision: (await currentSession(input.workspace)).revision, + }, + }), + ); + requireFlowSuccess( + await flowRunStart(input.workspace, { + request: { + operationId: `reviewer-run-start-${operationSuffix}`, + expectedRevision: (await currentSession(input.workspace)).revision, + featureId: FEATURE_ID, + }, + }), + ); + const prepared = await prepareWorkspaceValidation(input.workspace, { + expectedRevision: (await currentSession(input.workspace)).revision, + featureId: FEATURE_ID, + command: VALIDATION_COMMAND, + scope: "broad", + resultsPath: RESULTS_PATH, + }); + const validation = await verifySeedValidation(input.workspace); + await persistWorkspaceValidation(input.workspace, { + ...prepared, + captureId: `reviewer-validation-${operationSuffix}`, + exitCode: 0, + outputDigest: validation.outputDigest, + outputComplete: true, + hostPlatform: normalizeEvidencePlatform(process.platform), + observedAssertions: observeAssertions( + prepared.assertions, + validation.report, + ), + }); + requireFlowSuccess( + await flowReviewStart(input.workspace, { + request: { + operationId: `reviewer-review-start-${operationSuffix}`, + expectedRevision: (await currentSession(input.workspace)).revision, + featureId: FEATURE_ID, + artifactsChanged: Object.keys(input.fixture.files) + .sort() + .map((path) => ({ path })), + packet: { + summary: + "Verify the value function against the approved requirement. Baseline inventory: src/value.ts and src/value.test.ts are tracked regular non-executable files; there are no source deletions, renames, generated artifacts, symlinks, or file-mode changes. Host-owned .flow and .opencode files are outside the source change.", + riskLenses: ["functional correctness", "boundary behavior"], + }, + }, + }), + ); + const session = await currentSession(input.workspace); + const run = session.runs.find( + (candidate) => + candidate.featureId === FEATURE_ID && candidate.state === "active", + ); + const assignment = run?.reviews.find((review) => review.result === null); + if (!run || !assignment) { + throw new Error( + "Reviewer assignment setup did not persist a pending review.", + ); + } + return { + flowSessionId: session.id, + featureId: FEATURE_ID, + runId: run.id, + assignmentId: assignment.id, + }; +} + +export function durableReviewerSubmission(input: { + readonly session: Session | null; + readonly seed: SeededReviewerAssignment; +}): DurableReviewerSubmission { + const run = input.session?.runs.find( + (candidate) => candidate.id === input.seed.runId, + ); + const assignment = run?.reviews.find( + (candidate) => candidate.id === input.seed.assignmentId, + ); + if (!assignment) { + throw new Error( + "The seeded reviewer assignment disappeared from Flow state.", + ); + } + if ( + assignment.result === null || + assignment.result.terminalDisposition !== "submitted" + ) { + return { kind: "unsubmitted" }; + } + return { + kind: "submitted", + verdict: assignment.result.verdict, + findings: assignment.result.findings, + }; +} + +export async function readDurableReviewerSubmission(input: { + readonly workspace: string; + readonly seed: SeededReviewerAssignment; +}): Promise { + const repository = createFileSessionRepository(input.workspace); + const active = await repository.read(); + if (active?.runs.some((run) => run.id === input.seed.runId)) { + return durableReviewerSubmission({ session: active, seed: input.seed }); + } + const archived = await repository.transact((transaction) => + transaction.loadArchive(input.seed.flowSessionId), + ); + return durableReviewerSubmission({ + session: archived, + seed: input.seed, + }); +} diff --git a/evals/reviewer-calibration.ts b/evals/reviewer-calibration.ts new file mode 100644 index 0000000..be8b062 --- /dev/null +++ b/evals/reviewer-calibration.ts @@ -0,0 +1,496 @@ +import { z } from "zod"; +import { canonicalSha256 } from "./canonical-json.js"; +import type { ValidatedCaseCatalog } from "./catalog.js"; +import type { + ArtifactIdentity, + ModelIdentity, + ValidatedReport, +} from "./report.js"; +import { + type HumanLabel, + REVIEWER_CASES, + type ReviewerTruth, +} from "./reviewer-cases.js"; + +export type ReviewerObservation = { + readonly caseId: string; + readonly caseVersion: number; + readonly truth: ReviewerTruth; + readonly verdict: "passed" | "failed" | null; + readonly submitted: boolean; +}; + +export type WilsonInterval = readonly [number, number]; + +export type ReviewerConfusionMatrix = { + readonly truePositives: number; + readonly falseNegatives: number; + readonly falsePositives: number; + readonly trueNegatives: number; + readonly unsubmitted: number; +}; + +export type ReviewerCalibrationAnalysis = { + readonly matrix: ReviewerConfusionMatrix; + readonly defectCases: number; + readonly cleanCases: number; + readonly detectionRate: number | null; + readonly detectionInterval95: WilsonInterval | null; + readonly falsePositiveRate: number | null; + readonly falsePositiveInterval95: WilsonInterval | null; +}; + +export type LabelAssignment = HumanLabel & { + readonly caseId: string; + readonly caseVersion: number; +}; + +export type ReviewerPromotionRecord = { + readonly schemaVersion: 1; + readonly planSha256: string; + readonly calibrationReportSha256: string; + readonly caseCatalogSha256: string; + readonly humanLabelsSha256: string; + readonly artifactSha256: string; + readonly reviewerModels: readonly ModelIdentity[]; + readonly defectCases: number; + readonly cleanCases: number; + readonly ratersPerCase: number; + readonly agreement: { + readonly method: "krippendorff-alpha"; + readonly value: number; + readonly minimum: number; + }; + readonly observed: { + readonly detectionRate: number; + readonly detectionInterval95: WilsonInterval; + readonly falsePositiveRate: number; + readonly falsePositiveInterval95: WilsonInterval; + }; + readonly minimumDetectionRate: number; + readonly maximumFalsePositiveRate: number; + readonly recordedAt: string; +}; + +const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const TextSchema = z.string().min(1).max(4096).regex(/\S/); +const ModelIdentitySchema = z + .object({ + routeProvider: TextSchema, + gateway: TextSchema.nullable(), + family: TextSchema, + model: TextSchema, + revision: TextSchema.nullable(), + }) + .strict(); +export const ReviewerPromotionRecordSchema = z + .object({ + schemaVersion: z.literal(1), + planSha256: DigestSchema, + calibrationReportSha256: DigestSchema, + caseCatalogSha256: DigestSchema, + humanLabelsSha256: DigestSchema, + artifactSha256: DigestSchema, + reviewerModels: z.array(ModelIdentitySchema).min(1), + defectCases: z.number().int().positive(), + cleanCases: z.number().int().positive(), + ratersPerCase: z.number().int().min(2), + agreement: z + .object({ + method: z.literal("krippendorff-alpha"), + value: z.number().finite().min(-1).max(1), + minimum: z.number().finite().min(0).max(1), + }) + .strict(), + observed: z + .object({ + detectionRate: z.number().finite().min(0).max(1), + detectionInterval95: z.tuple([ + z.number().min(0).max(1), + z.number().min(0).max(1), + ]), + falsePositiveRate: z.number().finite().min(0).max(1), + falsePositiveInterval95: z.tuple([ + z.number().min(0).max(1), + z.number().min(0).max(1), + ]), + }) + .strict(), + minimumDetectionRate: z.number().finite().min(0).max(1), + maximumFalsePositiveRate: z.number().finite().min(0).max(1), + recordedAt: z.string().datetime({ offset: true }), + }) + .strict() + .superRefine((record, context) => { + for (const [name, interval] of [ + ["detectionInterval95", record.observed.detectionInterval95], + ["falsePositiveInterval95", record.observed.falsePositiveInterval95], + ] as const) { + if (interval[0] > interval[1]) { + context.addIssue({ + code: "custom", + path: ["observed", name], + message: "Interval lower bound must not exceed its upper bound.", + }); + } + } + if (record.agreement.value < record.agreement.minimum) { + context.addIssue({ + code: "custom", + message: "Agreement misses its promotion minimum.", + }); + } + if (record.observed.detectionInterval95[0] < record.minimumDetectionRate) { + context.addIssue({ + code: "custom", + message: "Detection lower bound misses its promotion minimum.", + }); + } + if ( + record.observed.falsePositiveInterval95[1] > + record.maximumFalsePositiveRate + ) { + context.addIssue({ + code: "custom", + message: "False-positive upper bound exceeds its promotion maximum.", + }); + } + }); + +function wilson(successes: number, total: number): WilsonInterval | null { + if (total === 0) return null; + const zValue = 1.959963984540054; + const rate = successes / total; + const square = zValue * zValue; + const denominator = 1 + square / total; + const center = (rate + square / (2 * total)) / denominator; + const margin = + (zValue / denominator) * + Math.sqrt((rate * (1 - rate) + square / (4 * total)) / total); + return [center - margin, center + margin]; +} + +export function analyzeReviewerCalibration( + observations: readonly ReviewerObservation[], +): ReviewerCalibrationAnalysis { + let truePositives = 0; + let falseNegatives = 0; + let falsePositives = 0; + let trueNegatives = 0; + let unsubmitted = 0; + for (const observation of observations) { + if (!observation.submitted || observation.verdict === null) { + unsubmitted += 1; + continue; + } + if (observation.truth === "defect") { + if (observation.verdict === "failed") truePositives += 1; + else falseNegatives += 1; + } else if (observation.verdict === "failed") falsePositives += 1; + else trueNegatives += 1; + } + const defectCases = truePositives + falseNegatives; + const cleanCases = falsePositives + trueNegatives; + return { + matrix: { + truePositives, + falseNegatives, + falsePositives, + trueNegatives, + unsubmitted, + }, + defectCases, + cleanCases, + detectionRate: defectCases === 0 ? null : truePositives / defectCases, + detectionInterval95: wilson(truePositives, defectCases), + falsePositiveRate: cleanCases === 0 ? null : falsePositives / cleanCases, + falsePositiveInterval95: wilson(falsePositives, cleanCases), + }; +} + +export function krippendorffNominalAlpha( + labels: readonly LabelAssignment[], +): number | null { + const byCase = new Map(); + for (const label of labels) { + const key = `${label.caseId}\u0000${label.caseVersion}`; + const values = byCase.get(key) ?? []; + values.push(label); + byCase.set(key, values); + } + let observedPairs = 0; + let disagreements = 0; + const categories = new Map(); + let total = 0; + for (const values of byCase.values()) { + for (const value of values) { + categories.set(value.truth, (categories.get(value.truth) ?? 0) + 1); + total += 1; + } + for (const left of values) { + for (const right of values) { + if (left.raterId === right.raterId) continue; + observedPairs += 1; + if (left.truth !== right.truth) disagreements += 1; + } + } + } + if (observedPairs === 0 || total < 2) return null; + const expectedAgreement = + [...categories.values()].reduce( + (sum, count) => sum + count * (count - 1), + 0, + ) / + (total * (total - 1)); + const expectedDisagreement = 1 - expectedAgreement; + if (expectedDisagreement === 0) return 1; + return 1 - disagreements / observedPairs / expectedDisagreement; +} + +function reviewerObservations( + report: ValidatedReport, +): readonly ReviewerObservation[] { + return report.attempts.flatMap((attempt) => + attempt.outcome.kind === "product" && + attempt.outcome.evidence.kind === "reviewer-only" + ? [ + { + caseId: attempt.caseId, + caseVersion: attempt.caseVersion, + truth: attempt.outcome.evidence.truth, + verdict: attempt.outcome.evidence.verdict, + submitted: attempt.outcome.evidence.submitted, + }, + ] + : [], + ); +} + +function orderedLabels( + labels: readonly LabelAssignment[], +): readonly LabelAssignment[] { + return [...labels].sort((left, right) => { + const leftKey = [ + left.caseId, + String(left.caseVersion), + left.raterId, + left.truth, + ].join("\u0000"); + const rightKey = [ + right.caseId, + String(right.caseVersion), + right.raterId, + right.truth, + ].join("\u0000"); + return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0; + }); +} + +export function createReviewerPromotion(input: { + readonly mode: "pilot" | "promotion"; + readonly report: ValidatedReport; + readonly catalog: ValidatedCaseCatalog; + readonly labels: readonly LabelAssignment[]; + readonly artifact: ArtifactIdentity; + readonly reviewerModels: readonly ModelIdentity[]; + readonly minimumDetectionRate: number; + readonly maximumFalsePositiveRate: number; + readonly minimumCasesPerTruth: number; + readonly recordedAt: string; +}): + | { + readonly kind: "advisory"; + readonly analysis: ReviewerCalibrationAnalysis; + readonly reasons: readonly string[]; + } + | { readonly kind: "promotion"; readonly record: ReviewerPromotionRecord } { + const observations = reviewerObservations(input.report); + const analysis = analyzeReviewerCalibration(observations); + const alpha = krippendorffNominalAlpha(input.labels); + const labelsByCase = new Map>(); + const labelAssignments = new Set(); + let duplicateRater = false; + for (const label of input.labels) { + const key = `${label.caseId}\u0000${label.caseVersion}`; + const assignment = `${key}\u0000${label.raterId}`; + if (labelAssignments.has(assignment)) duplicateRater = true; + labelAssignments.add(assignment); + const raters = labelsByCase.get(key) ?? new Set(); + raters.add(label.raterId); + labelsByCase.set(key, raters); + } + const minimumRaters = + labelsByCase.size === 0 + ? 0 + : Math.min(...[...labelsByCase.values()].map((raters) => raters.size)); + const plannedCases = new Set( + input.report.plan.cells.map( + (cell) => `${cell.caseId}\u0000${cell.caseVersion}`, + ), + ); + const labelsMatchPlan = + plannedCases.size === labelsByCase.size && + [...plannedCases].every((key) => labelsByCase.has(key)); + const labelsMatchTruth = observations.every((observation) => { + const key = `${observation.caseId}\u0000${observation.caseVersion}`; + return input.labels + .filter((label) => `${label.caseId}\u0000${label.caseVersion}` === key) + .every((label) => label.truth === observation.truth); + }); + const registeredLabels = REVIEWER_CASES.filter((entry) => + plannedCases.has(`${entry.caseId}\u0000${entry.caseVersion}`), + ).flatMap((entry) => + entry.humanLabels.map((label) => ({ + ...label, + caseId: entry.caseId, + caseVersion: entry.caseVersion, + })), + ); + const labelsMatchRegistry = + canonicalSha256( + "flow-reviewer-human-labels-v1", + orderedLabels(input.labels), + ) === + canonicalSha256( + "flow-reviewer-human-labels-v1", + orderedLabels(registeredLabels), + ); + const observedReviewerModels = input.report.attempts.flatMap((attempt) => + attempt.outcome.kind === "product" + ? attempt.actors.flatMap((actor) => + actor.role === "reviewer" && actor.actualModel.kind === "observed" + ? [actor.actualModel.value] + : [], + ) + : [], + ); + const reviewerIdentityComplete = + observedReviewerModels.length === observations.length && + input.reviewerModels.length > 0 && + observedReviewerModels.every((observed) => + input.reviewerModels.some( + (expected) => + canonicalSha256("flow-reviewer-model-v1", observed) === + canonicalSha256("flow-reviewer-model-v1", expected), + ), + ); + const reasons = [ + ...(input.mode === "pilot" + ? ["Pilot runs are advisory by definition."] + : []), + ...(input.report.plan.analysis.kind !== "reviewer" + ? ["Calibration requires a reviewer campaign plan."] + : []), + ...(input.report.completion.status !== "complete" || + input.report.completion.cause !== "fixed-target" + ? ["Calibration requires complete fixed-target execution."] + : []), + ...(analysis.matrix.unsubmitted > 0 + ? ["Calibration cannot contain unsubmitted reviewer outcomes."] + : []), + ...(analysis.defectCases < input.minimumCasesPerTruth || + analysis.cleanCases < input.minimumCasesPerTruth + ? ["Both truth classes need the preregistered sample size."] + : []), + ...(minimumRaters < 2 + ? ["Every calibration case needs at least two immutable human labels."] + : []), + ...(duplicateRater + ? ["A calibration case cannot reuse one human rater."] + : []), + ...(!labelsMatchPlan + ? ["Human labels must name exactly the frozen calibration cases."] + : []), + ...(!labelsMatchTruth + ? ["Human labels disagree with the executable fixed truth."] + : []), + ...(!labelsMatchRegistry + ? ["Human labels do not match the preregistered immutable labels."] + : []), + ...(alpha === null || alpha < 0.8 + ? ["Krippendorff nominal alpha is below 0.8."] + : []), + ...(analysis.detectionInterval95 === null || + analysis.detectionInterval95[0] < input.minimumDetectionRate + ? ["Detection lower bound misses its threshold."] + : []), + ...(analysis.falsePositiveInterval95 === null || + analysis.falsePositiveInterval95[1] > input.maximumFalsePositiveRate + ? ["False-positive upper bound exceeds its threshold."] + : []), + ...(!reviewerIdentityComplete + ? ["Every calibration outcome requires an observed reviewer identity."] + : []), + ...(input.report.attempts.some( + (attempt) => + canonicalSha256("flow-reviewer-artifact-v1", attempt.artifact) !== + canonicalSha256("flow-reviewer-artifact-v1", input.artifact), + ) + ? ["Calibration artifact does not exactly match every attempt."] + : []), + ]; + if (reasons.length > 0) return { kind: "advisory", analysis, reasons }; + const detectionInterval95 = analysis.detectionInterval95; + const falsePositiveInterval95 = analysis.falsePositiveInterval95; + const detectionRate = analysis.detectionRate; + const falsePositiveRate = analysis.falsePositiveRate; + if ( + !detectionInterval95 || + !falsePositiveInterval95 || + detectionRate === null || + falsePositiveRate === null || + alpha === null + ) { + return { + kind: "advisory", + analysis, + reasons: ["Calibration lacks a finite statistic."], + }; + } + const record = { + schemaVersion: 1 as const, + planSha256: input.report.plan.planSha256, + calibrationReportSha256: canonicalSha256( + "flow-reviewer-calibration-report-v1", + input.report, + ), + caseCatalogSha256: canonicalSha256( + "flow-reviewer-calibration-catalog-v1", + input.catalog, + ), + humanLabelsSha256: canonicalSha256( + "flow-reviewer-human-labels-v1", + orderedLabels(input.labels), + ), + artifactSha256: canonicalSha256( + "flow-reviewer-artifact-v1", + input.artifact, + ), + reviewerModels: input.reviewerModels, + defectCases: analysis.defectCases, + cleanCases: analysis.cleanCases, + ratersPerCase: minimumRaters, + agreement: { + method: "krippendorff-alpha" as const, + value: alpha, + minimum: 0.8, + }, + observed: { + detectionRate, + detectionInterval95, + falsePositiveRate, + falsePositiveInterval95, + }, + minimumDetectionRate: input.minimumDetectionRate, + maximumFalsePositiveRate: input.maximumFalsePositiveRate, + recordedAt: input.recordedAt, + }; + const parsed = ReviewerPromotionRecordSchema.safeParse(record); + if (!parsed.success) + return { + kind: "advisory", + analysis, + reasons: ["Promotion record failed strict validation."], + }; + return { kind: "promotion", record }; +} diff --git a/evals/reviewer-cases.ts b/evals/reviewer-cases.ts new file mode 100644 index 0000000..1457fdb --- /dev/null +++ b/evals/reviewer-cases.ts @@ -0,0 +1,84 @@ +import { canonicalSha256 } from "./canonical-json.js"; + +export type ReviewerTruth = "defect" | "clean"; + +export type HumanLabel = { + readonly raterId: string; + readonly truth: ReviewerTruth; +}; + +export type ReviewerCase = { + readonly caseId: string; + readonly caseVersion: 1; + readonly truth: ReviewerTruth; + readonly files: Readonly>; + readonly humanLabels: readonly [HumanLabel, HumanLabel]; + readonly truthSha256: string; +}; + +const DEFECT_LABELS: readonly [HumanLabel, HumanLabel] = [ + Object.freeze({ raterId: "labeler-a", truth: "defect" }), + Object.freeze({ raterId: "labeler-b", truth: "defect" }), +]; +const CLEAN_LABELS: readonly [HumanLabel, HumanLabel] = [ + Object.freeze({ raterId: "labeler-a", truth: "clean" }), + Object.freeze({ raterId: "labeler-b", truth: "clean" }), +]; + +const TEST_FILE = + 'import { expect, test } from "bun:test";\nimport { value } from "./value";\n\nconst examples = [Number.MIN_SAFE_INTEGER, -5, 0, 2, 100, Number.MAX_SAFE_INTEGER - 1];\n\ntest("value returns the next safe integer", () => {\n\tfor (const input of examples) expect(value(input)).toBe(input + 1);\n});\n'; +const DEFECT_FILES = Object.freeze({ + "src/value.ts": + "export function value(input: number): number {\n\treturn input === 7 ? input - 1 : input + 1;\n}\n", + "src/value.test.ts": TEST_FILE, +}); +const CLEAN_FILES = Object.freeze({ + "src/value.ts": + "export function value(input: number): number {\n\treturn input + 1;\n}\n", + "src/value.test.ts": TEST_FILE, +}); + +/** Opaque fixed-label cases; truth and fixture purpose never enter the prompt or path. */ +export const REVIEWER_CASES: readonly ReviewerCase[] = [ + Object.freeze({ + caseId: "review-case-a1", + caseVersion: 1, + truth: "defect", + files: DEFECT_FILES, + humanLabels: DEFECT_LABELS, + truthSha256: canonicalSha256("flow-reviewer-case-truth-v1", DEFECT_FILES), + }), + Object.freeze({ + caseId: "review-case-b7", + caseVersion: 1, + truth: "clean", + files: CLEAN_FILES, + humanLabels: CLEAN_LABELS, + truthSha256: canonicalSha256("flow-reviewer-case-truth-v1", CLEAN_FILES), + }), +]; + +export function assertReviewerCaseTruth(input: { + readonly caseId: string; + readonly caseVersion: number; + readonly files: Readonly>; +}): ReviewerTruth { + const fixture = REVIEWER_CASES.find( + (candidate) => + candidate.caseId === input.caseId && + candidate.caseVersion === input.caseVersion, + ); + if (!fixture) { + throw new Error( + "Reviewer fixture drifted from its fixed executable truth.", + ); + } + if ( + canonicalSha256("flow-reviewer-case-truth-v1", input.files) !== + fixture.truthSha256 + ) + throw new Error( + "Reviewer fixture drifted from its fixed executable truth.", + ); + return fixture.truth; +} diff --git a/evals/reviewer-run.ts b/evals/reviewer-run.ts new file mode 100644 index 0000000..9fae739 --- /dev/null +++ b/evals/reviewer-run.ts @@ -0,0 +1,414 @@ +#!/usr/bin/env bun + +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import packageJson from "../package.json" with { type: "json" }; +import { canonicalSha256 } from "./canonical-json.js"; +import { parseCaseCatalog } from "./catalog.js"; +import { + type CommandEnd, + EvalHost, + type Outcome, + packPlugin, + preparePackageCache, +} from "./harness.js"; +import { + evaluatorIdentity, + hostConfigSha256, + inspectArtifact, + instructionDelivery, + normalizeRequestedModel, + redactTranscript, +} from "./provenance.js"; +import type { + ActorIdentity, + AttemptRecordV2, + CampaignPlan, + InstructionDelivery, + ModelIdentity, +} from "./report.js"; +import { campaignPlanSha256 } from "./report.js"; +import { createReportStore } from "./report-store.js"; +import { + type DurableReviewerSubmission, + readDurableReviewerSubmission, + seedReviewerAssignment, +} from "./reviewer-assignment.js"; +import { REVIEWER_CASES, type ReviewerCase } from "./reviewer-cases.js"; + +const ANALYSIS_DIGEST = canonicalSha256("flow-reviewer-analysis-v1", { + kind: "reviewer", + interval: "wilson", + alpha: 0.05, +}); + +type Options = { + readonly model: string; + readonly limit: number; +}; + +function parseArgs(argv: readonly string[]): Options { + let model: string | undefined; + let limit = REVIEWER_CASES.length; + for (let index = 0; index < argv.length; index += 1) { + const flag = argv[index]; + const value = argv[index + 1]; + if (flag === "--model" && value) { + model = value; + index += 1; + } else if (flag === "--limit" && value) { + limit = Number.parseInt(value, 10); + index += 1; + } else if (flag === "--help" || flag === "-h") { + console.log( + "usage: bun run evals/reviewer-run.ts -- --model provider/model [--limit n]", + ); + process.exit(0); + } else { + throw new Error(`Unknown or incomplete argument: ${flag ?? ""}`); + } + } + if (!model) throw new Error("Pass --model provider/model."); + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new Error("--limit must be a positive integer."); + } + return { model, limit: Math.min(limit, REVIEWER_CASES.length) }; +} + +function requestedModel(modelId: string): ModelIdentity { + const boundary = modelId.indexOf("/"); + const model = boundary >= 0 ? modelId.slice(boundary + 1) : modelId; + return normalizeRequestedModel({ + modelId, + gateway: model.includes("/") ? modelId.slice(0, boundary) : null, + family: model, + revision: null, + }); +} + +function catalogFor(cases: readonly ReviewerCase[]) { + return cases.map((entry) => ({ + caseId: entry.caseId, + caseVersion: entry.caseVersion, + evidenceClass: "reviewer-only" as const, + oracle: "fixed-review-label" as const, + release: "report-only" as const, + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + })); +} + +function planFor( + cases: readonly ReviewerCase[], + model: ModelIdentity, +): CampaignPlan { + const cells = cases.map((entry, index) => ({ + cellId: `cell-${canonicalSha256("flow-reviewer-cell-v1", entry.caseId).slice(7)}`, + blockId: `block-${index}`, + caseId: entry.caseId, + caseVersion: entry.caseVersion, + armToken: null, + repetition: 0, + managerModel: null, + reviewerModel: model, + schedule: "primary" as const, + })); + const plan = { + schemaVersion: 1 as const, + planId: "flow-reviewer-pilot-v1", + planSha256: `sha256:${"0".repeat(64)}`, + randomizationSeed: canonicalSha256("flow-reviewer-seed-v1", { + caseIds: cases.map((entry) => entry.caseId), + model, + }), + cells, + abortPolicy: { retry: "never" as const, maxReplacementBlocks: 0 }, + stoppingRule: { kind: "fixed-attempts" as const, count: cells.length }, + analysis: { + kind: "reviewer" as const, + interval: "wilson" as const, + alpha: 0.05 as const, + versionSha256: ANALYSIS_DIGEST, + }, + budget: { + maxUsd: null, + unknownCostPolicy: "token-wall-clock-bounds" as const, + maxOutputTokens: Math.max(1, cells.length) * 200_000, + maxWallClockMs: Math.max(1, cells.length) * 20 * 60_000, + maxAttempts: cells.length, + }, + }; + plan.planSha256 = campaignPlanSha256(plan); + return plan; +} + +function reportReviewerActor( + model: ModelIdentity, + outcome: Outcome, +): ActorIdentity | null { + const observed = outcome.actors?.find((actor) => actor.role === "reviewer"); + if (!observed || observed.sessionIds.length === 0) return null; + const actualModel: ActorIdentity["actualModel"] = + observed.actualModel.kind === "observed" + ? { + kind: "unobserved", + reason: `Host observed reviewer providerID=${observed.actualModel.value.providerID} modelID=${observed.actualModel.value.modelID}; full family, gateway, and revision identity is unavailable.`, + } + : { + kind: "unobserved", + reason: `Reviewer identity unavailable: ${observed.actualModel.reason}.`, + }; + return { + role: "reviewer", + requestedModel: model, + actualModel, + sessionIds: [...observed.sessionIds], + }; +} + +export function reviewerOutcome( + entry: ReviewerCase, + submission: DurableReviewerSubmission, + endedBy: CommandEnd, +): AttemptRecordV2["outcome"] { + const verdict = submission.kind === "submitted" ? submission.verdict : null; + const passed = + verdict !== null && + (entry.truth === "defect" ? verdict === "failed" : verdict === "passed"); + const findings = + submission.kind === "submitted" + ? submission.findings.map((finding) => + [ + finding.severity, + finding.summary, + ...(finding.evidence ? [finding.evidence] : []), + ] + .join(": ") + .slice(0, 4096), + ) + : []; + return { + kind: "product", + passed, + endedBy: endedBy === "escalated" ? "user-escalation" : "quiet", + issues: passed ? [] : ["Reviewer verdict did not match the fixed label."], + evidence: { + kind: "reviewer-only", + truth: entry.truth, + verdict, + findings, + submitted: submission.kind === "submitted", + }, + }; +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + const cases = REVIEWER_CASES.slice(0, options.limit); + const model = requestedModel(options.model); + const repositoryRoot = join(import.meta.dir, ".."); + const opencodeVersion = packageJson.devDependencies["@opencode-ai/plugin"]; + const packDir = await mkdtemp(join(tmpdir(), "flow-reviewer-pack-")); + const reportDir = join(repositoryRoot, "evals", "results"); + await mkdir(reportDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const campaignDirectory = join(reportDir, `reviewer-${stamp}.v2`); + const catalogInput = catalogFor(cases); + const catalog = parseCatalog(catalogInput); + const plan = planFor(cases, model); + const store = createReportStore({ directory: campaignDirectory, catalog }); + await store.initialize(plan); + await store.writeCatalog(catalog); + const startedAt = new Date().toISOString(); + try { + const tarball = await packPlugin(repositoryRoot, packDir); + const artifact = await inspectArtifact({ + repositoryRoot, + tarballPath: tarball, + }); + await store.writeArtifact(tarball); + const evaluator = evaluatorIdentity({ + sourceCommit: artifact.sourceCommit, + caseCatalog: cases.map((entry) => ({ + caseId: entry.caseId, + files: entry.files, + })), + policyCatalog: catalog, + graderBundle: { sourceTreeSha256: artifact.sourceTreeSha256 }, + }); + const packageCache = await preparePackageCache(tarball, packDir); + const attempts: AttemptRecordV2[] = []; + for (const [index, entry] of cases.entries()) { + const cell = plan.cells[index]; + if (!cell) throw new Error("Reviewer campaign cell is missing."); + const started = Date.now(); + let host: EvalHost | null = null; + let attempt: AttemptRecordV2; + try { + host = await EvalHost.start({ + packageCache, + opencodeVersion, + files: entry.files, + reviewerModel: options.model, + }); + const catalogModels = await host.catalogModels(); + if (!catalogModels.includes(options.model)) { + throw new Error( + `Reviewer model ${options.model} is absent from the host catalog.`, + ); + } + const seed = await seedReviewerAssignment({ + workspace: host.project, + fixture: entry, + }); + const sessionId = await host.createSession("reviewer evaluation"); + const commandEnd = await host.runCommand( + sessionId, + "flow-review", + seed.assignmentId, + options.model, + ); + const outcome = await host.outcome([sessionId], Date.now() - started); + const submission = await readDurableReviewerSubmission({ + workspace: host.project, + seed, + }); + const transcript = redactTranscript({ + projectPath: host.project, + value: { calls: outcome.allCalls, finalText: outcome.finalText }, + }); + const storedTranscript = await store.writeTranscript({ + attemptId: `attempt-${cell.cellId}`, + text: transcript.text, + }); + const instructions: InstructionDelivery[] = [ + instructionDelivery({ + source: "command", + name: "flow-review", + sequence: 0, + text: seed.assignmentId, + }), + ]; + const actor = reportReviewerActor(model, outcome); + attempt = { + schemaVersion: 2, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: null, + repetition: 0, + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + reviewerModel: options.model, + }), + actors: actor ? [actor] : [], + instructions, + transcript: { + sha256: storedTranscript.sha256, + artifact: storedTranscript.artifact, + }, + outcome: reviewerOutcome(entry, submission, commandEnd), + usage: { + durationMs: outcome.durationMs, + outputTokens: outcome.tokens.output, + costUsd: outcome.costUsd, + }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + attempt = { + schemaVersion: 2, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: null, + repetition: 0, + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + reviewerModel: options.model, + }), + actors: [], + instructions: [], + transcript: null, + outcome: { + kind: "failure", + origin: "host", + code: message.slice(0, 512), + retryable: true, + }, + usage: { + durationMs: Date.now() - started, + outputTokens: 0, + costUsd: null, + }, + }; + } finally { + await host?.stop(); + } + await store.writeAttempt(attempt); + attempts.push(attempt); + } + const finishedAt = new Date().toISOString(); + const complete = attempts.every( + (attempt) => attempt.outcome.kind === "product", + ); + const costUsd = attempts.some((attempt) => attempt.usage.costUsd === null) + ? null + : attempts.reduce( + (total, attempt) => total + (attempt.usage.costUsd ?? 0), + 0, + ); + await store.finalize({ + reportId: `flow-reviewer-${stamp}`, + completion: { + status: complete ? "complete" : "stopped", + cause: complete ? "fixed-target" : "host", + startedAt, + finishedAt, + activatedReserveCellIds: [], + observed: { + attempts: attempts.length, + outputTokens: attempts.reduce( + (total, attempt) => total + attempt.usage.outputTokens, + 0, + ), + costUsd, + wallClockMs: Math.max( + Date.parse(finishedAt) - Date.parse(startedAt), + ...attempts.map((attempt) => attempt.usage.durationMs), + ), + }, + }, + allocationCommitmentSha256: null, + }); + console.log( + `Reviewer V2 report: ${join(campaignDirectory, "report.json")}`, + ); + } finally { + await rm(packDir, { recursive: true, force: true }); + } +} + +function parseCatalog(input: unknown) { + const parsed = parseCaseCatalog(input); + if (!parsed.ok) throw new Error(JSON.stringify(parsed.issues)); + return parsed.value; +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/reviewer-eval.test.ts b/tests/reviewer-eval.test.ts new file mode 100644 index 0000000..7275420 --- /dev/null +++ b/tests/reviewer-eval.test.ts @@ -0,0 +1,404 @@ +import { describe, expect, test } from "bun:test"; +import { parseCaseCatalog } from "../evals/catalog.js"; +import { campaignPlanSha256, parseReport } from "../evals/report.js"; +import { + analyzeReviewerCalibration, + createReviewerPromotion, + krippendorffNominalAlpha, + ReviewerPromotionRecordSchema, +} from "../evals/reviewer-calibration.js"; +import { + assertReviewerCaseTruth, + REVIEWER_CASES, +} from "../evals/reviewer-cases.js"; + +const digest = (letter: string) => `sha256:${letter.repeat(64)}`; + +function promotionFixture(unobserved = false, unsubmitted = false) { + const model = { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "reviewer", + revision: null, + }; + const artifact = { + packageVersion: "1.0.0", + sourceCommit: "commit", + sourceTreeSha256: digest("a"), + tarballSha256: digest("b"), + unpackedManifestSha256: digest("c"), + }; + const catalogInput = REVIEWER_CASES.map((fixture) => ({ + caseId: fixture.caseId, + caseVersion: fixture.caseVersion, + evidenceClass: "reviewer-only" as const, + oracle: "fixed-review-label" as const, + release: "report-only" as const, + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + })); + const catalog = parseCaseCatalog(catalogInput); + if (!catalog.ok) throw new Error("Fixture catalog must parse."); + const cells = REVIEWER_CASES.map((fixture, index) => ({ + cellId: `cell-${index}`, + blockId: `block-${index}`, + caseId: fixture.caseId, + caseVersion: fixture.caseVersion, + armToken: null, + repetition: index, + managerModel: null, + reviewerModel: model, + schedule: "primary" as const, + })); + const plan = { + schemaVersion: 1 as const, + planId: "reviewer-calibration", + planSha256: digest("d"), + randomizationSeed: "seed", + cells, + abortPolicy: { retry: "never" as const, maxReplacementBlocks: 0 }, + stoppingRule: { kind: "fixed-attempts" as const, count: cells.length }, + analysis: { + kind: "reviewer" as const, + interval: "wilson" as const, + alpha: 0.05 as const, + versionSha256: digest("e"), + }, + budget: { + maxUsd: 10, + unknownCostPolicy: "stop" as const, + maxOutputTokens: 100, + maxWallClockMs: 10_000, + maxAttempts: cells.length, + }, + }; + plan.planSha256 = campaignPlanSha256(plan); + const attempts = cells.map((cell) => { + const fixture = REVIEWER_CASES.find( + (candidate) => + candidate.caseId === cell.caseId && + candidate.caseVersion === cell.caseVersion, + ); + if (!fixture) throw new Error("Expected registered reviewer fixture."); + const defect = fixture.truth === "defect"; + const submitted = !unsubmitted || defect; + return { + schemaVersion: 2 as const, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: 1, + armToken: null, + repetition: cell.repetition, + artifact, + evaluator: { + sourceCommit: "evaluator", + caseCatalogSha256: digest("f"), + policyCatalogSha256: digest("0"), + graderBundleSha256: digest("1"), + }, + hostConfigSha256: digest("2"), + actors: [ + { + role: "reviewer" as const, + requestedModel: model, + actualModel: unobserved + ? { kind: "unobserved" as const, reason: "missing" } + : { kind: "observed" as const, value: model }, + sessionIds: [`session-${cell.cellId}`], + }, + ], + instructions: [ + { + source: "command" as const, + name: "review", + sequence: 0, + sha256: digest("3"), + bytes: 1, + }, + ], + transcript: { sha256: digest("4"), artifact: `${cell.cellId}.json` }, + outcome: { + kind: "product" as const, + passed: submitted, + endedBy: "quiet" as const, + issues: submitted ? [] : ["Reviewer did not submit."], + evidence: { + kind: "reviewer-only" as const, + truth: defect ? ("defect" as const) : ("clean" as const), + verdict: submitted + ? defect + ? ("failed" as const) + : ("passed" as const) + : null, + findings: submitted && defect ? ["defect"] : [], + submitted, + }, + }, + usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, + }; + }); + const parsed = parseReport( + { + schemaVersion: 2, + reportId: "calibration", + plan, + attempts, + completion: { + status: "complete", + cause: "fixed-target", + startedAt: "2026-08-25T00:00:00.000Z", + finishedAt: "2026-08-25T00:00:01.000Z", + activatedReserveCellIds: [], + observed: { + attempts: attempts.length, + outputTokens: attempts.length, + costUsd: 0, + wallClockMs: 1_000, + }, + }, + allocationCommitmentSha256: null, + }, + catalog.value, + ); + if (!parsed.ok) throw new Error(JSON.stringify(parsed.issues)); + const labels = REVIEWER_CASES.flatMap((fixture) => + fixture.humanLabels.map((label) => ({ + ...label, + caseId: fixture.caseId, + caseVersion: fixture.caseVersion, + })), + ); + return { + report: parsed.value, + catalog: catalog.value, + labels, + artifact, + model, + }; +} + +describe("reviewer fixed cases", () => { + test("uses opaque versioned fixtures with two immutable labels each", () => { + expect(REVIEWER_CASES).toHaveLength(2); + for (const fixture of REVIEWER_CASES) { + expect(fixture.humanLabels).toHaveLength(2); + expect(assertReviewerCaseTruth(fixture)).toBe(fixture.truth); + } + }); + + test("rejects executable truth drift", () => { + const fixture = REVIEWER_CASES[0]; + if (!fixture) throw new Error("Expected fixture."); + expect(() => + assertReviewerCaseTruth({ + caseId: fixture.caseId, + caseVersion: fixture.caseVersion, + files: { + ...fixture.files, + "src/value.test.ts": "test fixture drift\n", + }, + }), + ).toThrow("fixture drifted"); + }); +}); + +describe("reviewer calibration", () => { + test("counts every confusion-matrix cell and unsubmitted evidence", () => { + const analysis = analyzeReviewerCalibration([ + { + caseId: "a", + caseVersion: 1, + truth: "defect", + verdict: "failed", + submitted: true, + }, + { + caseId: "b", + caseVersion: 1, + truth: "defect", + verdict: "passed", + submitted: true, + }, + { + caseId: "c", + caseVersion: 1, + truth: "clean", + verdict: "failed", + submitted: true, + }, + { + caseId: "d", + caseVersion: 1, + truth: "clean", + verdict: "passed", + submitted: true, + }, + { + caseId: "e", + caseVersion: 1, + truth: "clean", + verdict: null, + submitted: false, + }, + ]); + expect(analysis.matrix).toEqual({ + truePositives: 1, + falseNegatives: 1, + falsePositives: 1, + trueNegatives: 1, + unsubmitted: 1, + }); + expect(analysis.detectionRate).toBe(0.5); + expect(analysis.falsePositiveRate).toBeCloseTo(1 / 2); + expect(analysis.detectionInterval95).not.toBeNull(); + expect(analysis.falsePositiveInterval95).not.toBeNull(); + }); + + test("computes nominal agreement and exposes disagreement", () => { + const agreed = krippendorffNominalAlpha([ + { caseId: "a", caseVersion: 1, raterId: "one", truth: "defect" }, + { caseId: "a", caseVersion: 1, raterId: "two", truth: "defect" }, + { caseId: "b", caseVersion: 1, raterId: "one", truth: "clean" }, + { caseId: "b", caseVersion: 1, raterId: "two", truth: "clean" }, + ]); + expect(agreed).toBe(1); + const disputed = krippendorffNominalAlpha([ + { caseId: "a", caseVersion: 1, raterId: "one", truth: "defect" }, + { caseId: "a", caseVersion: 1, raterId: "two", truth: "clean" }, + { caseId: "b", caseVersion: 1, raterId: "one", truth: "clean" }, + { caseId: "b", caseVersion: 1, raterId: "two", truth: "defect" }, + ]); + expect(disputed).not.toBeNull(); + expect(disputed ?? 1).toBeLessThan(0.8); + }); + + test("strict promotion records require complete frozen evidence and bounds", () => { + const base = { + schemaVersion: 1, + planSha256: digest("a"), + calibrationReportSha256: digest("b"), + caseCatalogSha256: digest("c"), + humanLabelsSha256: digest("d"), + artifactSha256: digest("e"), + reviewerModels: [ + { + routeProvider: "openai", + gateway: null, + family: "gpt", + model: "reviewer", + revision: null, + }, + ], + defectCases: 4, + cleanCases: 4, + ratersPerCase: 2, + agreement: { method: "krippendorff-alpha", value: 0.9, minimum: 0.8 }, + observed: { + detectionRate: 1, + detectionInterval95: [0.8, 1], + falsePositiveRate: 0, + falsePositiveInterval95: [0, 0.2], + }, + minimumDetectionRate: 0.8, + maximumFalsePositiveRate: 0.2, + recordedAt: "2026-08-25T00:00:00.000Z", + }; + expect(ReviewerPromotionRecordSchema.safeParse(base).success).toBe(true); + expect( + ReviewerPromotionRecordSchema.safeParse({ + ...base, + agreement: { ...base.agreement, value: 0.7 }, + }).success, + ).toBe(false); + expect( + ReviewerPromotionRecordSchema.safeParse({ + ...base, + observed: { ...base.observed, falsePositiveInterval95: [0, 0.3] }, + }).success, + ).toBe(false); + }); + + test("keeps pilots advisory and promotes only a complete observed calibration", () => { + const fixture = promotionFixture(); + const input = { + report: fixture.report, + catalog: fixture.catalog, + labels: fixture.labels, + artifact: fixture.artifact, + reviewerModels: [fixture.model], + minimumDetectionRate: 0.2, + maximumFalsePositiveRate: 0.8, + minimumCasesPerTruth: 1, + recordedAt: "2026-08-25T00:00:00.000Z", + }; + expect(createReviewerPromotion({ ...input, mode: "pilot" }).kind).toBe( + "advisory", + ); + expect(createReviewerPromotion({ ...input, mode: "promotion" }).kind).toBe( + "promotion", + ); + }); + + test("refuses duplicate, unrelated, and unobserved calibration evidence", () => { + const fixture = promotionFixture(); + const input = { + mode: "promotion" as const, + report: fixture.report, + catalog: fixture.catalog, + artifact: fixture.artifact, + reviewerModels: [fixture.model], + minimumDetectionRate: 0.2, + maximumFalsePositiveRate: 0.8, + minimumCasesPerTruth: 1, + recordedAt: "2026-08-25T00:00:00.000Z", + }; + const firstLabel = fixture.labels[0]; + if (!firstLabel) throw new Error("Expected fixed human label."); + expect( + createReviewerPromotion({ + ...input, + labels: [...fixture.labels, { ...firstLabel, raterId: "a" }], + }).kind, + ).toBe("advisory"); + expect( + createReviewerPromotion({ + ...input, + labels: [ + ...fixture.labels, + { caseId: "other", caseVersion: 1, raterId: "a", truth: "clean" }, + ], + }).kind, + ).toBe("advisory"); + const unobserved = promotionFixture(true); + expect( + createReviewerPromotion({ + ...input, + report: unobserved.report, + catalog: unobserved.catalog, + labels: unobserved.labels, + artifact: unobserved.artifact, + reviewerModels: [unobserved.model], + }).kind, + ).toBe("advisory"); + const unsubmitted = promotionFixture(false, true); + const unsubmittedPromotion = createReviewerPromotion({ + ...input, + report: unsubmitted.report, + catalog: unsubmitted.catalog, + labels: unsubmitted.labels, + artifact: unsubmitted.artifact, + reviewerModels: [unsubmitted.model], + }); + expect(unsubmittedPromotion.kind).toBe("advisory"); + if (unsubmittedPromotion.kind === "advisory") { + expect(unsubmittedPromotion.reasons).toContain( + "Calibration cannot contain unsubmitted reviewer outcomes.", + ); + } + }); +}); diff --git a/tests/reviewer-run.test.ts b/tests/reviewer-run.test.ts new file mode 100644 index 0000000..9a83d2c --- /dev/null +++ b/tests/reviewer-run.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + durableReviewerSubmission, + readDurableReviewerSubmission, + seedReviewerAssignment, +} from "../evals/reviewer-assignment.js"; +import { REVIEWER_CASES } from "../evals/reviewer-cases.js"; +import { reviewerOutcome } from "../evals/reviewer-run.js"; +import { createFileSessionRepository } from "../src/infrastructure/fs/session-repository.js"; +import { + flowFeatureComplete, + flowSessionClose, +} from "../src/infrastructure/fs/workspace-flow-service.js"; + +async function gitFixture( + files: Readonly>, +): Promise { + const workspace = await mkdtemp(join(tmpdir(), "flow-reviewer-test-")); + for (const [relativePath, contents] of Object.entries(files)) { + const path = join(workspace, relativePath); + await mkdir(join(path, ".."), { recursive: true }); + await writeFile(path, contents, "utf8"); + } + for (const command of [ + ["git", "init", "--initial-branch=main"], + ["git", "config", "user.email", "eval@example.com"], + ["git", "config", "user.name", "Flow Eval"], + ["git", "add", "-A"], + ["git", "commit", "-m", "fixture"], + ]) { + const process = Bun.spawn(command, { + cwd: workspace, + stdout: "ignore", + stderr: "pipe", + }); + if ((await process.exited) !== 0) { + throw new Error(await new Response(process.stderr).text()); + } + } + return workspace; +} + +describe("reviewer pilot adapters", () => { + test("scores only durable submissions and preserves command endings", () => { + const defect = REVIEWER_CASES[0]; + const clean = REVIEWER_CASES[1]; + if (!defect || !clean) throw new Error("Expected fixed reviewer cases."); + expect( + reviewerOutcome( + defect, + { + kind: "submitted", + verdict: "failed", + findings: [ + { + severity: "blocking", + summary: "Wrong result", + evidence: "src/value.ts", + }, + ], + }, + "quiet", + ), + ).toMatchObject({ kind: "product", passed: true, endedBy: "quiet" }); + expect( + reviewerOutcome( + clean, + { + kind: "submitted", + verdict: "failed", + findings: [ + { + severity: "blocking", + summary: "False alarm", + evidence: "src/value.ts", + }, + ], + }, + "quiet", + ), + ).toMatchObject({ kind: "product", passed: false }); + expect( + reviewerOutcome(clean, { kind: "unsubmitted" }, "escalated"), + ).toMatchObject({ + kind: "product", + passed: false, + endedBy: "user-escalation", + evidence: { submitted: false, verdict: null }, + }); + }); + + test("seeds and reads a real durable Flow review assignment", async () => { + const fixture = REVIEWER_CASES[1]; + if (!fixture) throw new Error("Expected clean reviewer case."); + const workspace = await gitFixture(fixture.files); + try { + const seed = await seedReviewerAssignment({ workspace, fixture }); + const before = await createFileSessionRepository(workspace).read(); + expect(durableReviewerSubmission({ session: before, seed })).toEqual({ + kind: "unsubmitted", + }); + if (!before) throw new Error("Expected seeded Flow state."); + const response = await flowFeatureComplete(workspace, { + request: { + operationId: "review-submit-test", + expectedRevision: before.revision, + featureId: seed.featureId, + assignmentId: seed.assignmentId, + summary: "Requirement verified.", + result: { + verdict: "passed", + findings: [], + terminalDisposition: "submitted", + }, + }, + }); + expect(response.status).toBe("ok"); + const completed = await createFileSessionRepository(workspace).read(); + if (!completed) throw new Error("Expected completed Flow state."); + const closed = await flowSessionClose(workspace, { + request: { + operationId: "review-close-test", + expectedRevision: completed.revision, + sessionId: completed.id, + kind: "completed", + summary: "Reviewer pilot fixture completed.", + }, + }); + expect(closed.status).toBe("ok"); + expect(await createFileSessionRepository(workspace).read()).toBeNull(); + expect(await readDurableReviewerSubmission({ workspace, seed })).toEqual({ + kind: "submitted", + verdict: "passed", + findings: [], + }); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); + + test("seeds the planted defect only after its declared test passes", async () => { + const fixture = REVIEWER_CASES[0]; + if (!fixture) throw new Error("Expected defect reviewer case."); + const workspace = await gitFixture(fixture.files); + try { + const seed = await seedReviewerAssignment({ workspace, fixture }); + expect(await readDurableReviewerSubmission({ workspace, seed })).toEqual({ + kind: "unsubmitted", + }); + } finally { + await rm(workspace, { recursive: true, force: true }); + } + }); +}); From b5c6bc4faa86c6e69575c0d8e3ee2606df482ea6 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Tue, 25 Aug 2026 09:16:21 +0200 Subject: [PATCH 2/2] docs(evals): record phase 6 evidence --- .../evidence/phase-6-pilot.json | 34 ++++++++++++++++ .../evidence/phase-6-review.md | 39 +++++++++++++++++++ .../phase-6-reviewer-calibration.md | 9 +++++ .audit/eval-engineering.tsv | 4 ++ 4 files changed, 86 insertions(+) create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-6-pilot.json create mode 100644 .agents/plans/02-eval-engineering/evidence/phase-6-review.md diff --git a/.agents/plans/02-eval-engineering/evidence/phase-6-pilot.json b/.agents/plans/02-eval-engineering/evidence/phase-6-pilot.json new file mode 100644 index 0000000..60536a5 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-6-pilot.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "reportId": "flow-reviewer-2026-08-25T07-08-21-624Z", + "reportSha256": "sha256:f5840f53260ca7e7e031c94bcaea80895e901d07beeef942930aaa752e2a6afc", + "planSha256": "sha256:75ac17c6ffdf7de1f8fb6c93b605a9e415a0a4995ea1dfd1b4b4ecf6a03e37b6", + "completionStatus": "complete", + "completionCause": "fixed-target", + "attemptCount": 2, + "submitted": 2, + "truePositives": 1, + "falseNegatives": 0, + "falsePositives": 0, + "trueNegatives": 1, + "detectionRate": 1, + "detectionInterval95": [0.20654931437723745, 1], + "falsePositiveRate": 0, + "falsePositiveInterval95": [0, 0.7934506856227626], + "outputTokens": 1747, + "costUsd": 0.14806019999999998, + "artifactTarballSha256": "sha256:c89f7363248ccc3e3f69728c1aa42044a25938cd533e470d9f73ef08bc64ad24", + "defectTranscriptSha256": "sha256:c09cd249220683b6b2ad3fb51d17ce7058c2c96637152de2758270ae6238bbfc", + "cleanTranscriptSha256": "sha256:fbd7065faaf18f421c4cd8e17b6077492fcec6158293c1b2309b5d4d4b834e11", + "reviewerProviderModelObserved": true, + "reviewerFullIdentityObserved": false, + "strictParsePassed": true, + "promotion": "advisory", + "promotionReasons": [ + "pilot-only", + "sample-floor-missed", + "detection-bound-missed", + "false-positive-bound-missed", + "full-reviewer-identity-unavailable" + ] +} diff --git a/.agents/plans/02-eval-engineering/evidence/phase-6-review.md b/.agents/plans/02-eval-engineering/evidence/phase-6-review.md new file mode 100644 index 0000000..360bd92 --- /dev/null +++ b/.agents/plans/02-eval-engineering/evidence/phase-6-review.md @@ -0,0 +1,39 @@ +# Phase 6 Interrogate review and pilot + +Phase 6 measures the reviewer without a manager model selecting, repairing, or +submitting the work. The evaluator seeds an approved plan, real JUnit-backed +validation, and a pending assignment through Flow's application services. The +normal `/flow-review` command creates the reviewer child. Scoring reads only the +durable assignment result from active or archived Flow state. + +Architect and Arena rejected a direct-agent shortcut because it bypassed child +lineage and production dispatch semantics. The selected design reuses the real +review command and existing reviewer identity observation. Requested identity +stays separate from host-observed provider and model fields; missing family, +gateway, and revision fields remain explicitly unobserved. + +The adversarial pass fixed four evidence defects before the accepted pilot: + +- fixed truth now hashes the complete fixture, and promotion labels must exactly + match the versioned immutable registry; +- unsubmitted outcomes enter neither rate denominator, and any unsubmitted or + non-fixed-target campaign is ineligible for promotion; +- the runner preserves user escalation and scores a verdict only when Flow + durably accepted the reviewer submission; +- completed sessions are read from the archive, and OpenCode's lazy package setup + finishes before source binding; +- both controls run the declared JUnit case, include baseline inventory, and use a + safe-integer requirement. The planted defect passes visible validation but still + violates the approved behavior. + +The accepted paid pilot produced two submitted outcomes. The reviewer detected the +planted defect and passed the clean control. Cost was $0.1480602 for 1,747 output +tokens. Strict v2 parsing and reviewer analysis passed. The apparent 100% rates are +descriptive only: one case per truth class gives a detection lower bound of 0.2065 +and a false-positive upper bound of 0.7935. Full reviewer identity is unavailable +on OpenCode 1.18.6. Promotion therefore remains advisory and release policy remains +report-only. + +Earlier interrupted and invalid campaigns are excluded. They exposed archive, +source-binding, and control-definition defects and support no performance claim. +The final four-model recheck found no unresolved blocker. diff --git a/.agents/plans/02-eval-engineering/phase-6-reviewer-calibration.md b/.agents/plans/02-eval-engineering/phase-6-reviewer-calibration.md index a3c12a7..ec05fde 100644 --- a/.agents/plans/02-eval-engineering/phase-6-reviewer-calibration.md +++ b/.agents/plans/02-eval-engineering/phase-6-reviewer-calibration.md @@ -34,3 +34,12 @@ measurement, not calibration by itself. Stop gate. Release promotion needs a preregistered human-labelled set, at least two raters per case, Krippendorff alpha at or above 0.8, and confidence bounds that meet the recorded detection and false-positive thresholds. + +## Outcome + +Implemented and verified. The packed-host pilot exercised two real reviewer child +assignments with durable submissions and exact fixed labels. It detected the +planted defect and passed the clean control, but remains advisory because the +sample is below the preregistered floor, confidence bounds miss promotion +thresholds, and the pinned host cannot expose a full reviewer identity. See +`evidence/phase-6-review.md` and `evidence/phase-6-pilot.json`. diff --git a/.audit/eval-engineering.tsv b/.audit/eval-engineering.tsv index 3545026..c2c1021 100644 --- a/.audit/eval-engineering.tsv +++ b/.audit/eval-engineering.tsv @@ -45,3 +45,7 @@ ts phase decision why evidence result 2026-08-25T04:54:04Z phase-5 fixed the multi-model Interrogate findings workflow contracts, record handoff, full artifact identity, catalog hashing, per-attempt host config, and required policies affected release integrity .agents/plans/02-eval-engineering/evidence/phase-5-review.md VERIFIED no unresolved blocker 2026-08-25T04:54:04Z phase-5 completed the throughput checkpoint the vertical slice must reduce maintenance load before new evidence families .agents/plans/02-eval-engineering/evidence/phase-5-review.md VERIFIED new scenario touches scenario, policy, and one test; legacy path has no authority 2026-08-25T04:54:04Z phase-5 ran Deslop, workflow lint, and the full repository gate the cutover must land atomically and regression-free actionlint; bun run check VERIFIED 473 pass, 1 skip, 0 fail +2026-08-25T07:12:18Z phase-6 selected the production reviewer path through Architect and Arena direct agent prompts bypass reviewer-child lineage and production dispatch semantics .agents/plans/02-eval-engineering/evidence/phase-6-review.md VERIFIED normal /flow-review child over evaluator-seeded durable assignment +2026-08-25T07:12:18Z phase-6 fixed truth, labels, durable scoring, archive recovery, source binding, and controls reviewer calibration cannot trust prose, partial fixture hashes, mutable labels, or evaluator-created drift evals/reviewer-cases.ts; evals/reviewer-assignment.ts; evals/reviewer-run.ts; evals/reviewer-calibration.ts VERIFIED focused integration and strict parser gates green +2026-08-25T07:12:18Z phase-6 ran the final paid packed-host reviewer pilot the phase needs real defect and clean evidence without manager selection or repair confounding .agents/plans/02-eval-engineering/evidence/phase-6-pilot.json VERIFIED 1 detection, 0 false positives, 2 submissions, advisory only, $0.1480602 +2026-08-25T07:12:18Z phase-6 ran Deslop, four-model Interrogate, and the full repository gate the phase must remain reviewable, regression-free, and make no unsupported promotion claim bun run check; .agents/plans/02-eval-engineering/evidence/phase-6-review.md VERIFIED 483 pass, 1 skip, 0 fail; no unresolved blocker