diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 29ed1345..77b2f497 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.109.1" +version = "0.110.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index f027db6e..ca672174 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.109.1" + __version__ = "0.110.0" __all__ = [ "Client", diff --git a/package.json b/package.json index 9c50c69a..825946cf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.109.1", + "version": "0.110.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { @@ -39,11 +39,6 @@ "import": "./dist/rl.js", "default": "./dist/rl.js" }, - "./diagnose": { - "types": "./dist/diagnose.d.ts", - "import": "./dist/diagnose.js", - "default": "./dist/diagnose.js" - }, "./fuzz": { "types": "./dist/fuzz.d.ts", "import": "./dist/fuzz.js", @@ -54,16 +49,6 @@ "import": "./dist/traces.js", "default": "./dist/traces.js" }, - "./telemetry": { - "types": "./dist/telemetry/index.d.ts", - "import": "./dist/telemetry/index.js", - "default": "./dist/telemetry/index.js" - }, - "./telemetry/file": { - "types": "./dist/telemetry/file.d.ts", - "import": "./dist/telemetry/file.js", - "default": "./dist/telemetry/file.js" - }, "./wire": { "types": "./dist/wire/index.d.ts", "import": "./dist/wire/index.js", @@ -84,41 +69,16 @@ "import": "./dist/meta-eval/index.js", "default": "./dist/meta-eval/index.js" }, - "./prm": { - "types": "./dist/prm/index.d.ts", - "import": "./dist/prm/index.js", - "default": "./dist/prm/index.js" - }, "./builder-eval": { "types": "./dist/builder-eval/index.d.ts", "import": "./dist/builder-eval/index.js", "default": "./dist/builder-eval/index.js" }, - "./governance": { - "types": "./dist/governance/index.d.ts", - "import": "./dist/governance/index.js", - "default": "./dist/governance/index.js" - }, - "./knowledge": { - "types": "./dist/knowledge/index.d.ts", - "import": "./dist/knowledge/index.js", - "default": "./dist/knowledge/index.js" - }, "./matrix": { "types": "./dist/matrix/index.d.ts", "import": "./dist/matrix/index.js", "default": "./dist/matrix/index.js" }, - "./perf": { - "types": "./dist/perf/index.d.ts", - "import": "./dist/perf/index.js", - "default": "./dist/perf/index.js" - }, - "./product-benchmark": { - "types": "./dist/product-benchmark/index.d.ts", - "import": "./dist/product-benchmark/index.js", - "default": "./dist/product-benchmark/index.js" - }, "./multishot": { "types": "./dist/multishot/index.d.ts", "import": "./dist/multishot/index.js", @@ -139,51 +99,21 @@ "import": "./dist/authenticity/index.js", "default": "./dist/authenticity/index.js" }, - "./groundedness": { - "types": "./dist/groundedness/index.d.ts", - "import": "./dist/groundedness/index.js", - "default": "./dist/groundedness/index.js" - }, "./belief-state": { "types": "./dist/belief-state/index.d.ts", "import": "./dist/belief-state/index.js", "default": "./dist/belief-state/index.js" }, - "./workflow": { - "types": "./dist/workflow/index.d.ts", - "import": "./dist/workflow/index.js", - "default": "./dist/workflow/index.js" - }, "./contract": { "types": "./dist/contract/index.d.ts", "import": "./dist/contract/index.js", "default": "./dist/contract/index.js" }, - "./adapters/langchain": { - "types": "./dist/adapters/langchain.d.ts", - "import": "./dist/adapters/langchain.js", - "default": "./dist/adapters/langchain.js" - }, - "./adapters/http": { - "types": "./dist/adapters/http.d.ts", - "import": "./dist/adapters/http.js", - "default": "./dist/adapters/http.js" - }, - "./adapters/otel": { - "types": "./dist/adapters/otel.d.ts", - "import": "./dist/adapters/otel.js", - "default": "./dist/adapters/otel.js" - }, "./hosted": { "types": "./dist/hosted/index.d.ts", "import": "./dist/hosted/index.js", "default": "./dist/hosted/index.js" }, - "./testing": { - "types": "./dist/testing.d.ts", - "import": "./dist/testing.js", - "default": "./dist/testing.js" - }, "./openapi.json": { "default": "./dist/openapi.json" } diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 27f24d53..94e166a4 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -28,16 +28,12 @@ try { const packageJson = JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8')) const requiredExports = { '.': ['import', 'types'], - './workflow': ['import', 'types'], './campaign': ['import', 'types'], './traces': ['import', 'types'], './rl': ['import', 'types'], - './prm': ['import', 'types'], './meta-eval': ['import', 'types'], - './product-benchmark': ['import', 'types'], './belief-state': ['import', 'types'], './wire': ['import', 'types'], - './testing': ['import', 'types'], './openapi.json': ['default'], } @@ -60,27 +56,6 @@ try { '--input-type=module', '--eval', ` - const workflow = await import('@tangle-network/agent-eval/workflow') - const expected = [ - 'buildWorkflowAnalystFeedbackPack', - 'buildWorkflowPartnerReport', - 'buildWorkflowTraceIntelligenceEnvelope', - 'decideWorkflowDriverPromotion', - 'sanitizeWorkflowTraceEnvelope', - 'summarizeWorkflowExecution', - 'validateWorkflowTraceEnvelope', - 'validateWorkflowTraceEvent', - 'validateWorkflowTraceEventKind', - 'validateWorkflowTraceEventPayload', - 'workflowEventsToTraceEnvelope', - 'workflowPhaseGraph', - 'workflowRuntimeResultToTraceEnvelope', - 'workflowTraceToFeedbackTrajectory', - 'workflowTraceToRunRecord', - ] - for (const name of expected) { - if (!(name in workflow)) throw new Error('missing workflow export ' + name) - } `, ], appDir, @@ -95,8 +70,6 @@ try { for (const name of ['campaignToRunRecords', 'extractPreferences', 'buildRlDataset', 'toSftRows', 'runRLCampaign']) { if (!(name in rl)) throw new Error('missing rl export ' + name) } - const prm = await import('@tangle-network/agent-eval/prm') - if (!('PrmGrader' in prm)) throw new Error('missing prm export PrmGrader') const metaEval = await import('@tangle-network/agent-eval/meta-eval') if (!('InMemoryOutcomeStore' in metaEval)) throw new Error('missing meta-eval export InMemoryOutcomeStore') const wire = await import('@tangle-network/agent-eval/wire') @@ -105,20 +78,6 @@ try { if (!('analyzeBeliefPolicy' in beliefState)) { throw new Error('missing belief-state export analyzeBeliefPolicy') } - const testing = await import('@tangle-network/agent-eval/testing') - if (typeof testing.resetLockedAppendersForTesting !== 'function') { - throw new Error('missing testing reset helper') - } - const productBenchmark = await import('@tangle-network/agent-eval/product-benchmark') - for (const name of [ - 'findProductBenchmarkArtifacts', - 'productBenchmarkSplits', - 'validateProductBenchmarkManifest', - 'validateProductBenchmarkRecord', - 'validateProductBenchmarkRun', - ]) { - if (!(name in productBenchmark)) throw new Error('missing product-benchmark export ' + name) - } `, ], appDir, diff --git a/src/diagnose/causal-sweep.ts b/src/diagnose/causal-sweep.ts deleted file mode 100644 index 5e19389f..00000000 --- a/src/diagnose/causal-sweep.ts +++ /dev/null @@ -1,243 +0,0 @@ -/** - * Causal sweep — WHY did this run fail? - * - * Orchestrates the dormant counterfactual primitives into a responsibility - * report: for each candidate step, run `reps` counterfactual replays per - * mutation (via `runCounterfactual` — the consumer's `CounterfactualRunner` - * is the execution seam) and reduce the per-rep score deltas into a mean - * effect + bootstrap confidence interval (via `confidenceInterval`). - * - * Why `reps` is REQUIRED: a single intervention delta is one stochastic - * draw — LLM re-execution from a prefix is sampled, so one replay cannot - * distinguish "this step caused the failure" from sampling noise. The - * signal is the distribution of deltas across reps; the CI over that - * distribution is what lets a caller say "this step's effect excludes - * zero" instead of eyeballing a point estimate. - * - * Budget discipline: the sweep never silently drops cells. When the - * remaining budget cannot fund a full `reps`-sized cell, the sweep halts - * and every step not fully probed is named in `uncovered`. - */ - -import { - attributeCounterfactuals, - type CounterfactualMutation, - type CounterfactualResult, - type CounterfactualRunner, - runCounterfactual, -} from '../counterfactual' -import { ValidationError } from '../errors' -import { confidenceInterval } from '../statistics' -import type { Span } from '../trace/schema' -import type { TraceStore } from '../trace/store' -import { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory' - -/** Stable reference to a trajectory step — carried through reports, - * findings, and corpus records so evidence stays addressable. */ -export interface StepRef { - index: number - spanId: string - kind: Span['kind'] - name: string -} - -export function stepRefOf(step: TrajectoryStep): StepRef { - return { - index: step.index, - spanId: step.span.spanId, - kind: step.span.kind, - name: step.span.name, - } -} - -export interface CausalSweepOptions { - store: TraceStore - /** The failed run to diagnose. Its `outcome.score` is the baseline every - * counterfactual delta is measured against. */ - runId: string - /** Execution seam — identical contract to `runCounterfactual`: re-runs the - * agent from the mutation point and MUST `endRun` with a numeric score. */ - runner: CounterfactualRunner - /** Trajectory indices to probe. Default: every llm + tool span — the kinds - * the existing `CounterfactualMutation` set targets. */ - candidateSteps?: number[] - /** - * Mutations to probe a given step with. Returned mutations MUST target - * `step.index`. Default probes are the payload-free existing kinds: - * - tool span → `swap-tool-result` with `newResult: null` (knockout: - * how much did the run depend on this tool's information?) - * - llm span → `truncate-after` (re-roll: how much did the realized - * turn deviate from the policy's typical continuation?) - * `swap-model` / `inject-system-message` need consumer payloads, so they - * are opt-in via this callback. - */ - mutationsPerStep?: (step: TrajectoryStep) => CounterfactualMutation[] - /** Replays per (step, mutation) cell. Minimum 2 — see module doc. */ - reps: number - /** Hard cap on total counterfactual replays across the whole sweep. */ - budget: number - /** Seed for the bootstrap CI resampler. Deterministic default so two - * sweeps over the same deltas report identical intervals. */ - ciSeed?: number - /** Bootstrap CI confidence level. Default 0.95. */ - ciConfidence?: number -} - -export interface StepResponsibility { - stepRef: StepRef - mutationKind: CounterfactualMutation['kind'] - /** Mean of per-rep score deltas (counterfactual − original). */ - meanEffect: number - /** Bootstrap CI over the per-rep deltas. */ - ci: { mean: number; lower: number; upper: number } - /** `ci.lower > 0 || ci.upper < 0` — the effect is distinguishable from noise. */ - ciExcludesZero: boolean - reps: number - /** Raw per-rep deltas — downstream evidence, never re-derived. */ - deltas: number[] - /** Replay run ids (layer='meta', parentRunId=original) for audit. */ - counterfactualRunIds: string[] -} - -export interface CausalResponsibilityReport { - runId: string - originalScore: number - /** Ranked by |meanEffect| descending — the blame ordering. */ - steps: StepResponsibility[] - /** Kind-level aggregate from the existing `attributeCounterfactuals`. */ - byMutationKind: ReturnType - replaysUsed: number - budget: number - /** Steps planned but not fully probed before the budget ran out. - * Named, never silent: an absent step is "no effect found"; an - * uncovered step is "not measured". */ - uncovered: StepRef[] -} - -const DEFAULT_CI_SEED = 0x5eed - -function defaultMutations(step: TrajectoryStep): CounterfactualMutation[] { - if (step.span.kind === 'tool') { - return [{ kind: 'swap-tool-result', at: step.index, newResult: null }] - } - if (step.span.kind === 'llm') { - return [{ kind: 'truncate-after', at: step.index }] - } - return [] -} - -export async function causalSweep(opts: CausalSweepOptions): Promise { - if (!Number.isInteger(opts.reps) || opts.reps < 2) { - throw new ValidationError( - `causalSweep: reps must be an integer >= 2 (got ${opts.reps}) — a single-intervention delta is one stochastic draw, not a measurement`, - ) - } - if (!Number.isInteger(opts.budget) || opts.budget < 1) { - throw new ValidationError(`causalSweep: budget must be an integer >= 1 (got ${opts.budget})`) - } - - const originalRun = await opts.store.getRun(opts.runId) - if (!originalRun) throw new ValidationError(`causalSweep: run ${opts.runId} not found`) - const originalScore = originalRun.outcome?.score - if (typeof originalScore !== 'number' || !Number.isFinite(originalScore)) { - throw new ValidationError( - `causalSweep: run ${opts.runId} has no numeric outcome.score — deltas have no baseline`, - ) - } - - const trajectory = await buildTrajectory(opts.store, opts.runId) - const candidates = resolveCandidates(trajectory, opts.candidateSteps) - const mutationsFor = opts.mutationsPerStep ?? defaultMutations - - interface Cell { - step: TrajectoryStep - mutation: CounterfactualMutation - } - const cells: Cell[] = [] - for (const step of candidates) { - const mutations = mutationsFor(step) - for (const m of mutations) { - if (m.at !== step.index) { - throw new ValidationError( - `causalSweep: mutationsPerStep returned a mutation targeting at=${m.at} for step index=${step.index} — mutations must target the step they were asked for`, - ) - } - cells.push({ step, mutation: m }) - } - } - - const responsibilities: StepResponsibility[] = [] - const allResults: CounterfactualResult[] = [] - const uncoveredIndices = new Set() - let replaysUsed = 0 - let halted = false - - for (const cell of cells) { - if (halted || replaysUsed + opts.reps > opts.budget) { - // A partial cell would report a CI over fewer reps than requested — - // weaker evidence masquerading as the real thing. Halt and name it. - halted = true - uncoveredIndices.add(cell.step.index) - continue - } - const deltas: number[] = [] - const cfRunIds: string[] = [] - for (let rep = 0; rep < opts.reps; rep++) { - const result = await runCounterfactual(opts.store, opts.runId, cell.mutation, opts.runner) - replaysUsed++ - const d = result.delta.deltaScore - if (typeof d !== 'number' || !Number.isFinite(d)) { - throw new ValidationError( - `causalSweep: counterfactual replay for step ${cell.step.index} (${cell.mutation.kind}) rep ${rep} produced no numeric score — the runner must endRun with a numeric outcome.score`, - ) - } - deltas.push(d) - cfRunIds.push(result.counterfactualRunId) - allResults.push(result) - } - const ci = confidenceInterval(deltas, opts.ciConfidence ?? 0.95, { - seed: opts.ciSeed ?? DEFAULT_CI_SEED, - }) - responsibilities.push({ - stepRef: stepRefOf(cell.step), - mutationKind: cell.mutation.kind, - meanEffect: ci.mean, - ci, - ciExcludesZero: ci.lower > 0 || ci.upper < 0, - reps: opts.reps, - deltas, - counterfactualRunIds: cfRunIds, - }) - } - - responsibilities.sort((a, b) => Math.abs(b.meanEffect) - Math.abs(a.meanEffect)) - - // A step probed under one mutation but cut off under another appears in - // BOTH steps and uncovered — partial coverage is named, not blended. - const uncovered = candidates.filter((s) => uncoveredIndices.has(s.index)).map(stepRefOf) - - return { - runId: opts.runId, - originalScore, - steps: responsibilities, - byMutationKind: attributeCounterfactuals(allResults), - replaysUsed, - budget: opts.budget, - uncovered, - } -} - -function resolveCandidates(trajectory: Trajectory, indices?: number[]): TrajectoryStep[] { - if (indices === undefined) { - return trajectory.steps.filter((s) => s.span.kind === 'llm' || s.span.kind === 'tool') - } - return indices.map((i) => { - const step = trajectory.steps[i] - if (!step) { - throw new ValidationError( - `causalSweep: candidateSteps index ${i} out of range [0, ${trajectory.steps.length})`, - ) - } - return step - }) -} diff --git a/src/diagnose/index.ts b/src/diagnose/index.ts deleted file mode 100644 index df815e6b..00000000 --- a/src/diagnose/index.ts +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Diagnose chain — WHY a run failed, WHAT should have happened, HOW to - * make it happen. - * - * The full remediation pipeline this subpath closes: - * - * fuzz finds → sweep blames → repair prescribes (validated) → - * findings / corpus / invariant remediate → gates verify - * - * Three stages, all orchestration over existing primitives — nothing here - * re-implements replay, mutation, or attribution: - * - * 1. `causalSweep` — WHY. Runs `reps` counterfactual replays per - * (step, mutation) cell through `runCounterfactual` (the consumer's - * `CounterfactualRunner` is the execution seam) and reduces the - * per-rep deltas into a responsibility ranking with bootstrap CIs - * (`confidenceInterval`). Budget-bounded; unprobed steps are named - * in `uncovered`, never dropped. - * 2. `prescribeRepair` — WHAT SHOULD HAVE HAPPENED. Consumer-supplied - * `proposeFix` (LLM-backed in live use) proposes candidate mutations - * for the blamed steps; each candidate is machine-verified by - * replaying WITH it. Only candidates whose every validation rep - * crosses `flipThreshold` become repairs; the rest are rejected - * with a typed reason. - * 3. Remediation adapters — HOW. `toAnalystFindings` feeds the analyst - * registry, `toCorpusRecord` pins the failure as a permanent corpus - * scenario, `suggestInvariant` emits the trace-contracts hint shape. - */ - -// The execution-seam types consumers must implement live in counterfactual.ts; -// re-exported so a diagnose consumer imports from one subpath. -export type { - CounterfactualContext, - CounterfactualMutation, - CounterfactualResult, - CounterfactualRunner, -} from '../counterfactual' -export type { - CausalResponsibilityReport, - CausalSweepOptions, - StepRef, - StepResponsibility, -} from './causal-sweep' -export { causalSweep, stepRefOf } from './causal-sweep' -export type { InvariantHint } from './remediation' -export { - DIAGNOSE_ANALYST_ID, - describeMutation, - severityFromEffect, - suggestInvariant, - toAnalystFindings, - toCorpusRecord, -} from './remediation' -export type { - PrescribeRepairOptions, - RejectedRepair, - RepairContext, - RepairReport, - ValidatedRepair, -} from './repair' -export { prescribeRepair } from './repair' diff --git a/src/diagnose/remediation.ts b/src/diagnose/remediation.ts deleted file mode 100644 index 600e1f76..00000000 --- a/src/diagnose/remediation.ts +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Remediation adapters — HOW DO WE MAKE IT HAPPEN? - * - * The diagnose chain ends by feeding existing improvement machinery, - * not by building new machinery: - * - * - `toAnalystFindings` → the analyst contract (`makeFinding`), so - * responsibility evidence flows into the same registry / steering / - * diff pipeline every other analyst feeds. - * - `toCorpusRecord` → the RL corpus (`CorpusRecord`), pinning the - * diagnosed failure + validated repair as a permanent scenario. - * - `suggestInvariant` → a plain-data hint in the shape the - * trace-contracts machinery consumes (`never` / `without` clauses). - */ - -import type { AnalystFinding, AnalystSeverity, EvidenceRef } from '../analyst/types' -import { makeFinding } from '../analyst/types' -import type { CounterfactualMutation } from '../counterfactual' -import { ValidationError } from '../errors' -import type { CorpusRecord } from '../rl/corpus' -import type { RunRecord } from '../run-record' -import { validateRunRecord } from '../run-record' -import type { CausalResponsibilityReport, StepResponsibility } from './causal-sweep' -import type { RepairReport, ValidatedRepair } from './repair' - -export const DIAGNOSE_ANALYST_ID = 'diagnose-causal-sweep' - -/** Severity from causal effect size. Effects whose CI includes zero are - * 'info' regardless of magnitude — an indistinguishable-from-noise effect - * must not steer remediation priority. */ -export function severityFromEffect(responsibility: StepResponsibility): AnalystSeverity { - if (!responsibility.ciExcludesZero) return 'info' - const magnitude = Math.abs(responsibility.meanEffect) - if (magnitude >= 0.5) return 'critical' - if (magnitude >= 0.25) return 'high' - if (magnitude >= 0.1) return 'medium' - return 'low' -} - -/** Deterministic human-readable rendering of a mutation — used in - * recommended actions, corpus completions, and invariant hints. */ -export function describeMutation(mutation: CounterfactualMutation): string { - switch (mutation.kind) { - case 'swap-model': - return `use model '${mutation.newModel}' at step ${mutation.at}` - case 'swap-tool-result': - return `replace the tool result at step ${mutation.at} with ${JSON.stringify(mutation.newResult)}` - case 'truncate-after': - return `stop the run after step ${mutation.at}` - case 'inject-system-message': - return `inject system message at step ${mutation.at}: ${mutation.content}` - case 'custom': - return `${mutation.describe} (step ${mutation.at})` - } -} - -/** - * Lift a responsibility report (and optionally its validated repairs) into - * `AnalystFinding`s via the real `makeFinding` factory. One finding per - * probed step; a validated repair for that step upgrades the finding with - * a `recommended_action` + the replay-validation evidence. - * - * Findings are OBSERVED causal probes (replay deltas), not judge verdicts, - * so `derived_from_judge` stays unset and they may steer. - */ -export function toAnalystFindings( - report: CausalResponsibilityReport, - repairs?: RepairReport, -): AnalystFinding[] { - const repairByStep = new Map() - for (const r of repairs?.repairs ?? []) { - if (!repairByStep.has(r.stepRef.spanId)) repairByStep.set(r.stepRef.spanId, r) - } - - return report.steps.map((resp) => { - const repair = repairByStep.get(resp.stepRef.spanId) - const evidence: EvidenceRef[] = [ - { - kind: 'span', - uri: `span://${resp.stepRef.spanId}`, - excerpt: `step ${resp.stepRef.index} (${resp.stepRef.kind} '${resp.stepRef.name}') meanEffect=${resp.meanEffect.toFixed(4)} ci=[${resp.ci.lower.toFixed(4)}, ${resp.ci.upper.toFixed(4)}] reps=${resp.reps}`, - }, - { - kind: 'metric', - uri: `metric://diagnose/${report.runId}/step/${resp.stepRef.index}/${resp.mutationKind}`, - excerpt: `deltas=[${resp.deltas.map((d) => d.toFixed(4)).join(', ')}]`, - }, - ...resp.counterfactualRunIds.map((id): EvidenceRef => ({ kind: 'span', uri: `run://${id}` })), - ] - return makeFinding({ - analyst_id: DIAGNOSE_ANALYST_ID, - severity: severityFromEffect(resp), - area: 'causal-attribution', - claim: `step '${resp.stepRef.name}' (${resp.stepRef.kind}) is causally responsible for the run outcome under ${resp.mutationKind}`, - rationale: resp.ciExcludesZero - ? `mean effect ${resp.meanEffect.toFixed(4)} over ${resp.reps} counterfactual replays; CI [${resp.ci.lower.toFixed(4)}, ${resp.ci.upper.toFixed(4)}] excludes zero` - : `mean effect ${resp.meanEffect.toFixed(4)} over ${resp.reps} counterfactual replays; CI [${resp.ci.lower.toFixed(4)}, ${resp.ci.upper.toFixed(4)}] includes zero — not distinguishable from noise`, - evidence_refs: evidence, - recommended_action: repair ? describeMutation(repair.mutation) : undefined, - validation_plan: repair - ? `replay-validated: ${repair.reps}/${repair.reps} reps scored >= ${repairs!.flipThreshold} (mean ${repair.meanScore.toFixed(4)}, delta ${repair.deltaScore.toFixed(4)})` - : undefined, - confidence: repair ? 0.95 : resp.ciExcludesZero ? 0.85 : 0.3, - subject: resp.stepRef.spanId, - metadata: { - stepRef: resp.stepRef, - mutationKind: resp.mutationKind, - meanEffect: resp.meanEffect, - ci: resp.ci, - deltas: resp.deltas, - counterfactualRunIds: resp.counterfactualRunIds, - ...(repair ? { repair: { mutation: repair.mutation, meanScore: repair.meanScore } } : {}), - }, - }) - }) -} - -/** - * Pin the diagnosed failure as a permanent corpus scenario. Takes the - * original run's `RunRecord` projection plus a validated repair and emits - * a fresh `CorpusRecord` (new runId, so corpus dedup keeps both the raw - * failure and the diagnosed entry). - * - * `completion` defaults to the validated mutation's rendering — "what - * should have happened" in machine-derived form. Supply `prompt` (and - * optionally a richer `completion`) when the trajectory text is available - * so the record is harvestable by `buildDatasetFromCorpus`. - */ -export function toCorpusRecord( - run: RunRecord, - repair: ValidatedRepair, - opts: { prompt?: string; completion?: string } = {}, -): CorpusRecord { - const record: CorpusRecord = { - ...run, - runId: `${run.runId}#repair:${repair.stepRef.spanId}`, - outcome: { - ...run.outcome, - raw: { - ...run.outcome.raw, - diagnose_blamed_step_index: repair.stepRef.index, - diagnose_repair_mean_score: repair.meanScore, - diagnose_repair_delta_score: repair.deltaScore, - diagnose_repair_reps: repair.reps, - }, - }, - prompt: opts.prompt, - completion: opts.completion ?? describeMutation(repair.mutation), - } - // Boundary check — a corpus record that fails RunRecord validation would - // poison every downstream harvest. - validateRunRecord(record) - return record -} - -/** Plain-data invariant hint. The trace-contracts machinery consumes this - * shape: `never` is a pattern that must not appear in a passing trace; - * `without` is a guard whose absence makes the failure reachable. */ -export interface InvariantHint { - description: string - never?: string - without?: string -} - -/** - * Derive an invariant hint from a validated repair. Deterministic per - * mutation kind — the hint names the contract a trace must satisfy so - * the diagnosed failure cannot silently recur. - */ -export function suggestInvariant(repair: ValidatedRepair): InvariantHint { - const { stepRef, mutation } = repair - const at = `step ${stepRef.index} (${stepRef.kind} '${stepRef.name}')` - switch (mutation.kind) { - case 'swap-tool-result': - return { - description: `the result of tool '${stepRef.name}' was causally responsible for the failure; a replaced result flipped the outcome (delta ${repair.deltaScore.toFixed(4)})`, - never: `unvalidated result from tool '${stepRef.name}' flows downstream`, - without: `result guard on tool '${stepRef.name}'`, - } - case 'swap-model': - return { - description: `swapping the model at ${at} to '${mutation.newModel}' flipped the outcome (delta ${repair.deltaScore.toFixed(4)})`, - never: `llm span '${stepRef.name}' runs on a model other than '${mutation.newModel}'`, - } - case 'inject-system-message': - return { - description: `injecting a system message at ${at} flipped the outcome (delta ${repair.deltaScore.toFixed(4)})`, - without: `system message present at '${stepRef.name}': ${mutation.content}`, - } - case 'truncate-after': - return { - description: `stopping after ${at} flipped the outcome (delta ${repair.deltaScore.toFixed(4)}) — continuation past this step caused the failure`, - never: `spans execute after '${stepRef.name}' (index ${stepRef.index})`, - } - case 'custom': - return { - description: `${mutation.describe} at ${at} flipped the outcome (delta ${repair.deltaScore.toFixed(4)})`, - } - default: { - const exhausted: never = mutation - throw new ValidationError( - `suggestInvariant: unknown mutation kind ${JSON.stringify(exhausted)}`, - ) - } - } -} diff --git a/src/diagnose/repair.ts b/src/diagnose/repair.ts deleted file mode 100644 index 451a1657..00000000 --- a/src/diagnose/repair.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Replay-validated repair — WHAT SHOULD HAVE HAPPENED? - * - * Takes the blamed steps from a `CausalResponsibilityReport`, asks a - * consumer-supplied `proposeFix` (LLM-backed in live use) for candidate - * mutations, and machine-verifies each candidate by replaying the run - * WITH the mutation applied (through the same `runCounterfactual` seam - * the sweep uses). - * - * A repair is "what should have happened" ONLY when every validation - * replay crosses `flipThreshold` — a prescription is never speculated, - * it is demonstrated. Candidates that don't flip, or whose replay - * errors, land in `rejected` with a typed reason; nothing is dropped - * silently. - */ - -import { - type CounterfactualMutation, - type CounterfactualRunner, - runCounterfactual, -} from '../counterfactual' -import { ValidationError } from '../errors' -import type { TraceStore } from '../trace/store' -import { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory' -import type { StepRef, StepResponsibility } from './causal-sweep' - -/** Context handed to `proposeFix` so an LLM-backed proposer can see the - * full trajectory plus the responsibility evidence for the blamed step. */ -export interface RepairContext { - runId: string - trajectory: Trajectory - originalScore: number - responsibility: StepResponsibility -} - -export interface PrescribeRepairOptions { - store: TraceStore - /** The failed run the sweep diagnosed. */ - runId: string - /** Execution seam — same `CounterfactualRunner` contract as the sweep. */ - runner: CounterfactualRunner - /** Blamed steps from `causalSweep` — typically `report.steps.slice(0, k)`. */ - blamed: StepResponsibility[] - /** Candidate-fix generator. Consumer-supplied; LLM-backed in live use. - * Returned mutations MUST target the blamed step's index. */ - proposeFix: (step: TrajectoryStep, context: RepairContext) => Promise - /** Score every validation replay must reach for the repair to count. Default 0.5. */ - flipThreshold?: number - /** Validation replays per candidate mutation. Default 3. */ - repsToValidate?: number - /** Max candidate mutations tried per step. Default: all proposed. */ - maxAttemptsPerStep?: number -} - -export interface ValidatedRepair { - stepRef: StepRef - mutation: CounterfactualMutation - /** Always true — presence in `repairs` IS the machine-verified claim. */ - validated: true - /** Mean counterfactual score across the validation reps. */ - meanScore: number - /** meanScore − originalScore. */ - deltaScore: number - reps: number - /** Replay run ids backing the validation — audit trail. */ - counterfactualRunIds: string[] -} - -export interface RejectedRepair { - stepRef: StepRef - mutation: CounterfactualMutation - reason: 'did-not-flip' | 'error' - /** Present for 'did-not-flip': mean delta over the reps that ran. */ - deltaScore?: number - /** Present for 'error': the message, preserved for diagnosis. */ - error?: string -} - -export interface RepairReport { - runId: string - originalScore: number - flipThreshold: number - repairs: ValidatedRepair[] - rejected: RejectedRepair[] - replaysUsed: number -} - -export async function prescribeRepair(opts: PrescribeRepairOptions): Promise { - const flipThreshold = opts.flipThreshold ?? 0.5 - const repsToValidate = opts.repsToValidate ?? 3 - if (!Number.isInteger(repsToValidate) || repsToValidate < 1) { - throw new ValidationError( - `prescribeRepair: repsToValidate must be an integer >= 1 (got ${repsToValidate})`, - ) - } - const maxAttempts = opts.maxAttemptsPerStep ?? Number.POSITIVE_INFINITY - if (maxAttempts < 1) { - throw new ValidationError( - `prescribeRepair: maxAttemptsPerStep must be >= 1 (got ${opts.maxAttemptsPerStep})`, - ) - } - if (opts.blamed.length === 0) { - throw new ValidationError('prescribeRepair: blamed is empty — nothing to repair') - } - - const originalRun = await opts.store.getRun(opts.runId) - if (!originalRun) throw new ValidationError(`prescribeRepair: run ${opts.runId} not found`) - const originalScore = originalRun.outcome?.score - if (typeof originalScore !== 'number' || !Number.isFinite(originalScore)) { - throw new ValidationError( - `prescribeRepair: run ${opts.runId} has no numeric outcome.score — flips have no baseline`, - ) - } - - const trajectory = await buildTrajectory(opts.store, opts.runId) - - const repairs: ValidatedRepair[] = [] - const rejected: RejectedRepair[] = [] - let replaysUsed = 0 - - for (const responsibility of opts.blamed) { - const step = trajectory.steps[responsibility.stepRef.index] - if (!step || step.span.spanId !== responsibility.stepRef.spanId) { - throw new ValidationError( - `prescribeRepair: blamed step index=${responsibility.stepRef.index} spanId=${responsibility.stepRef.spanId} does not match run ${opts.runId} — stale report?`, - ) - } - - const candidates = await opts.proposeFix(step, { - runId: opts.runId, - trajectory, - originalScore, - responsibility, - }) - const toTry = candidates.slice(0, maxAttempts) - - for (const mutation of toTry) { - if (mutation.at !== step.index) { - throw new ValidationError( - `prescribeRepair: proposeFix returned a mutation targeting at=${mutation.at} for blamed step index=${step.index}`, - ) - } - const scores: number[] = [] - const cfRunIds: string[] = [] - let failure: string | undefined - for (let rep = 0; rep < repsToValidate; rep++) { - try { - const result = await runCounterfactual(opts.store, opts.runId, mutation, opts.runner) - replaysUsed++ - const score = result.delta.counterfactualOutcomeScore - if (typeof score !== 'number' || !Number.isFinite(score)) { - failure = `validation rep ${rep} produced no numeric score — the runner must endRun with a numeric outcome.score` - break - } - scores.push(score) - cfRunIds.push(result.counterfactualRunId) - } catch (err) { - replaysUsed++ - failure = err instanceof Error ? err.message : String(err) - break - } - } - - if (failure !== undefined) { - rejected.push({ - stepRef: responsibility.stepRef, - mutation, - reason: 'error', - error: failure, - }) - continue - } - - const meanScore = scores.reduce((a, b) => a + b, 0) / scores.length - const everyRepFlipped = scores.every((s) => s >= flipThreshold) - if (everyRepFlipped) { - repairs.push({ - stepRef: responsibility.stepRef, - mutation, - validated: true, - meanScore, - deltaScore: meanScore - originalScore, - reps: repsToValidate, - counterfactualRunIds: cfRunIds, - }) - // First validated repair per step IS the prescription; remaining - // candidates are untried, not rejected — we don't fabricate verdicts. - break - } - rejected.push({ - stepRef: responsibility.stepRef, - mutation, - reason: 'did-not-flip', - deltaScore: meanScore - originalScore, - }) - } - } - - return { runId: opts.runId, originalScore, flipThreshold, repairs, rejected, replaysUsed } -} diff --git a/src/governance/eu-ai-act.ts b/src/governance/eu-ai-act.ts deleted file mode 100644 index c3cedf9d..00000000 --- a/src/governance/eu-ai-act.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * EU AI Act — risk-class classification + compliance checklist. - * - * Classification is declarative: caller supplies the domain/use-case - * signals (biometric? critical infrastructure? education? employment? - * access to services?) and we map to the Act's risk tiers: - * - "unacceptable" (prohibited) - * - "high" (Annex III — strict obligations) - * - "limited" (transparency obligations) - * - "minimal" (voluntary codes of conduct) - * - * Then the compliance checklist enumerates Article 9 (risk mgmt), - * 10 (data + data governance), 11 (technical documentation), 13 - * (transparency), 14 (human oversight), 15 (accuracy + robustness) - * requirements and flags gaps. - */ - -import type { GovernanceContext, GovernanceFinding, GovernanceReport } from './types' -import { summarize } from './types' - -export type EuRiskClass = 'unacceptable' | 'high' | 'limited' | 'minimal' - -export interface UseCaseSignals { - /** Used for biometric identification in public spaces? (Art. 5 — unacceptable). */ - biometricPublic?: boolean - /** Social scoring by public authorities? (Art. 5). */ - socialScoring?: boolean - /** Subliminal manipulation? (Art. 5). */ - subliminal?: boolean - /** Annex III sector: critical infrastructure / education / employment / - * access to essential services / law enforcement / migration / - * administration of justice / democratic processes? */ - annexIII?: boolean - /** Interacts directly with natural persons (chatbot, agent)? — limited risk. */ - chatbot?: boolean - /** Generates synthetic media (image/audio/video/text deepfakes)? — limited risk. */ - generatesSyntheticMedia?: boolean -} - -export function classifyEuAiRisk(signals: UseCaseSignals): EuRiskClass { - if (signals.biometricPublic || signals.socialScoring || signals.subliminal) return 'unacceptable' - if (signals.annexIII) return 'high' - if (signals.chatbot || signals.generatesSyntheticMedia) return 'limited' - return 'minimal' -} - -export async function euAiActReport( - ctx: GovernanceContext, - signals: UseCaseSignals, -): Promise { - const riskClass = classifyEuAiRisk(signals) - const findings: GovernanceFinding[] = [] - - if (riskClass === 'unacceptable') { - findings.push({ - id: 'EU-ART-5', - severity: 'critical', - control: 'EU-AI-ACT:Article-5', - summary: 'Use case matches a prohibited practice under Article 5.', - remediation: 'Discontinue or substantially redesign the use case.', - }) - } - - if (riskClass === 'high') { - // Article 9 — risk management - if (!ctx.redTeam) { - findings.push({ - id: 'EU-ART-9', - severity: 'high', - control: 'EU-AI-ACT:Article-9', - summary: - 'High-risk system lacks documented adversarial-testing evidence (Art. 9 risk mgmt).', - remediation: 'Run redTeamDataset() + attach the report.', - }) - } - // Article 10 — data + data governance - if (ctx.datasets.length === 0) { - findings.push({ - id: 'EU-ART-10', - severity: 'high', - control: 'EU-AI-ACT:Article-10', - summary: 'No training/eval datasets recorded with provenance (Art. 10).', - }) - } - // Article 11 — technical documentation (traces + runs) - const runs = await ctx.traceStore.listRuns({ - since: Date.parse(ctx.periodStart), - until: Date.parse(ctx.periodEnd), - }) - if (runs.length === 0) { - findings.push({ - id: 'EU-ART-11', - severity: 'high', - control: 'EU-AI-ACT:Article-11', - summary: 'No eval runs recorded (Art. 11 technical documentation).', - }) - } - // Article 13 — transparency to users - if (!signals.chatbot && !signals.generatesSyntheticMedia) { - // High-risk but not a chatbot — transparency may still apply; flag informational - } else { - findings.push({ - id: 'EU-ART-13', - severity: 'info', - control: 'EU-AI-ACT:Article-13', - summary: - 'Chatbot/synthetic-media transparency obligations apply; verify user-facing disclosures.', - }) - } - // Article 14 — human oversight - if (!ctx.owner?.email) { - findings.push({ - id: 'EU-ART-14', - severity: 'high', - control: 'EU-AI-ACT:Article-14', - summary: 'No designated human overseer (Art. 14).', - remediation: 'Populate GovernanceContext.owner with the responsible individual.', - }) - } - // Article 15 — accuracy + robustness - if (!ctx.outcomeStore) { - findings.push({ - id: 'EU-ART-15', - severity: 'medium', - control: 'EU-AI-ACT:Article-15', - summary: 'No post-deployment outcome measurement; accuracy + robustness are un-attested.', - remediation: 'Attach an OutcomeStore + run correlationStudy() over the reporting period.', - }) - } - } - - if (riskClass === 'limited') { - findings.push({ - id: 'EU-ART-52', - severity: 'info', - control: 'EU-AI-ACT:Article-52', - summary: 'Transparency obligations apply: disclose AI nature + synthetic content labeling.', - remediation: 'Ensure user-facing surfaces label AI-generated content.', - }) - } - - const payload = { - riskClass, - signals, - articlesReviewed: - riskClass === 'high' - ? ['5', '9', '10', '11', '13', '14', '15'] - : riskClass === 'limited' - ? ['52'] - : ['none'], - } - - return { - framework: 'EU-AI-ACT', - version: 'Regulation-2024-1689', - context: { - organization: ctx.organization, - systemName: ctx.systemName, - periodStart: ctx.periodStart, - periodEnd: ctx.periodEnd, - owner: ctx.owner, - }, - summary: summarize(findings), - findings, - payload, - generatedAt: new Date().toISOString(), - } -} diff --git a/src/governance/index.ts b/src/governance/index.ts deleted file mode 100644 index 7068da90..00000000 --- a/src/governance/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './eu-ai-act' -export * from './nist-ai-rmf' -export * from './soc2' -export * from './types' diff --git a/src/governance/nist-ai-rmf.ts b/src/governance/nist-ai-rmf.ts deleted file mode 100644 index 7b5d00d5..00000000 --- a/src/governance/nist-ai-rmf.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * NIST AI RMF 1.0 — Govern / Map / Measure / Manage mapping. - * - * Each subcategory derives its status from concrete framework state: - * MEASURE 2.x: do we have a calibration regime? contamination controls? - * MEASURE 2.7: are red-team results available? - * MANAGE 1.x: are outcome metrics captured? correlation measured? - * GOVERN 1.x: dataset + prompt provenance recorded? - * - * We ship the mapping and the derivation rules; consumers supply the - * GovernanceContext. - */ - -import type { GovernanceContext, GovernanceFinding, GovernanceReport } from './types' -import { summarize } from './types' - -export async function nistAiRmfReport(ctx: GovernanceContext): Promise { - const findings: GovernanceFinding[] = [] - - // GOVERN 1.1 — "Accountable individual identified" - if (!ctx.owner?.email) { - findings.push({ - id: 'G-1.1', - severity: 'high', - control: 'NIST-AI-RMF:GOVERN-1.1', - summary: 'No responsible owner recorded for the AI system.', - remediation: 'Assign an accountable individual + email in GovernanceContext.owner.', - }) - } - - // GOVERN 1.3 — "Inventory + lifecycle tracking" - if (ctx.datasets.length === 0) { - findings.push({ - id: 'G-1.3', - severity: 'high', - control: 'NIST-AI-RMF:GOVERN-1.3', - summary: 'No versioned datasets recorded for the evaluation period.', - remediation: 'Register each dataset with a Dataset manifest (content hash + provenance).', - }) - } else { - // Validate content hashes are stable - for (const manifest of ctx.datasets) { - if (!manifest.contentHash || manifest.contentHash.length < 16) { - findings.push({ - id: 'G-1.3-hash', - severity: 'medium', - control: 'NIST-AI-RMF:GOVERN-1.3', - summary: `Dataset "${manifest.name}" has weak or missing content hash.`, - evidence: `contentHash="${manifest.contentHash}"`, - remediation: - 'Call dataset.manifest() to compute SHA-256; commit the manifest alongside releases.', - }) - } - } - } - - // MEASURE 2.6 — "Safety + adversarial testing" - if (!ctx.redTeam) { - findings.push({ - id: 'M-2.6', - severity: 'high', - control: 'NIST-AI-RMF:MEASURE-2.6', - summary: 'No red-team evaluation attached to the report period.', - remediation: - 'Run redTeamDataset() against the system and attach the RedTeamReport to context.redTeam.', - }) - } else if (ctx.redTeam.overallPassRate < 0.8) { - findings.push({ - id: 'M-2.6-rate', - severity: 'high', - control: 'NIST-AI-RMF:MEASURE-2.6', - summary: `Red-team pass rate ${(ctx.redTeam.overallPassRate * 100).toFixed(1)}% below 80% threshold.`, - evidence: JSON.stringify(ctx.redTeam.passRateByCategory), - remediation: 'Harden the failing categories; rerun the battery.', - }) - } - - // MEASURE 2.1 — "Test results against defined metrics" - const runs = await ctx.traceStore.listRuns({ - since: Date.parse(ctx.periodStart), - until: Date.parse(ctx.periodEnd), - }) - if (runs.length === 0) { - findings.push({ - id: 'M-2.1', - severity: 'critical', - control: 'NIST-AI-RMF:MEASURE-2.1', - summary: 'No eval runs recorded for the reporting period.', - remediation: 'Emit traces for every deployment-relevant evaluation.', - }) - } - - // MEASURE 2.11 — "Calibration + validation regime" - if (!ctx.judgeCalibration || ctx.judgeCalibration.length === 0) { - findings.push({ - id: 'M-2.11', - severity: 'medium', - control: 'NIST-AI-RMF:MEASURE-2.11', - summary: 'No judge-vs-human calibration recorded.', - remediation: - 'Build a human golden set; run calibrateJudge() before trusting LLM judge scores.', - }) - } else { - const weak = ctx.judgeCalibration.filter((c) => Number.isFinite(c.pearson) && c.pearson < 0.6) - if (weak.length > 0) { - findings.push({ - id: 'M-2.11-weak', - severity: 'medium', - control: 'NIST-AI-RMF:MEASURE-2.11', - summary: `${weak.length} judge(s) show weak agreement with humans (Pearson < 0.6).`, - remediation: 'Retrain or replace the underperforming judges.', - }) - } - } - - // MANAGE 1.1 — "Outcomes tracked post-deployment" - if (!ctx.outcomeStore) { - findings.push({ - id: 'MN-1.1', - severity: 'medium', - control: 'NIST-AI-RMF:MANAGE-1.1', - summary: 'No deployment outcomes captured — meta-eval correlation cannot be computed.', - remediation: 'Attach an OutcomeStore and ingest production outcome metrics.', - }) - } else { - const outcomes = await ctx.outcomeStore.list({ - since: Date.parse(ctx.periodStart), - until: Date.parse(ctx.periodEnd), - }) - if (outcomes.length === 0) { - findings.push({ - id: 'MN-1.1-empty', - severity: 'medium', - control: 'NIST-AI-RMF:MANAGE-1.1', - summary: 'OutcomeStore present but no outcomes captured for the period.', - }) - } - } - - // Validate that dataset manifests carry strong SHA-256-shaped content hashes. - const hashChecks: Array<{ name: string; ok: boolean }> = [] - for (const manifest of ctx.datasets) { - // We don't persist the scenarios here; the check is that the caller's - // manifest already carries a hash in the expected hex format. - hashChecks.push({ name: manifest.name, ok: /^[0-9a-f]{64}$/.test(manifest.contentHash) }) - } - - const payload = { - controlsEvaluated: [ - 'GOVERN-1.1', - 'GOVERN-1.3', - 'MEASURE-2.1', - 'MEASURE-2.6', - 'MEASURE-2.11', - 'MANAGE-1.1', - ], - runCount: runs.length, - redTeamPassRate: ctx.redTeam?.overallPassRate ?? null, - datasetHashChecks: hashChecks, - } - - return { - framework: 'NIST-AI-RMF', - version: '1.0.0', - context: { - organization: ctx.organization, - systemName: ctx.systemName, - periodStart: ctx.periodStart, - periodEnd: ctx.periodEnd, - owner: ctx.owner, - }, - summary: summarize(findings), - findings, - payload, - generatedAt: new Date().toISOString(), - } -} diff --git a/src/governance/soc2.ts b/src/governance/soc2.ts deleted file mode 100644 index 5bb3154d..00000000 --- a/src/governance/soc2.ts +++ /dev/null @@ -1,123 +0,0 @@ -/** - * SOC 2 — Common Criteria 7 (system operations + change management) - * audit trail derived from the trace corpus. - * - * This is NOT a formal SOC2 report — that requires an external - * auditor. What we ship is the machine-readable *evidence* package - * that an auditor consumes: run counts, deploy events, access log - * summary, anomaly tracking, response-time SLOs. - */ - -import type { GovernanceContext, GovernanceFinding, GovernanceReport } from './types' -import { summarize } from './types' - -export async function soc2Report(ctx: GovernanceContext): Promise { - const findings: GovernanceFinding[] = [] - const start = Date.parse(ctx.periodStart) - const end = Date.parse(ctx.periodEnd) - const runs = await ctx.traceStore.listRuns({ since: start, until: end }) - - // CC7.1 — "Monitoring to detect anomalies" - const failureRate = - runs.length > 0 ? runs.filter((r) => r.outcome?.pass === false).length / runs.length : null - if (failureRate !== null && failureRate > 0.2) { - findings.push({ - id: 'CC7.1-fail-rate', - severity: 'medium', - control: 'SOC2:CC7.1', - summary: `System failure rate ${(failureRate * 100).toFixed(1)}% over the period exceeds 20%.`, - remediation: 'Investigate failure clusters (failureClusterView) + prioritize remediation.', - }) - } - if (runs.length === 0) { - findings.push({ - id: 'CC7.1-coverage', - severity: 'high', - control: 'SOC2:CC7.1', - summary: 'No telemetry runs recorded for the period — monitoring regime is incomplete.', - }) - } - - // CC7.2 — "Anomaly investigation" - const aborted = runs.filter((r) => r.status === 'aborted') - if (aborted.length > runs.length * 0.05 && aborted.length >= 3) { - findings.push({ - id: 'CC7.2-abort', - severity: 'medium', - control: 'SOC2:CC7.2', - summary: `${aborted.length} run(s) aborted — investigate pattern.`, - remediation: 'Use the bisector + failureClusterView to localize the trigger.', - }) - } - - // CC7.3 — "Response to incidents" — require an event tag for resolved incidents - const incidentEvents = await ctx.traceStore.events({ - kind: 'policy_violation', - since: start, - until: end, - }) - const errorEvents = await ctx.traceStore.events({ kind: 'error', since: start, until: end }) - const totalIncidents = incidentEvents.length + errorEvents.length - if (totalIncidents > 0) { - // No formal resolution tracking yet — flag medium by default - findings.push({ - id: 'CC7.3-resolution', - severity: 'low', - control: 'SOC2:CC7.3', - summary: `${totalIncidents} incident-class event(s) recorded; resolution tracking is informal.`, - remediation: - 'Emit a resolution event (kind="log" with payload.resolves=) per remediated incident.', - }) - } - - // CC7.4 — "Configuration change tracking" - const modelFingerprints = new Set(runs.map((r) => r.modelFingerprint).filter(Boolean) as string[]) - const promptHashes = new Set(runs.map((r) => r.promptSha).filter(Boolean) as string[]) - const codeSha = new Set(runs.map((r) => r.codeSha).filter(Boolean) as string[]) - if (codeSha.size === 0) { - findings.push({ - id: 'CC7.4-code', - severity: 'high', - control: 'SOC2:CC7.4', - summary: 'No codeSha recorded on runs — cannot attribute scores to a specific release.', - remediation: 'Populate Run.codeSha with the git SHA of the system at run time.', - }) - } - if (promptHashes.size === 0) { - findings.push({ - id: 'CC7.4-prompt', - severity: 'medium', - control: 'SOC2:CC7.4', - summary: 'No promptSha recorded — prompt changes are untracked.', - }) - } - - const payload = { - controls: ['CC7.1', 'CC7.2', 'CC7.3', 'CC7.4'], - runCount: runs.length, - failureRate, - abortedCount: aborted.length, - incidentEventCount: totalIncidents, - distinctReleases: { - codeShas: codeSha.size, - promptHashes: promptHashes.size, - modelFingerprints: modelFingerprints.size, - }, - } - - return { - framework: 'SOC2', - version: '2017-Common-Criteria', - context: { - organization: ctx.organization, - systemName: ctx.systemName, - periodStart: ctx.periodStart, - periodEnd: ctx.periodEnd, - owner: ctx.owner, - }, - summary: summarize(findings), - findings, - payload, - generatedAt: new Date().toISOString(), - } -} diff --git a/src/governance/types.ts b/src/governance/types.ts deleted file mode 100644 index e4b90f07..00000000 --- a/src/governance/types.ts +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Governance reporting — shared types. - * - * The framework collects a `GovernanceContext` (traces + outcomes + - * dataset manifests + red-team results + judge calibration) and each - * specific template (NIST AI RMF, SOC2, EU AI Act) renders a - * structured report from it. - * - * Reports are machine-readable JSON first; human-readable Markdown is a - * pure transform on top. External auditors consume the Markdown; CI - * consumes the JSON. - */ - -import type { DatasetManifest } from '../dataset' -import type { CalibrationResult } from '../judge-calibration' -import type { OutcomeStore } from '../meta-eval/outcome-store' -import type { RedTeamReport } from '../red-team' -import type { TraceStore } from '../trace/store' - -export interface GovernanceContext { - /** Legal / org identity for the report. */ - organization: string - /** System / agent identifier. */ - systemName: string - /** ISO8601 period the report covers. */ - periodStart: string - periodEnd: string - /** Versioned dataset manifests used during the period. */ - datasets: DatasetManifest[] - traceStore: TraceStore - outcomeStore?: OutcomeStore - /** Cached red-team results for the period, if available. */ - redTeam?: RedTeamReport - /** Judge-vs-human calibration results, if measured. */ - judgeCalibration?: CalibrationResult[] - /** Responsible owner for the system — role + name + email. */ - owner: { role: string; name: string; email: string } -} - -export interface GovernanceFinding { - id: string - severity: 'info' | 'low' | 'medium' | 'high' | 'critical' - /** Control reference the finding maps to (e.g. "NIST-AI-RMF:MEASURE-2.1"). */ - control: string - summary: string - evidence?: string - remediation?: string -} - -export interface GovernanceReport { - framework: 'NIST-AI-RMF' | 'SOC2' | 'EU-AI-ACT' - version: string - context: Pick< - GovernanceContext, - 'organization' | 'systemName' | 'periodStart' | 'periodEnd' | 'owner' - > - summary: { - findings: number - byeverity: Record - overall: 'compliant' | 'compliant-with-findings' | 'non-compliant' - } - findings: GovernanceFinding[] - /** Framework-specific structured payload (mapped controls, risk class, etc.). */ - payload: Record - generatedAt: string -} - -export function renderMarkdown(report: GovernanceReport): string { - const sevEmoji: Record = { - info: 'ℹ︎', - low: '·', - medium: '!', - high: '!!', - critical: '‼', - } - const lines: string[] = [] - lines.push(`# ${report.framework} report — ${report.context.systemName}`) - lines.push('') - lines.push(`- Organization: **${report.context.organization}**`) - lines.push(`- Period: ${report.context.periodStart} → ${report.context.periodEnd}`) - lines.push( - `- Owner: ${report.context.owner.role} ${report.context.owner.name} <${report.context.owner.email}>`, - ) - lines.push(`- Generated: ${report.generatedAt}`) - lines.push('') - lines.push(`## Summary — ${report.summary.overall}`) - lines.push('') - lines.push(`${report.summary.findings} finding(s).`) - for (const [sev, n] of Object.entries(report.summary.byeverity) as Array< - [GovernanceFinding['severity'], number] - >) { - if (n > 0) lines.push(`- ${sevEmoji[sev]} ${sev}: ${n}`) - } - lines.push('') - lines.push('## Findings') - lines.push('') - for (const f of report.findings) { - lines.push(`### ${sevEmoji[f.severity]} ${f.id} — ${f.control}`) - lines.push('') - lines.push(f.summary) - if (f.evidence) { - lines.push('') - lines.push(`**Evidence:** ${f.evidence}`) - } - if (f.remediation) { - lines.push('') - lines.push(`**Remediation:** ${f.remediation}`) - } - lines.push('') - } - return lines.join('\n') -} - -export function summarize(findings: GovernanceFinding[]): GovernanceReport['summary'] { - const byeverity: GovernanceReport['summary']['byeverity'] = { - info: 0, - low: 0, - medium: 0, - high: 0, - critical: 0, - } - for (const f of findings) byeverity[f.severity]++ - const overall: GovernanceReport['summary']['overall'] = - byeverity.critical + byeverity.high > 0 - ? 'non-compliant' - : byeverity.medium + byeverity.low > 0 - ? 'compliant-with-findings' - : 'compliant' - return { findings: findings.length, byeverity, overall } -} diff --git a/src/groundedness/index.test.ts b/src/groundedness/index.test.ts deleted file mode 100644 index f6d75a67..00000000 --- a/src/groundedness/index.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { Span } from '../trace/schema' -import { - defaultProviderToolMatcher, - extractRetrievedText, - scoreGroundedness, - scoreGroundednessForRun, -} from './index' - -describe('scoreGroundedness', () => { - it('scores the share of required knowledge the provider surfaced, case-insensitively', () => { - const text = 'The current API uses createMiddleware from hono/factory and streamSSE.' - const r = scoreGroundedness(text, ['createMiddleware', 'streamSSE', 'getRuntimeKey']) - expect(r.total).toBe(3) - expect(r.found.sort()).toEqual(['createMiddleware', 'streamSSE']) - expect(r.missing).toEqual(['getRuntimeKey']) - expect(r.score).toBeCloseTo(2 / 3) - expect(r.hadResults).toBe(true) - }) - - it('matches case-insensitively but reports keys in their original casing', () => { - const r = scoreGroundedness('imports * as Z from ZOD', ['z', 'zod']) - expect(r.found).toEqual(['z', 'zod']) - expect(r.score).toBe(1) - }) - - it('dedupes required keys (case-insensitive) so the denominator cannot be inflated', () => { - const r = scoreGroundedness('uses viem', ['viem', 'VIEM', ' viem ']) - expect(r.total).toBe(1) - expect(r.score).toBe(1) - }) - - it('fails open when there is no required knowledge (nothing to ground)', () => { - const r = scoreGroundedness('', []) - expect(r.score).toBe(1) - expect(r.total).toBe(0) - expect(r.hadResults).toBe(false) - }) - - it('distinguishes "no results" from "results that missed the facts"', () => { - const empty = scoreGroundedness('', ['useReadContract']) - expect(empty.hadResults).toBe(false) - expect(empty.score).toBe(0) - - const missed = scoreGroundedness('here is some unrelated prose', ['useReadContract']) - expect(missed.hadResults).toBe(true) - expect(missed.score).toBe(0) - }) -}) - -describe('extractRetrievedText', () => { - const base = { runId: 'r1', name: 'n', startedAt: 0 } as const - - it('reads RetrievalSpan hit content', () => { - const spans: Span[] = [ - { - ...base, - spanId: 's1', - kind: 'retrieval', - query: 'hono factory', - hits: [ - { docId: 'd1', score: 0.9, content: 'createMiddleware from hono/factory' }, - { docId: 'd2', score: 0.4, content: 'streamSSE from hono/streaming' }, - { docId: 'd3', score: 0.1 }, // no content — skipped, not crashed - ], - }, - ] - const text = extractRetrievedText(spans) - expect(text).toContain('createMiddleware') - expect(text).toContain('streamSSE') - }) - - it('reads provider ToolSpan results by the default matcher, skipping fetch + non-provider tools', () => { - const spans: Span[] = [ - { - ...base, - spanId: 's1', - kind: 'tool', - toolName: 'web_search', - args: { q: 'viem v2' }, - result: { snippets: ['useReadContract is current'] }, - }, - { - ...base, - spanId: 's2', - kind: 'tool', - toolName: 'fetch_url', // search/research-not-fetch default excludes this - args: {}, - result: 'getContract legacy', - }, - { - ...base, - spanId: 's3', - kind: 'tool', - toolName: 'write_file', // not a provider tool at all - args: {}, - result: 'irrelevant', - }, - ] - const text = extractRetrievedText(spans) - expect(text).toContain('useReadContract') - expect(text).not.toContain('getContract legacy') - expect(text).not.toContain('irrelevant') - }) - - it('honors an injected provider matcher (no benchmark literal baked in)', () => { - const spans: Span[] = [ - { - ...base, - spanId: 's1', - kind: 'tool', - toolName: 'youcom', - args: {}, - result: 'surfaced fact', - }, - ] - const isProviderTool = (name: string) => name === 'youcom' - expect(extractRetrievedText(spans, { isProviderTool })).toContain('surfaced fact') - // default matcher would NOT pick up 'youcom' - expect(extractRetrievedText(spans)).toBe('') - }) - - it('default matcher accepts search/research and rejects fetch', () => { - expect(defaultProviderToolMatcher('web_search')).toBe(true) - expect(defaultProviderToolMatcher('deep_research')).toBe(true) - expect(defaultProviderToolMatcher('fetch_url')).toBe(false) - expect(defaultProviderToolMatcher('read_file')).toBe(false) - }) -}) - -describe('scoreGroundednessForRun', () => { - it('extracts provider text from spans then scores it in one call', () => { - const base = { runId: 'r1', name: 'n', startedAt: 0 } as const - const spans: Span[] = [ - { - ...base, - spanId: 's1', - kind: 'retrieval', - query: 'wagmi v2', - hits: [{ docId: 'd1', score: 0.9, content: 'useReadContract and useWriteContract' }], - }, - ] - const r = scoreGroundednessForRun(spans, [ - 'useReadContract', - 'useWriteContract', - 'useContractRead', - ]) - expect(r.found.sort()).toEqual(['useReadContract', 'useWriteContract']) - expect(r.missing).toEqual(['useContractRead']) - expect(r.score).toBeCloseTo(2 / 3) - }) -}) diff --git a/src/groundedness/index.ts b/src/groundedness/index.ts deleted file mode 100644 index 75c233b0..00000000 --- a/src/groundedness/index.ts +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Groundedness — "did the retrieval PROVIDER surface what the task needed?" - * - * A search/research provider returns text; the task needed certain facts or - * symbols to be solvable (the CURRENT API, a version number, a function name). - * This module scores how much of that required knowledge the provider's results - * actually surfaced — isolating PROVIDER quality (was the right thing - * retrievable / returned) from AGENT skill (did the agent then use it). A high - * groundedness score with a failed run blames the agent; a low score blames the - * provider. That separation is the whole point — pass/fail alone cannot make it. - * - * Structural sibling of `../authenticity`: - * - authenticity scores the agent's PRODUCED files for realness. - * - groundedness scores the provider's RETRIEVED text for coverage. - * Both are pure deterministic scorers whose DOMAIN config is supplied by the - * consumer (authenticity: `AuthenticitySignals`; groundedness: - * `requiredKnowledge: string[]`) — neither bakes in a benchmark's vocabulary. - * - * Relationship to `keyword-coverage-judge`: that judge scores the agent's - * SERVED OUTPUT (HTML + assets) for expected concepts — a different input - * (produced deliverable) answering a different question (deliverable quality). - * Groundedness reads the RETRIEVAL side (provider results). They are - * complementary coverage scorers over different stages of the run, not - * duplicates; do not collapse one into the other. - * - * Two seams, neither forked: - * - PURE SCORER `scoreGroundedness(resultText, requiredKnowledge)` — case- - * insensitive substring containment over a deduped key set. Fail-open: with - * no required knowledge there is nothing to ground, so `score = 1`. - * - TRACE EXTRACTOR `extractRetrievedText(spans, opts?)` — pulls the provider's - * returned text out of the canonical `TraceSchema` spans (`RetrievalSpan.hits` - * + provider `ToolSpan.result`) instead of re-parsing bespoke run files. This - * is the retrieval-side analog of `extractProducedState` (events → produced - * files): structural span input, no IO, no disk walking. - */ - -import type { RetrievalSpan, Span, ToolSpan } from '../trace/schema' -import { isRetrievalSpan, isToolSpan } from '../trace/schema' - -// ── Pure scorer ────────────────────────────────────────────────────────────── - -export interface GroundednessResult { - /** 0..1 share of required knowledge surfaced by the provider's results. - * 1 when there is nothing to ground (`requiredKnowledge` empty) — fail-open. */ - score: number - /** The required-knowledge keys the result text surfaced (deduped, original casing). */ - found: string[] - /** The required-knowledge keys the result text did NOT surface. */ - missing: string[] - /** Distinct required-knowledge keys after dedup — the denominator of `score`. */ - total: number - /** Did the provider return any result text at all? Distinguishes "provider - * surfaced nothing" (`!hadResults`) from "returned text but missed the facts" - * (`hadResults && score < 1`) — the same provider-vs-agent split as the score. */ - hadResults: boolean -} - -/** - * Dedup a knowledge-key list, case-insensitively, keeping first-seen casing and - * dropping blanks. The score denominator is distinct keys, so a config that - * lists the same symbol twice (or with different casing) can't inflate `total`. - */ -function dedupeKeys(keys: readonly string[]): string[] { - const seen = new Set() - const out: string[] = [] - for (const raw of keys) { - const k = raw.trim() - if (!k) continue - const lower = k.toLowerCase() - if (seen.has(lower)) continue - seen.add(lower) - out.push(k) - } - return out -} - -/** - * Score how much of `requiredKnowledge` the retrieval provider's `resultText` - * surfaced. Pure — same inputs, same output. No IO, no LLM. - * - * Matching is case-insensitive substring containment: each required key is - * checked against the lower-cased result text. This is intentionally the same - * cheap, deterministic containment the authenticity scorer uses for its - * structural signals — a key is "surfaced" if the provider's returned text - * mentions it. Semantic / paraphrase coverage is a separate (LLM) layer a - * consumer can stack on top, exactly as authenticity stacks its nuance judge. - * - * Fail-open at `total === 0`: a task with no required knowledge has nothing for - * the provider to ground, so it cannot be penalized (`score = 1`). The benchmark - * caller decides what `requiredKnowledge` is — the substrate never derives it. - */ -export function scoreGroundedness( - resultText: string, - requiredKnowledge: readonly string[], -): GroundednessResult { - const keys = dedupeKeys(requiredKnowledge) - const total = keys.length - const text = resultText ?? '' - const hadResults = text.trim().length > 0 - const haystack = text.toLowerCase() - - if (total === 0) { - return { score: 1, found: [], missing: [], total: 0, hadResults } - } - - const found: string[] = [] - const missing: string[] = [] - for (const key of keys) { - if (haystack.includes(key.toLowerCase())) found.push(key) - else missing.push(key) - } - - return { score: found.length / total, found, missing, total, hadResults } -} - -// ── Trace extractor ──────────────────────────────────────────────────────── - -/** - * Predicate selecting which `ToolSpan`s are retrieval-PROVIDER calls (whose - * `result` carries returned text), by tool name. A parameter — never a baked-in - * literal — so the substrate stays free of any one benchmark's tool vocabulary, - * exactly as `AuthenticitySignals` keeps all domain regexes consumer-supplied. - */ -export type ProviderToolMatcher = (toolName: string) => boolean - -/** - * Default provider matcher: tool names that look like search/research but not a - * plain fetch/read. A sensible starting point for the common "search arm" shape; - * any consumer with different tool names passes its own matcher. `RetrievalSpan`s - * are ALWAYS included regardless of this matcher (they are retrieval by kind); - * the matcher only selects which generic `ToolSpan`s also count as provider calls. - */ -export const defaultProviderToolMatcher: ProviderToolMatcher = (name) => - /search|research/i.test(name) && !/fetch/i.test(name) - -export interface ExtractRetrievedTextOptions { - /** Which `ToolSpan`s count as provider calls. Default: {@link defaultProviderToolMatcher}. */ - isProviderTool?: ProviderToolMatcher -} - -/** Stringify a `ToolSpan.result` of unknown shape into searchable text. */ -function resultToText(result: unknown): string { - if (result == null) return '' - if (typeof result === 'string') return result - try { - return JSON.stringify(result) - } catch { - return String(result) - } -} - -/** Pull the retrieved text out of a `RetrievalSpan`: every hit's `content`. */ -function retrievalSpanText(span: RetrievalSpan): string { - return span.hits - .map((h) => h.content ?? '') - .filter((c) => c.length > 0) - .join('\n') -} - -/** - * Extract the retrieval PROVIDER's returned text from a span stream — the - * retrieval-side analog of `extractProducedState`. Reads the canonical - * `TraceSchema` carriers, NOT bespoke run files: - * - every `RetrievalSpan`'s `hits[].content` (kind 'retrieval' — the - * substrate's first-class search/research result carrier; the same `.hits` - * the `bad_retrieval` failure detector already reads), and - * - `ToolSpan.result` for tool spans whose `toolName` the provider matcher - * accepts (kind 'tool'). - * - * Pure and total: spans of other kinds, and provider tools with no result, are - * skipped. Returns one text blob ready for `scoreGroundedness`. - */ -export function extractRetrievedText( - spans: readonly Span[], - opts: ExtractRetrievedTextOptions = {}, -): string { - const isProviderTool = opts.isProviderTool ?? defaultProviderToolMatcher - const parts: string[] = [] - for (const span of spans) { - if (isRetrievalSpan(span)) { - const t = retrievalSpanText(span) - if (t) parts.push(t) - } else if (isToolSpan(span)) { - const ts = span as ToolSpan - if (isProviderTool(ts.toolName)) { - const t = resultToText(ts.result) - if (t) parts.push(t) - } - } - } - return parts.join('\n') -} - -// ── Convenience: extract-then-score ─────────────────────────────────────────── - -/** - * Extract the provider's retrieved text from a run's spans and score it against - * `requiredKnowledge` in one call — the analog of authenticity's file-in - * convenience. The primary contract is the standalone `scoreGroundedness`; this - * is the ergonomic path for a consumer holding a persisted run's `Span[]` - * (e.g. from `TraceStore.spans(...)`). - */ -export function scoreGroundednessForRun( - spans: readonly Span[], - requiredKnowledge: readonly string[], - opts: ExtractRetrievedTextOptions = {}, -): GroundednessResult { - return scoreGroundedness(extractRetrievedText(spans, opts), requiredKnowledge) -} diff --git a/src/index.ts b/src/index.ts index 5f797085..a1e1e0c0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -652,7 +652,7 @@ export { export * from './trace' -// `knowledge`, `governance`, and `trace` remain re-exported at root because +// `knowledge` and `trace` remain re-exported at root because // they're load-bearing for the capture-integrity story documented in the // README. Every other module is reachable only through its subpath // (`/rl`, `/pipelines`, `/meta-eval`, `/prm`, `/builder-eval`, `/traces`). @@ -1035,10 +1035,6 @@ export type { } from './self-play' export { runSelfPlay } from './self-play' -// ── Governance templates ───────────────────────────────────────────── - -export * from './governance' - // ── LLM client, multi-layer verifier, semantic concept judge, error-count ─ export type { @@ -1472,25 +1468,6 @@ export { ATTESTATION_ALGORITHM, attest, verifyAttestation } from './attestation' // percentile ratchet (summarize → baseline → gate). Scores LATENCY / // RELIABILITY over flat metric records; the judge-panel BenchmarkRunner // (./benchmark) scores QUALITY. Also on the `/perf` subpath. -export type { - IntegrityResult, - IntegrityViolation, - JourneySpec, - PerfBaseline, - PerfGateResult, - PerfRegression, - PerfScenario, - PerfStat, - ScenarioAxes, -} from './perf' -export { - assertRecordIntegrity, - checkRecordIntegrity, - expandMatrix, - gatePerf, - scenarioKey, - summarizeRecords, -} from './perf' // Product-owned benchmark bundles: portable product runs for agent-lab research. export type { AgentProfileRuntimeReceipt, diff --git a/src/perf/index.ts b/src/perf/index.ts deleted file mode 100644 index ab93d11d..00000000 --- a/src/perf/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * @tangle-network/agent-eval/perf - * - * Domain-agnostic infra-performance benchmarking substrate: a journeys × - * axes scenario matrix, record-integrity contracts over flat metric - * records, and a percentile ratchet (summarize → baseline → gate). - * - * Complements the judge-panel `BenchmarkRunner` (root): that one scores - * QUALITY; this one scores LATENCY / RELIABILITY over flat metric records. - */ - -export type { IntegrityResult, IntegrityViolation } from './integrity' -export { assertRecordIntegrity, checkRecordIntegrity } from './integrity' -export type { JourneySpec, PerfScenario, ScenarioAxes } from './journey' -export { expandMatrix, scenarioKey } from './journey' -export type { PerfBaseline, PerfGateResult, PerfRegression, PerfStat } from './ratchet' -export { gatePerf, summarizeRecords } from './ratchet' diff --git a/src/perf/integrity.ts b/src/perf/integrity.ts deleted file mode 100644 index 9c00d93b..00000000 --- a/src/perf/integrity.ts +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Record-integrity contracts for perf metric records. - * - * A record that claims `pass === true` must actually carry the journey's - * required measurements — a "passing" provision run with a null - * `total_ms` is a lying record, not a pass. Failed records are exempt: - * a run that errored mid-flight legitimately has nulls. - */ - -import type { JourneySpec } from './journey' - -export interface IntegrityViolation { - recordIndex: number - journeyId: string - field: string - reason: 'null-required-field' | 'below-minimum' - detail: string -} - -export interface IntegrityResult { - succeeded: boolean - violations: IntegrityViolation[] -} - -function isMissing(value: unknown): boolean { - return value === null || value === undefined -} - -/** - * Validates flat metric records (Record with a boolean - * `pass` field) against their journey contract. Only records with - * pass === true are checked — a failed record may legitimately have nulls. - * resolveJourney maps a record to its JourneySpec (or null to skip). - */ -export function checkRecordIntegrity( - records: ReadonlyArray>, - resolveJourney: (record: Record) => JourneySpec | null, -): IntegrityResult { - const violations: IntegrityViolation[] = [] - for (const [recordIndex, record] of records.entries()) { - if (record.pass !== true) continue - const journey = resolveJourney(record) - if (journey === null) continue - for (const field of journey.requiredFields) { - if (isMissing(record[field])) { - violations.push({ - recordIndex, - journeyId: journey.id, - field, - reason: 'null-required-field', - detail: `required field '${field}' is ${record[field] === null ? 'null' : 'undefined'} on a passing '${journey.id}' record`, - }) - } - } - for (const field of journey.phaseFields ?? []) { - if (isMissing(record[field])) { - violations.push({ - recordIndex, - journeyId: journey.id, - field, - reason: 'null-required-field', - detail: `phase field '${field}' is ${record[field] === null ? 'null' : 'undefined'} on a passing '${journey.id}' record`, - }) - } - } - for (const { field, min } of journey.minimums ?? []) { - const value = record[field] - if (isMissing(value)) continue // null-ness is the required/phase fields' contract - if (typeof value !== 'number' || Number.isNaN(value)) { - violations.push({ - recordIndex, - journeyId: journey.id, - field, - reason: 'below-minimum', - detail: `field '${field}' has non-numeric value ${JSON.stringify(value)} on a passing '${journey.id}' record (minimum ${min})`, - }) - continue - } - if (value < min) { - violations.push({ - recordIndex, - journeyId: journey.id, - field, - reason: 'below-minimum', - detail: `field '${field}' is ${value}, below minimum ${min} on a passing '${journey.id}' record`, - }) - } - } - } - return { succeeded: violations.length === 0, violations } -} - -/** Throws an Error listing every violation when the result fails. */ -export function assertRecordIntegrity( - records: ReadonlyArray>, - resolveJourney: (record: Record) => JourneySpec | null, -): void { - const result = checkRecordIntegrity(records, resolveJourney) - if (result.succeeded) return - const lines = result.violations.map( - (v) => ` [record ${v.recordIndex}] ${v.journeyId}.${v.field} (${v.reason}): ${v.detail}`, - ) - throw new Error( - `Record integrity check failed with ${result.violations.length} violation(s):\n${lines.join('\n')}`, - ) -} diff --git a/src/perf/journey.ts b/src/perf/journey.ts deleted file mode 100644 index af9ff5be..00000000 --- a/src/perf/journey.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Journey × axes matrix for infra performance benchmarks. - * - * A journey is one measurable user path ("provision.cold", "chat.ttft"); - * axes are free-form scenario dimensions (driver, region, image…). The - * matrix expansion is pure bookkeeping — running the scenarios and - * recording metrics is the caller's job. This module complements the - * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores - * QUALITY via judges, this one structures LATENCY / RELIABILITY runs - * over flat metric records. - */ - -/** One measurable user journey (e.g. "provision.cold", "chat.ttft"). */ -export interface JourneySpec { - id: string - description: string - /** Needs a real LLM call — schedule nightly, not per-PR. */ - requiresLLM: boolean - /** - * Fields that MUST be non-null on a passing record of this journey. - * A "passing" record missing one is an integrity violation, not a pass. - */ - requiredFields: ReadonlyArray - /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */ - minimums?: ReadonlyArray<{ field: string; min: number }> - /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */ - phaseFields?: ReadonlyArray -} - -export interface ScenarioAxes { - /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */ - [dimension: string]: ReadonlyArray -} - -export interface PerfScenario { - /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */ - key: string - journey: JourneySpec - axes: Record -} - -/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */ -export function scenarioKey(journeyId: string, axes: Record): string { - const parts = Object.keys(axes) - .sort() - .map((dim) => `${dim}=${axes[dim]}`) - return [journeyId, ...parts].join('|') -} - -/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */ -export function expandMatrix( - journeys: ReadonlyArray, - axes: ScenarioAxes, - filter?: (journeyId: string, combo: Record) => boolean, -): PerfScenario[] { - const dims = Object.keys(axes).sort() - let combos: Record[] = [{}] - for (const dim of dims) { - const values = axes[dim] as ReadonlyArray - const next: Record[] = [] - for (const combo of combos) { - for (const value of values) { - next.push({ ...combo, [dim]: value }) - } - } - combos = next - } - const scenarios: PerfScenario[] = [] - for (const journey of journeys) { - for (const combo of combos) { - if (filter && !filter(journey.id, combo)) continue - scenarios.push({ key: scenarioKey(journey.id, combo), journey, axes: combo }) - } - } - return scenarios -} diff --git a/src/perf/ratchet.ts b/src/perf/ratchet.ts deleted file mode 100644 index a5ce8edb..00000000 --- a/src/perf/ratchet.ts +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Percentile ratchet over perf metric records. - * - * `summarizeRecords` folds flat records into per-scenario p50/p90 stats; - * `gatePerf` compares a current summary against a committed baseline and - * trips on regressions beyond tolerance. Percentiles use nearest-rank on - * sorted values. Null / non-numeric metric values are excluded from the - * stat (n reflects only real samples); a field with zero samples is - * omitted entirely — no fake zeros. - */ - -export interface PerfStat { - p50: number - p90: number - n: number -} - -export interface PerfBaseline { - version: 1 - /** key → metric field → stat. */ - scenarios: Record> -} - -export interface PerfRegression { - scenarioKey: string - field: string - baseline: PerfStat - current: PerfStat - /** percent over baseline p50 / p90, whichever tripped. */ - overBy: { p50Pct: number; p90Pct: number } -} - -export interface PerfGateResult { - succeeded: boolean - regressions: PerfRegression[] - /** Negative overBy: strictly better than baseline on both percentiles. */ - improvements: PerfRegression[] - /** In baseline but absent (or under-sampled, n < minSamples) in current. */ - missingScenarios: string[] - /** In records but absent from baseline. */ - newScenarios: string[] -} - -/** Nearest-rank percentile over a non-empty ascending-sorted array (p in (0, 100]). */ -function nearestRank(sorted: ReadonlyArray, p: number): number { - if (sorted.length === 0) throw new Error('nearestRank requires at least one sample') - const rank = Math.max(1, Math.ceil((p / 100) * sorted.length)) - return sorted[rank - 1] as number -} - -export function summarizeRecords( - records: ReadonlyArray>, - keyOf: (record: Record) => string | null, - metricFields: ReadonlyArray, -): PerfBaseline { - const samples = new Map>() - for (const record of records) { - const key = keyOf(record) - if (key === null) continue - let byField = samples.get(key) - if (!byField) { - byField = new Map() - samples.set(key, byField) - } - for (const field of metricFields) { - const value = record[field] - if (typeof value !== 'number' || Number.isNaN(value)) continue - let values = byField.get(field) - if (!values) { - values = [] - byField.set(field, values) - } - values.push(value) - } - } - const scenarios: Record> = {} - for (const [key, byField] of samples) { - const stats: Record = {} - for (const [field, values] of byField) { - if (values.length === 0) continue - const sorted = [...values].sort((a, b) => a - b) - stats[field] = { - p50: nearestRank(sorted, 50), - p90: nearestRank(sorted, 90), - n: sorted.length, - } - } - scenarios[key] = stats - } - return { version: 1, scenarios } -} - -/** Percent over baseline; baseline 0 → 0% when equal, Infinity when current grew. */ -function pctOver(currentValue: number, baselineValue: number): number { - if (baselineValue === 0) { - return currentValue === 0 ? 0 : Number.POSITIVE_INFINITY - } - return ((currentValue - baselineValue) / baselineValue) * 100 -} - -export function gatePerf( - current: PerfBaseline, - baseline: PerfBaseline, - options?: { tolerancePct?: number; minSamples?: number }, -): PerfGateResult { - const tolerancePct = options?.tolerancePct ?? 10 - const minSamples = options?.minSamples ?? 3 - const regressions: PerfRegression[] = [] - const improvements: PerfRegression[] = [] - const missing = new Set() - - for (const [scenarioKey, baselineStats] of Object.entries(baseline.scenarios)) { - const currentStats = current.scenarios[scenarioKey] - if (!currentStats) { - missing.add(scenarioKey) - continue - } - for (const [field, baselineStat] of Object.entries(baselineStats)) { - const currentStat = currentStats[field] - if (!currentStat || currentStat.n < minSamples) { - // Under-sampled current data cannot gate — surface it instead of - // pretending the scenario passed. - missing.add(scenarioKey) - continue - } - const overBy = { - p50Pct: pctOver(currentStat.p50, baselineStat.p50), - p90Pct: pctOver(currentStat.p90, baselineStat.p90), - } - const entry: PerfRegression = { - scenarioKey, - field, - baseline: baselineStat, - current: currentStat, - overBy, - } - if (overBy.p50Pct > tolerancePct || overBy.p90Pct > tolerancePct) { - regressions.push(entry) - } else if (overBy.p50Pct < 0 && overBy.p90Pct < 0) { - improvements.push(entry) - } - } - } - - const newScenarios = Object.keys(current.scenarios).filter((key) => !(key in baseline.scenarios)) - - return { - succeeded: regressions.length === 0, - regressions, - improvements, - missingScenarios: [...missing], - newScenarios, - } -} diff --git a/src/telemetry/client.ts b/src/telemetry/client.ts deleted file mode 100644 index 9872cd85..00000000 --- a/src/telemetry/client.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Telemetry client — thin wrapper that builds envelopes from `EmitArgs` and - * delegates to a `TelemetrySink`. Pure logic; no I/O. Use this from any - * runtime — Workers, Node, browser — and choose the sink accordingly. - * - * For an opinionated singleton with env-var-driven sink wiring (the bad CLI - * pattern), see `./node-client.ts`. - */ - -import type { TelemetryEnvelope, TelemetryKind, TelemetryModel, TelemetrySource } from './schema' -import { TELEMETRY_SCHEMA_VERSION } from './schema' -import type { TelemetrySink } from './sink-fetch' - -export interface EmitArgs { - kind: TelemetryKind - runId: string - parentRunId?: string - ok: boolean - durationMs: number - data?: Record - metrics?: Record - tags?: Record - model?: TelemetryModel - error?: string - /** Override the source for this envelope. Falls back to `defaultSource`. */ - source?: TelemetrySource -} - -export class TelemetryClient { - constructor( - private readonly sink: TelemetrySink, - private readonly defaultSource: TelemetrySource, - ) {} - - emit(args: EmitArgs): void { - const envelope: TelemetryEnvelope = { - schemaVersion: TELEMETRY_SCHEMA_VERSION, - envelopeId: makeEnvelopeId(), - runId: args.runId, - timestamp: new Date().toISOString(), - source: args.source ?? this.defaultSource, - kind: args.kind, - ok: args.ok, - durationMs: args.durationMs, - data: args.data ?? {}, - metrics: args.metrics ?? {}, - ...(args.parentRunId ? { parentRunId: args.parentRunId } : {}), - ...(args.model ? { model: args.model } : {}), - ...(args.tags ? { tags: args.tags } : {}), - ...(args.error ? { error: args.error } : {}), - } - try { - this.sink.emit(envelope) - } catch { - // swallow — telemetry never breaks the calling code path - } - } - - async close(): Promise { - await this.sink.close?.() - } -} - -/** Generate a UUIDv4 with whatever crypto is available (Node, Workers, browsers). */ -function makeEnvelopeId(): string { - if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { - return crypto.randomUUID() - } - // Last-resort fallback. Lower entropy but never throws. - return `env-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}` -} - -export const SECRET_FLAGS = new Set(['--api-key', '--bearer', '--token', '--password']) - -/** Strip likely-secret values from argv, preserving structure. */ -export function sanitiseArgv(argv: string[]): string[] { - const out: string[] = [] - for (let i = 0; i < argv.length; i++) { - const a = argv[i]! - if (SECRET_FLAGS.has(a)) { - out.push(a, '') - i++ - continue - } - if (/^(?:--api-key|--bearer|--token|--password)=/.test(a)) { - out.push(a.replace(/=.*$/, '=')) - continue - } - out.push(a) - } - return out -} diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts deleted file mode 100644 index eaf147a6..00000000 --- a/src/telemetry/index.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Public entry for the telemetry sub-module. - * - * Workers-safe by default — only `fetch` + pure JS. The Node file sink is - * exported separately via './telemetry/file' so consumers that import this entry - * cannot accidentally pull `node:fs` into a Worker bundle. - * - * Consume: - * import { TelemetryClient, HttpTelemetrySink, FanoutTelemetrySink } - * from '@tangle-network/agent-eval/telemetry' - * - * For Node: - * import { FileTelemetrySink, defaultTelemetryDir } - * from '@tangle-network/agent-eval/telemetry/file' - */ - -export { - type EmitArgs, - SECRET_FLAGS, - sanitiseArgv, - TelemetryClient, -} from './client' -export type { - TelemetryEnvelope, - TelemetryKind, - TelemetryModel, - TelemetrySource, -} from './schema' -export { TELEMETRY_SCHEMA_VERSION } from './schema' -export { - FanoutTelemetrySink, - HttpTelemetrySink, - InMemoryTelemetrySink, - NullTelemetrySink, - type TelemetrySink, -} from './sink-fetch' diff --git a/src/telemetry/schema.ts b/src/telemetry/schema.ts deleted file mode 100644 index 7b6ba751..00000000 --- a/src/telemetry/schema.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Fleet telemetry envelope — agent-eval's portable observability shape. - * - * Designed so any consumer (Node CLI, Cloudflare Worker, Lambda, browser - * extension) can emit structured rows describing one unit of work — a page - * audit, a tool call, an evolve round, a full agent run — to a central sink. - * - * The schema is intentionally a strict superset of agent-eval's `Run` shape - * so a future TraceStore adapter can promote envelopes into traces without - * translation. - */ - -export const TELEMETRY_SCHEMA_VERSION = 1 - -/** Discriminator for the unit of work this envelope describes. */ -export type TelemetryKind = - | 'agent-run' - | 'design-audit-page' - | 'design-audit-run' - | 'design-evolve-round' - | 'design-evolve-run' - | 'gepa-trial' - | 'gepa-generation' - | 'tool-call' - | 'judge-verdict' - | 'custom' - -export interface TelemetryEnvelope { - schemaVersion: typeof TELEMETRY_SCHEMA_VERSION - envelopeId: string - runId: string - timestamp: string - parentRunId?: string - - source: TelemetrySource - model?: TelemetryModel - kind: TelemetryKind - ok: boolean - durationMs: number - - data: Record - metrics: Record - tags?: Record - - error?: string -} - -export interface TelemetrySource { - /** Repo identity — basename of cwd plus git remote if discoverable. */ - repo: string - cwd: string - gitSha?: string - gitBranch?: string - cliVersion: string - /** What was invoked, e.g. `design-audit`, `bad run`, `gepa --target`. */ - invocation: string - /** Sanitised argv minus secrets. */ - argv?: string[] - /** - * Multi-tenant identity. Set when the consumer runs inside a hosted - * product so a fleet rollup can group by tenant without leaking customer - * URLs or PII. - */ - tenantId?: string - /** Optional sub-tenant identity (project, suite, walkthrough, customer). */ - customerId?: string - /** SHA-256 (12 hex) of the API key used to authenticate this run, when applicable. */ - apiKeyHash?: string -} - -export interface TelemetryModel { - provider: string - name: string - /** SHA-256 (12 hex chars) of the prompt(s) used. */ - promptHash?: string - /** SHA-256 (12 hex chars) of the composed rubric body, if applicable. */ - rubricHash?: string -} diff --git a/src/telemetry/sink-fetch.ts b/src/telemetry/sink-fetch.ts deleted file mode 100644 index 5f5d974b..00000000 --- a/src/telemetry/sink-fetch.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Workers-safe telemetry sinks — only `fetch` and pure JS. No `fs`, no - * `child_process`. Safe to import from a Cloudflare Worker, Lambda, edge - * function, or browser extension. - * - * For Node-only file persistence, import from '@tangle-network/agent-eval/telemetry/file'. - */ - -import type { TelemetryEnvelope } from './schema' - -export interface TelemetrySink { - emit(envelope: TelemetryEnvelope): Promise | void - close?(): Promise | void -} - -/** Best-effort POST to a remote collector. Fire-and-forget; never throws. */ -export class HttpTelemetrySink implements TelemetrySink { - private inflight = new Set>() - - constructor( - private readonly endpoint: string, - private readonly bearer?: string, - ) {} - - emit(envelope: TelemetryEnvelope): void { - const body = JSON.stringify(envelope) - const headers: Record = { 'content-type': 'application/json' } - if (this.bearer) headers.authorization = `Bearer ${this.bearer}` - const promise = fetch(this.endpoint, { method: 'POST', headers, body }) - .then(() => undefined) - .catch(() => undefined) - this.inflight.add(promise) - promise.finally(() => this.inflight.delete(promise)) - } - - async close(): Promise { - await Promise.allSettled(Array.from(this.inflight)) - } -} - -/** Fanout to multiple sinks — failures in one do not affect others. */ -export class FanoutTelemetrySink implements TelemetrySink { - constructor(private readonly sinks: TelemetrySink[]) {} - - emit(envelope: TelemetryEnvelope): void { - for (const sink of this.sinks) { - try { - const result = sink.emit(envelope) - if (result && typeof (result as Promise).catch === 'function') { - ;(result as Promise).catch(() => undefined) - } - } catch { - // swallow — telemetry must never break a run - } - } - } - - async close(): Promise { - await Promise.allSettled(this.sinks.map((s) => Promise.resolve(s.close?.()))) - } -} - -/** No-op sink — used when telemetry is explicitly disabled. */ -export class NullTelemetrySink implements TelemetrySink { - emit(): void {} -} - -/** In-memory sink — useful for tests + downstream adapters. */ -export class InMemoryTelemetrySink implements TelemetrySink { - readonly envelopes: TelemetryEnvelope[] = [] - emit(envelope: TelemetryEnvelope): void { - this.envelopes.push(envelope) - } - clear(): void { - this.envelopes.length = 0 - } -} diff --git a/src/telemetry/sink-file.ts b/src/telemetry/sink-file.ts deleted file mode 100644 index 45c1edaa..00000000 --- a/src/telemetry/sink-file.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Node-only file sink. Imports `node:fs` — DO NOT import this from a Worker - * or edge runtime; use `./sink-fetch` instead. - */ - -import * as fs from 'node:fs' -import * as path from 'node:path' -import type { TelemetryEnvelope } from './schema' -import type { TelemetrySink } from './sink-fetch' - -/** Append envelopes to a JSONL file, partitioned by repo + date. */ -export class FileTelemetrySink implements TelemetrySink { - private streams = new Map() - - constructor(private readonly baseDir: string) { - fs.mkdirSync(baseDir, { recursive: true }) - } - - emit(envelope: TelemetryEnvelope): void { - const date = envelope.timestamp.slice(0, 10) // YYYY-MM-DD - const repo = envelope.source.repo || 'unknown' - const key = `${repo}/${date}` - let stream = this.streams.get(key) - if (!stream) { - const dir = path.join(this.baseDir, repo) - fs.mkdirSync(dir, { recursive: true }) - stream = fs.createWriteStream(path.join(dir, `${date}.jsonl`), { - flags: 'a', - encoding: 'utf-8', - }) - this.streams.set(key, stream) - } - stream.write(`${JSON.stringify(envelope)}\n`) - } - - async close(): Promise { - const closes = Array.from(this.streams.values()).map( - (s) => new Promise((resolve) => s.end(() => resolve())), - ) - this.streams.clear() - await Promise.all(closes) - } -} - -/** Default location for local telemetry, mirroring bad CLI's convention. */ -export function defaultTelemetryDir(homeDir: string, override?: string): string { - return override || path.join(homeDir, '.agent-eval', 'telemetry') -} diff --git a/src/workflow/event-schema.ts b/src/workflow/event-schema.ts deleted file mode 100644 index f55f3376..00000000 --- a/src/workflow/event-schema.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { ValidationError } from '../errors' -import type { WorkflowTraceEventKind } from './types' - -const WORKFLOW_EVENT_KINDS = [ - 'workflow.started', - 'workflow.phase', - 'workflow.log', - 'workflow.parallel.started', - 'workflow.parallel.ended', - 'workflow.pipeline.started', - 'workflow.pipeline.ended', - 'workflow.branch.started', - 'workflow.branch.ended', - 'workflow.branch.failed', - 'workflow.agent.started', - 'workflow.agent.ended', - 'workflow.agent.failed', - 'workflow.loop.started', - 'workflow.loop.ended', - 'workflow.loop.failed', - 'workflow.verifier.started', - 'workflow.verifier.ended', - 'workflow.verifier.failed', - 'workflow.analyst.started', - 'workflow.analyst.ended', - 'workflow.analyst.failed', - 'workflow.reviewer.started', - 'workflow.reviewer.ended', - 'workflow.reviewer.failed', - 'workflow.failed', - 'workflow.ended', -] as const satisfies readonly WorkflowTraceEventKind[] - -export const WORKFLOW_TRACE_EVENT_KINDS: readonly WorkflowTraceEventKind[] = WORKFLOW_EVENT_KINDS - -const WORKFLOW_EVENT_KIND_SET = new Set(WORKFLOW_EVENT_KINDS) -const BRANCH_OPERATIONS = new Set(['parallel', 'pipeline']) - -export function validateWorkflowTraceEventKind(kind: string): WorkflowTraceEventKind { - if (!WORKFLOW_EVENT_KIND_SET.has(kind)) { - throw new ValidationError(`unknown workflow trace event kind: ${kind}`) - } - return kind as WorkflowTraceEventKind -} - -export function validateWorkflowTraceEventPayload( - kind: WorkflowTraceEventKind, - payload: Record, -): Record { - switch (kind) { - case 'workflow.started': - requireRecord(payload.meta, `${kind}.payload.meta`) - requireInteger(payload.depth, `${kind}.payload.depth`, { min: 0 }) - requireRecord(payload.caps, `${kind}.payload.caps`) - return payload - case 'workflow.phase': - requireString(payload.title, `${kind}.payload.title`) - return payload - case 'workflow.log': - requireString(payload.message, `${kind}.payload.message`) - optionalString(payload.phase, `${kind}.payload.phase`) - return payload - case 'workflow.parallel.started': - requireInteger(payload.branchCount, `${kind}.payload.branchCount`, { min: 0 }) - optionalString(payload.phase, `${kind}.payload.phase`) - return payload - case 'workflow.parallel.ended': - requireInteger(payload.branchCount, `${kind}.payload.branchCount`, { min: 0 }) - requireNonNegativeNumber(payload.durationMs, `${kind}.payload.durationMs`) - optionalString(payload.phase, `${kind}.payload.phase`) - return payload - case 'workflow.pipeline.started': - requireInteger(payload.itemCount, `${kind}.payload.itemCount`, { min: 0 }) - requireInteger(payload.stageCount, `${kind}.payload.stageCount`, { min: 1 }) - optionalString(payload.phase, `${kind}.payload.phase`) - return payload - case 'workflow.pipeline.ended': - requireInteger(payload.itemCount, `${kind}.payload.itemCount`, { min: 0 }) - requireInteger(payload.stageCount, `${kind}.payload.stageCount`, { min: 1 }) - requireNonNegativeNumber(payload.durationMs, `${kind}.payload.durationMs`) - optionalString(payload.phase, `${kind}.payload.phase`) - return payload - case 'workflow.branch.started': - validateBranchPayload(kind, payload, { terminal: false }) - return payload - case 'workflow.branch.ended': - validateBranchPayload(kind, payload, { terminal: true }) - return payload - case 'workflow.branch.failed': - validateBranchPayload(kind, payload, { terminal: true }) - requireString(payload.message, `${kind}.payload.message`) - optionalString(payload.code, `${kind}.payload.code`) - if (payload.stageIndex !== undefined) { - requireInteger(payload.stageIndex, `${kind}.payload.stageIndex`, { min: 0 }) - } - return payload - case 'workflow.agent.started': - validateIndexedPayload(kind, payload) - requireInteger(payload.promptChars, `${kind}.payload.promptChars`, { min: 0 }) - optionalRecord(payload.metadata, `${kind}.payload.metadata`) - return payload - case 'workflow.agent.ended': - validateDelegateEndedPayload(kind, payload) - return payload - case 'workflow.agent.failed': - validateDelegateFailedPayload(kind, payload) - return payload - case 'workflow.loop.started': - case 'workflow.verifier.started': - case 'workflow.analyst.started': - case 'workflow.reviewer.started': - validateIndexedPayload(kind, payload) - optionalRecord(payload.metadata, `${kind}.payload.metadata`) - return payload - case 'workflow.loop.ended': - case 'workflow.verifier.ended': - case 'workflow.analyst.ended': - case 'workflow.reviewer.ended': - validateDelegateEndedPayload(kind, payload) - return payload - case 'workflow.loop.failed': - case 'workflow.verifier.failed': - case 'workflow.analyst.failed': - case 'workflow.reviewer.failed': - validateDelegateFailedPayload(kind, payload) - return payload - case 'workflow.failed': - requireString(payload.message, `${kind}.payload.message`) - optionalString(payload.code, `${kind}.payload.code`) - optionalString(payload.phase, `${kind}.payload.phase`) - return payload - case 'workflow.ended': - requireNonNegativeNumber(payload.durationMs, `${kind}.payload.durationMs`) - requireNonNegativeNumber(payload.costUsd, `${kind}.payload.costUsd`) - validateTokenUsage(payload.tokenUsage, `${kind}.payload.tokenUsage`) - requireInteger(payload.agentCalls, `${kind}.payload.agentCalls`, { min: 0 }) - requireInteger(payload.loopCalls, `${kind}.payload.loopCalls`, { min: 0 }) - return payload - } -} - -function validateBranchPayload( - kind: WorkflowTraceEventKind, - payload: Record, - options: { terminal: boolean }, -): void { - const operation = requireString(payload.operation, `${kind}.payload.operation`) - if (!BRANCH_OPERATIONS.has(operation)) { - throw new ValidationError(`${kind}.payload.operation: expected parallel or pipeline`) - } - requireInteger(payload.branchIndex, `${kind}.payload.branchIndex`, { min: 0 }) - optionalString(payload.phase, `${kind}.payload.phase`) - if (payload.stageCount !== undefined) { - requireInteger(payload.stageCount, `${kind}.payload.stageCount`, { min: 1 }) - } - if (options.terminal) { - requireNonNegativeNumber(payload.durationMs, `${kind}.payload.durationMs`) - } -} - -function validateIndexedPayload( - kind: WorkflowTraceEventKind, - payload: Record, -): void { - requireInteger(payload.index, `${kind}.payload.index`, { min: 0 }) - optionalString(payload.label, `${kind}.payload.label`) - optionalString(payload.phase, `${kind}.payload.phase`) -} - -function validateDelegateEndedPayload( - kind: WorkflowTraceEventKind, - payload: Record, -): void { - validateIndexedPayload(kind, payload) - requireNonNegativeNumber(payload.durationMs, `${kind}.payload.durationMs`) - requireNonNegativeNumber(payload.costUsd, `${kind}.payload.costUsd`) - validateTokenUsage(payload.tokenUsage, `${kind}.payload.tokenUsage`) -} - -function validateDelegateFailedPayload( - kind: WorkflowTraceEventKind, - payload: Record, -): void { - validateIndexedPayload(kind, payload) - requireNonNegativeNumber(payload.durationMs, `${kind}.payload.durationMs`) - requireString(payload.message, `${kind}.payload.message`) - optionalString(payload.code, `${kind}.payload.code`) -} - -function validateTokenUsage(value: unknown, path: string): void { - const tokenUsage = requireRecord(value, path) - requireNonNegativeNumber(tokenUsage.input, `${path}.input`) - requireNonNegativeNumber(tokenUsage.output, `${path}.output`) - if (tokenUsage.cached !== undefined) { - requireNonNegativeNumber(tokenUsage.cached, `${path}.cached`) - } -} - -function requireRecord(value: unknown, path: string): Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new ValidationError(`${path}: expected object`) - } - return value as Record -} - -function optionalRecord(value: unknown, path: string): void { - if (value !== undefined) requireRecord(value, path) -} - -function requireString(value: unknown, path: string): string { - if (typeof value !== 'string' || value.length === 0) { - throw new ValidationError(`${path}: expected non-empty string`) - } - return value -} - -function optionalString(value: unknown, path: string): void { - if (value !== undefined) requireString(value, path) -} - -function requireNonNegativeNumber(value: unknown, path: string): number { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - throw new ValidationError(`${path}: expected finite non-negative number`) - } - return value -} - -function requireInteger(value: unknown, path: string, options: { min?: number } = {}): number { - if (typeof value !== 'number' || !Number.isInteger(value)) { - throw new ValidationError(`${path}: expected integer`) - } - if (options.min !== undefined && value < options.min) { - throw new ValidationError(`${path}: expected integer >= ${options.min}`) - } - return value -} diff --git a/src/workflow/feedback-pack.ts b/src/workflow/feedback-pack.ts deleted file mode 100644 index 49942258..00000000 --- a/src/workflow/feedback-pack.ts +++ /dev/null @@ -1,484 +0,0 @@ -import type { AnalystFinding, AnalystSeverity, EvidenceRef } from '../analyst/types' -import type { FailureClusterInsight } from '../contract/insight-report' -import type { LayerResult, VerificationReport } from '../multi-layer-verifier' -import type { FailureClusterReport } from '../pipelines/failure-cluster' -import { summarizeWorkflowTrace, validateWorkflowTraceEnvelope } from './schema' -import type { WorkflowTraceEnvelope, WorkflowTraceSummary } from './types' - -export type WorkflowFeedbackPackVersion = 'workflow-feedback-pack-v1' - -export type WorkflowFeedbackSeverity = AnalystSeverity - -export interface WorkflowToolUsageSummary { - totalCalls: number - erroredCalls: number - byTool: Record -} - -export interface WorkflowVerifierFindingSummary { - severity: WorkflowFeedbackSeverity - message: string - evidence?: string - detail?: Record -} - -export interface WorkflowVerifierLayerSummary { - layer: string - status: LayerResult['status'] - score?: number - durationMs: number - reason?: string - findings: WorkflowVerifierFindingSummary[] - diagnostics?: Record -} - -export interface WorkflowVerifierSummary { - allPass: boolean - blendedScore: number - durationMs: number - failedLayers: string[] - layers: WorkflowVerifierLayerSummary[] -} - -export interface WorkflowFailureClusterInput { - id: string - name: string - share?: number - runCount?: number - exemplars?: readonly string[] - suggestedFix?: string - metadata?: Record -} - -export interface WorkflowFailureClusterSummary { - id: string - name: string - share: number - runCount?: number - exemplars: string[] - suggestedFix?: string - source: 'failure-cluster-view' | 'insight-report' | 'custom' - metadata?: Record -} - -export interface WorkflowAnalystFindingSummary { - findingId: string - analystId: string - severity: WorkflowFeedbackSeverity - area: string - claim: string - confidence: number - subject?: string - recommendedAction?: string - evidenceRefs: EvidenceRef[] -} - -export interface WorkflowAnalystFeedbackPack { - schemaVersion: WorkflowFeedbackPackVersion - runId: string - generatedAt: string - summary: WorkflowTraceSummary - verifier?: WorkflowVerifierSummary - toolUsage: WorkflowToolUsageSummary - failureClusters: WorkflowFailureClusterSummary[] - findings: WorkflowAnalystFindingSummary[] - recommendations: string[] - driverContextLines: string[] -} - -export interface WorkflowFeedbackPackLimits { - findings?: number - clusters?: number - layerFindings?: number - recommendations?: number - contextLines?: number -} - -export interface BuildWorkflowAnalystFeedbackPackOptions { - envelope: WorkflowTraceEnvelope | unknown - verifier?: VerificationReport - analystFindings?: readonly AnalystFinding[] - failureClusters?: - | FailureClusterReport - | FailureClusterInsight - | readonly WorkflowFailureClusterInput[] - generatedAt?: string - limits?: WorkflowFeedbackPackLimits -} - -const PACK_VERSION: WorkflowFeedbackPackVersion = 'workflow-feedback-pack-v1' - -const DEFAULT_LIMITS: Required = { - findings: 12, - clusters: 8, - layerFindings: 5, - recommendations: 10, - contextLines: 24, -} - -export function buildWorkflowAnalystFeedbackPack( - options: BuildWorkflowAnalystFeedbackPackOptions, -): WorkflowAnalystFeedbackPack { - const limits = { ...DEFAULT_LIMITS, ...(options.limits ?? {}) } - const envelope = validateWorkflowTraceEnvelope(options.envelope) - const summary = summarizeWorkflowTrace(envelope) - const verifier = options.verifier ? summarizeVerifier(options.verifier, limits) : undefined - const toolUsage = summarizeToolUsage(envelope) - const failureClusters = normalizeFailureClusters(options.failureClusters) - .sort((a, b) => b.share - a.share) - .slice(0, limits.clusters) - const findings = summarizeFindings(options.analystFindings ?? []) - .sort(compareFindings) - .slice(0, limits.findings) - const recommendations = uniqueStrings([ - ...recommendFromVerifier(verifier), - ...failureClusters.flatMap((cluster) => (cluster.suggestedFix ? [cluster.suggestedFix] : [])), - ...findings.flatMap((finding) => - finding.recommendedAction ? [finding.recommendedAction] : [], - ), - ...(summary.failed && summary.failureMessage - ? [`Inspect workflow failure: ${summary.failureMessage}`] - : []), - ]).slice(0, limits.recommendations) - - const pack: WorkflowAnalystFeedbackPack = { - schemaVersion: PACK_VERSION, - runId: envelope.runId, - generatedAt: options.generatedAt ?? new Date().toISOString(), - summary, - ...(verifier ? { verifier } : {}), - toolUsage, - failureClusters, - findings, - recommendations, - driverContextLines: [], - } - return { - ...pack, - driverContextLines: renderDriverContextLines(pack).slice(0, limits.contextLines), - } -} - -export function renderWorkflowFeedbackPack( - pack: WorkflowAnalystFeedbackPack, - options: { maxChars?: number } = {}, -): string { - const lines = [ - `Workflow feedback pack for ${pack.runId}`, - `status=${pack.summary.failed ? 'failed' : 'completed'} durationMs=${pack.summary.durationMs} costUsd=${pack.summary.costUsd.toFixed(6)} tokens=${pack.summary.tokenUsage.input}/${pack.summary.tokenUsage.output} events=${pack.summary.eventCount}`, - renderDelegateFailureCounts(pack.summary), - pack.verifier - ? `verifier=${pack.verifier.allPass ? 'pass' : 'fail'} blendedScore=${pack.verifier.blendedScore.toFixed(3)} failedLayers=${pack.verifier.failedLayers.join(',') || 'none'}` - : undefined, - pack.toolUsage.totalCalls > 0 - ? `tools=${pack.toolUsage.totalCalls} errors=${pack.toolUsage.erroredCalls} byTool=${formatToolUsage(pack.toolUsage)}` - : undefined, - pack.failureClusters.length > 0 ? 'Failure clusters:' : undefined, - ...pack.failureClusters.map( - (cluster) => - `- ${cluster.name} share=${cluster.share.toFixed(3)} exemplars=${cluster.exemplars.join(',') || 'none'}${cluster.suggestedFix ? ` fix=${cluster.suggestedFix}` : ''}`, - ), - pack.findings.length > 0 ? 'Analyst findings:' : undefined, - ...pack.findings.map( - (finding) => - `- ${finding.severity} ${finding.area}: ${finding.claim}${finding.recommendedAction ? ` action=${finding.recommendedAction}` : ''}`, - ), - pack.recommendations.length > 0 ? 'Recommended next moves:' : undefined, - ...pack.recommendations.map((recommendation) => `- ${recommendation}`), - ].filter((line): line is string => Boolean(line)) - const rendered = lines.join('\n') - const maxChars = options.maxChars - if (maxChars === undefined || rendered.length <= maxChars) return rendered - if (maxChars <= 1) return rendered.slice(0, Math.max(0, maxChars)) - return `${rendered.slice(0, maxChars - 1)}…` -} - -function renderDelegateFailureCounts(summary: WorkflowTraceSummary): string | undefined { - const entries: Array<[string, number]> = [ - ['agent', summary.agentFailures], - ['loop', summary.loopFailures], - ['verifier', summary.verifierFailures], - ['analyst', summary.analystFailures], - ['reviewer', summary.reviewerFailures], - ] - const failures = entries.filter((entry) => entry[1] > 0) - if (failures.length === 0) return undefined - return `delegateFailures=${failures.map(([kind, count]) => `${kind}:${count}`).join(',')}` -} - -function summarizeVerifier( - verifier: VerificationReport, - limits: Required, -): WorkflowVerifierSummary { - const layers = verifier.layers.map((layer) => ({ - layer: layer.layer, - status: layer.status, - ...(layer.score !== undefined ? { score: layer.score } : {}), - durationMs: layer.durationMs, - ...(layer.reason ? { reason: layer.reason } : {}), - findings: layer.findings - .map((finding) => ({ - severity: verifierSeverity(finding.severity), - message: finding.message, - ...(finding.evidence ? { evidence: finding.evidence } : {}), - ...(finding.detail ? { detail: finding.detail } : {}), - })) - .slice(0, limits.layerFindings), - ...(layer.diagnostics ? { diagnostics: layer.diagnostics } : {}), - })) - return { - allPass: verifier.allPass, - blendedScore: verifier.blendedScore, - durationMs: verifier.durationMs, - failedLayers: verifier.layers - .filter( - (layer) => - layer.status === 'fail' || layer.status === 'error' || layer.status === 'timeout', - ) - .map((layer) => layer.layer), - layers, - } -} - -function summarizeToolUsage(envelope: WorkflowTraceEnvelope): WorkflowToolUsageSummary { - const summary: WorkflowToolUsageSummary = { totalCalls: 0, erroredCalls: 0, byTool: {} } - for (const event of envelope.events) { - collectToolUsagePayload(summary, event.payload.toolUsage) - collectToolCalls(summary, event.payload.toolCalls) - collectSingleTool(summary, event.payload) - } - return summary -} - -function collectToolUsagePayload(summary: WorkflowToolUsageSummary, value: unknown): void { - if (!isRecord(value)) return - if (!isRecord(value.byTool)) { - summary.totalCalls += finiteNumber(value.totalCalls) - summary.erroredCalls += finiteNumber(value.erroredCalls) - return - } - let addedByTool = false - for (const [tool, raw] of Object.entries(value.byTool)) { - if (!isRecord(raw)) continue - const calls = finiteNumber(raw.calls) - const errors = finiteNumber(raw.errors) - addTool(summary, tool, calls, errors) - addedByTool ||= calls > 0 || errors > 0 - } - if (!addedByTool) { - summary.totalCalls += finiteNumber(value.totalCalls) - summary.erroredCalls += finiteNumber(value.erroredCalls) - } -} - -function collectToolCalls(summary: WorkflowToolUsageSummary, value: unknown): void { - if (!Array.isArray(value)) return - for (const call of value) { - if (!isRecord(call)) continue - const name = stringValue(call.toolName) ?? stringValue(call.name) - if (!name) continue - const errored = - call.status === 'error' || - call.error !== undefined || - call.ok === false || - call.success === false - addTool(summary, name, 1, errored ? 1 : 0) - } -} - -function collectSingleTool( - summary: WorkflowToolUsageSummary, - payload: Record, -): void { - const name = stringValue(payload.toolName) - if (!name) return - const errored = - payload.status === 'error' || - payload.error !== undefined || - payload.ok === false || - payload.success === false - addTool(summary, name, 1, errored ? 1 : 0) -} - -function addTool( - summary: WorkflowToolUsageSummary, - name: string, - calls: number, - errors: number, -): void { - if (calls === 0 && errors === 0) return - const current = summary.byTool[name] ?? { calls: 0, errors: 0 } - current.calls += calls - current.errors += errors - summary.byTool[name] = current - summary.totalCalls += calls - summary.erroredCalls += errors -} - -function normalizeFailureClusters( - input: - | FailureClusterReport - | FailureClusterInsight - | readonly WorkflowFailureClusterInput[] - | undefined, -): WorkflowFailureClusterSummary[] { - if (!input) return [] - if (Array.isArray(input)) { - return input.map((cluster) => ({ - id: cluster.id, - name: cluster.name, - share: clamp01(cluster.share ?? 0), - ...(cluster.runCount !== undefined ? { runCount: cluster.runCount } : {}), - exemplars: [...(cluster.exemplars ?? [])], - ...(cluster.suggestedFix ? { suggestedFix: cluster.suggestedFix } : {}), - source: 'custom', - ...(cluster.metadata ? { metadata: cluster.metadata } : {}), - })) - } - const report = input as FailureClusterReport | FailureClusterInsight - const clusters = report.clusters ?? [] - const first = clusters[0] as unknown - if (isRecord(first) && typeof first.failureClass === 'string') { - const failureReport = report as FailureClusterReport - return failureReport.clusters.map((cluster) => ({ - id: [ - cluster.failureClass, - cluster.toolName ?? '', - cluster.argPrefix ?? '', - cluster.dimension ?? '', - ].join('|'), - name: [ - cluster.failureClass, - cluster.toolName ? `tool:${cluster.toolName}` : undefined, - cluster.dimension ? `dimension:${cluster.dimension}` : undefined, - ] - .filter(Boolean) - .join(' '), - share: failureReport.totalFailures > 0 ? cluster.runCount / failureReport.totalFailures : 0, - runCount: cluster.runCount, - exemplars: [cluster.exampleRunId], - source: 'failure-cluster-view', - metadata: { - scenarioIds: cluster.scenarioIds, - exampleError: cluster.exampleError, - argPrefix: cluster.argPrefix, - }, - })) - } - const insight = report as FailureClusterInsight - return insight.clusters.map((cluster) => ({ - id: cluster.id, - name: cluster.name, - share: clamp01(cluster.share), - exemplars: [...cluster.exemplars], - ...(cluster.suggestedFix ? { suggestedFix: cluster.suggestedFix } : {}), - source: 'insight-report', - })) -} - -function summarizeFindings(findings: readonly AnalystFinding[]): WorkflowAnalystFindingSummary[] { - return findings.map((finding) => ({ - findingId: finding.finding_id, - analystId: finding.analyst_id, - severity: finding.severity, - area: finding.area, - claim: finding.claim, - confidence: clamp01(finding.confidence), - ...(finding.subject ? { subject: finding.subject } : {}), - ...(finding.recommended_action ? { recommendedAction: finding.recommended_action } : {}), - evidenceRefs: finding.evidence_refs, - })) -} - -function recommendFromVerifier(verifier: WorkflowVerifierSummary | undefined): string[] { - if (!verifier || verifier.allPass) return [] - return verifier.layers - .filter((layer) => verifier.failedLayers.includes(layer.layer)) - .map((layer) => { - const firstFinding = layer.findings[0]?.message - const detail = layer.reason ?? firstFinding ?? layer.status - return `Fix verifier layer "${layer.layer}": ${detail}` - }) -} - -function renderDriverContextLines(pack: WorkflowAnalystFeedbackPack): string[] { - return renderWorkflowFeedbackPack(pack) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) -} - -function compareFindings( - left: WorkflowAnalystFindingSummary, - right: WorkflowAnalystFindingSummary, -): number { - const severityDelta = severityRank(right.severity) - severityRank(left.severity) - if (severityDelta !== 0) return severityDelta - return right.confidence - left.confidence -} - -function verifierSeverity(severity: string): WorkflowFeedbackSeverity { - switch (severity) { - case 'critical': - return 'critical' - case 'major': - return 'high' - case 'minor': - return 'low' - case 'info': - return 'info' - default: - return 'medium' - } -} - -function severityRank(severity: WorkflowFeedbackSeverity): number { - switch (severity) { - case 'critical': - return 5 - case 'high': - return 4 - case 'medium': - return 3 - case 'low': - return 2 - case 'info': - return 1 - } -} - -function uniqueStrings(values: readonly string[]): string[] { - const out: string[] = [] - const seen = new Set() - for (const value of values) { - const trimmed = value.trim() - if (trimmed.length === 0 || seen.has(trimmed)) continue - seen.add(trimmed) - out.push(trimmed) - } - return out -} - -function formatToolUsage(summary: WorkflowToolUsageSummary): string { - return Object.entries(summary.byTool) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([tool, usage]) => `${tool}:${usage.calls}/${usage.errors}`) - .join(',') -} - -function finiteNumber(value: unknown): number { - return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0 -} - -function clamp01(value: number): number { - if (!Number.isFinite(value)) return 0 - return Math.max(0, Math.min(1, value)) -} - -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value) -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} diff --git a/src/workflow/index.ts b/src/workflow/index.ts deleted file mode 100644 index 85d8babb..00000000 --- a/src/workflow/index.ts +++ /dev/null @@ -1,110 +0,0 @@ -export { - validateWorkflowTraceEventKind, - validateWorkflowTraceEventPayload, - WORKFLOW_TRACE_EVENT_KINDS, -} from './event-schema' -export type { - BuildWorkflowAnalystFeedbackPackOptions, - WorkflowAnalystFeedbackPack, - WorkflowAnalystFindingSummary, - WorkflowFailureClusterInput, - WorkflowFailureClusterSummary, - WorkflowFeedbackPackLimits, - WorkflowFeedbackPackVersion, - WorkflowFeedbackSeverity, - WorkflowToolUsageSummary, - WorkflowVerifierFindingSummary, - WorkflowVerifierLayerSummary, - WorkflowVerifierSummary, -} from './feedback-pack' -export { - buildWorkflowAnalystFeedbackPack, - renderWorkflowFeedbackPack, -} from './feedback-pack' -export type { - BuildWorkflowTraceIntelligenceEnvelopeOptions, - WorkflowTraceArtifactEvidence, - WorkflowTraceCompactEvidence, - WorkflowTraceExportGrant, - WorkflowTraceExportGrantScope, - WorkflowTraceExportGrantSubject, - WorkflowTraceHashEvidence, - WorkflowTraceIntelligenceEnvelope, - WorkflowTraceIntelligenceEnvelopeVersion, -} from './intelligence-export' -export { - buildWorkflowTraceIntelligenceEnvelope, - validateWorkflowTraceIntelligenceEnvelope, -} from './intelligence-export' -export type { - BuildWorkflowPartnerReportOptions, - WorkflowPartnerFinding, - WorkflowPartnerReport, - WorkflowPartnerReportVersion, -} from './partner-report' -export { - buildWorkflowPartnerReport, - renderWorkflowPartnerReport, - validateWorkflowPartnerReport, -} from './partner-report' -export type { - WorkflowPhaseGraph, - WorkflowPhaseGraphBranch, - WorkflowPhaseGraphNode, -} from './phase-graph' -export { workflowPhaseGraph } from './phase-graph' -export type { - DecideWorkflowDriverPromotionOptions, - WorkflowDriverPromotionDecision, - WorkflowDriverPromotionDecisionVersion, - WorkflowDriverPromotionEvidence, - WorkflowDriverPromotionPair, - WorkflowDriverPromotionRejectionCode, -} from './promotion-gate' -export { decideWorkflowDriverPromotion } from './promotion-gate' -export { - type WorkflowTraceRunRecordOptions, - workflowTraceToRunRecord, -} from './run-record' -export type { - WorkflowRuntimeResultLike, - WorkflowRuntimeResultToTraceEnvelopeOptions, - WorkflowTraceEnvelopeFromEventsOptions, -} from './runtime-adapter' -export { - workflowEventsToTraceEnvelope, - workflowRuntimeResultToTraceEnvelope, -} from './runtime-adapter' -export type { - SanitizedWorkflowTraceEnvelopeResult, - SanitizeWorkflowTraceEnvelopeOptions, - WorkflowTraceSanitizationReport, -} from './sanitize' -export { sanitizeWorkflowTraceEnvelope } from './sanitize' -export { - summarizeWorkflowTrace, - validateWorkflowTraceEnvelope, - validateWorkflowTraceEvent, -} from './schema' -export type { - SummarizeWorkflowExecutionOptions, - WorkflowCheckpointTraceSummary, - WorkflowDelegateTraceSummary, - WorkflowExecutionSummary, -} from './summary' -export { summarizeWorkflowExecution } from './summary' -export { - type WorkflowTraceTrajectoryOptions, - workflowTraceToFeedbackTrajectory, -} from './trajectory' -export type { - WorkflowTopology, - WorkflowTraceArtifact, - WorkflowTraceEnvelope, - WorkflowTraceEvent, - WorkflowTraceEventKind, - WorkflowTraceExportLinks, - WorkflowTraceProjectionMetadata, - WorkflowTraceSummary, - WorkflowTraceVersion, -} from './types' diff --git a/src/workflow/intelligence-export.ts b/src/workflow/intelligence-export.ts deleted file mode 100644 index 7d0e5c80..00000000 --- a/src/workflow/intelligence-export.ts +++ /dev/null @@ -1,473 +0,0 @@ -import { ValidationError } from '../errors' -import { - type SanitizedWorkflowTraceEnvelopeResult, - type SanitizeWorkflowTraceEnvelopeOptions, - sanitizeWorkflowTraceEnvelope, - type WorkflowTraceSanitizationReport, -} from './sanitize' -import { validateWorkflowTraceEnvelope } from './schema' -import { summarizeWorkflowExecution, type WorkflowExecutionSummary } from './summary' -import type { WorkflowTraceEnvelope, WorkflowTraceEvent, WorkflowTraceExportLinks } from './types' - -export type WorkflowTraceIntelligenceEnvelopeVersion = 'workflow-trace-intelligence-envelope-v1' -export type WorkflowTraceExportGrantScope = 'workflow-trace:export' | 'workflow-trace:read' | '*' -export type WorkflowTraceExportGrantSubject = 'product' | 'partner' | 'tenant' - -export interface WorkflowTraceExportGrant { - grantId: string - subject: WorkflowTraceExportGrantSubject - subjectId: string - scopes: readonly WorkflowTraceExportGrantScope[] - grantedAt?: string - expiresAt?: string - metadata?: Record -} - -export interface WorkflowTraceHashEvidence { - path: string - sha256: string - shape?: unknown -} - -export interface WorkflowTraceArtifactEvidence { - kind: string - uri: string - contentType?: string - sha256?: string -} - -export interface WorkflowTraceCompactEvidence { - eventKinds: Record - phases: string[] - toolNames: string[] - redactedHashes: WorkflowTraceHashEvidence[] - artifacts: WorkflowTraceArtifactEvidence[] - failureMessage?: string -} - -export interface WorkflowTraceIntelligenceEnvelope { - schemaVersion: WorkflowTraceIntelligenceEnvelopeVersion - destination: string - generatedAt: string - productId: string - partnerId?: string - runId: string - grantIds: string[] - traceEnvelope: WorkflowTraceEnvelope - summary: WorkflowExecutionSummary - compactEvidence: WorkflowTraceCompactEvidence - sanitization: SanitizedWorkflowTraceEnvelopeResult['report'] - links?: WorkflowTraceExportLinks -} - -export interface BuildWorkflowTraceIntelligenceEnvelopeOptions { - envelope: WorkflowTraceEnvelope | unknown - productId: string - partnerId?: string - grants: readonly WorkflowTraceExportGrant[] - generatedAt?: string - destination?: string - sanitize?: SanitizeWorkflowTraceEnvelopeOptions - links?: WorkflowTraceExportLinks - metadata?: Record -} - -const ENVELOPE_VERSION: WorkflowTraceIntelligenceEnvelopeVersion = - 'workflow-trace-intelligence-envelope-v1' -const DEFAULT_DESTINATION = 'intelligence.tangle.tools' - -export function buildWorkflowTraceIntelligenceEnvelope( - options: BuildWorkflowTraceIntelligenceEnvelopeOptions, -): WorkflowTraceIntelligenceEnvelope { - const productId = requireNonEmpty(options.productId, 'productId') - const partnerId = - options.partnerId !== undefined ? requireNonEmpty(options.partnerId, 'partnerId') : undefined - const generatedAt = options.generatedAt ?? new Date().toISOString() - const grantIds = activeExportGrantIds({ - grants: options.grants, - productId, - partnerId, - nowMs: Date.parse(generatedAt), - }) - const base = validateWorkflowTraceEnvelope(options.envelope) - const sourceEnvelope: WorkflowTraceEnvelope = - options.metadata === undefined - ? base - : { - ...base, - metadata: { - ...(base.metadata ?? {}), - intelligenceExport: options.metadata, - }, - } - const sanitized = sanitizeWorkflowTraceEnvelope(sourceEnvelope, options.sanitize) - const summary = summarizeWorkflowExecution(sanitized.envelope) - return { - schemaVersion: ENVELOPE_VERSION, - destination: options.destination ?? DEFAULT_DESTINATION, - generatedAt, - productId, - ...(partnerId ? { partnerId } : {}), - runId: sanitized.envelope.runId, - grantIds, - traceEnvelope: sanitized.envelope, - summary, - compactEvidence: compactEvidence(sanitized.envelope, summary), - sanitization: sanitized.report, - ...(options.links ? { links: options.links } : {}), - } -} - -export function validateWorkflowTraceIntelligenceEnvelope( - input: unknown, -): WorkflowTraceIntelligenceEnvelope { - const obj = expectRecord(input, 'workflow intelligence envelope') - if (obj.schemaVersion !== ENVELOPE_VERSION) { - throw new ValidationError(`workflow intelligence schemaVersion must be ${ENVELOPE_VERSION}`) - } - const destination = expectString(obj.destination, 'destination') - const generatedAt = expectString(obj.generatedAt, 'generatedAt') - const productId = expectString(obj.productId, 'productId') - const partnerId = - obj.partnerId !== undefined ? expectString(obj.partnerId, 'partnerId') : undefined - const runId = expectString(obj.runId, 'runId') - const grantIds = expectStringArray(obj.grantIds, 'grantIds') - const traceEnvelope = validateWorkflowTraceEnvelope(obj.traceEnvelope) - if (traceEnvelope.runId !== runId) { - throw new ValidationError(`workflow intelligence runId ${runId} does not match trace envelope`) - } - const summary = summarizeWorkflowExecution(traceEnvelope) - const compact = validateCompactEvidence(obj.compactEvidence) - const expectedCompact = compactEvidence(traceEnvelope, summary) - assertCompactEvidenceEqual(compact, expectedCompact) - const sanitization = validateSanitizationReport(obj.sanitization) - return { - schemaVersion: ENVELOPE_VERSION, - destination, - generatedAt, - productId, - ...(partnerId ? { partnerId } : {}), - runId, - grantIds, - traceEnvelope, - summary, - compactEvidence: expectedCompact, - sanitization, - ...(obj.links !== undefined ? { links: validateLinks(obj.links) } : {}), - } -} - -function activeExportGrantIds(args: { - grants: readonly WorkflowTraceExportGrant[] - productId: string - partnerId?: string - nowMs: number -}): string[] { - if (!Array.isArray(args.grants) || args.grants.length === 0) { - throw new ValidationError('workflow intelligence export requires at least one opt-in grant') - } - const nowMs = Number.isFinite(args.nowMs) ? args.nowMs : Date.now() - const ids = args.grants - .filter((grant) => grantMatchesSubject(grant, args.productId, args.partnerId)) - .filter((grant) => grant.scopes.includes('workflow-trace:export') || grant.scopes.includes('*')) - .filter((grant) => !grantExpired(grant, nowMs)) - .map((grant) => grant.grantId) - .filter((id) => id.length > 0) - if (ids.length === 0) { - throw new ValidationError( - 'workflow intelligence export requires an active workflow-trace:export grant for the product or partner', - ) - } - return ids -} - -function grantMatchesSubject( - grant: WorkflowTraceExportGrant, - productId: string, - partnerId: string | undefined, -): boolean { - if (grant.subject === 'product') return grant.subjectId === productId - if (grant.subject === 'partner') return partnerId !== undefined && grant.subjectId === partnerId - return grant.subjectId === productId || grant.subjectId === partnerId -} - -function grantExpired(grant: WorkflowTraceExportGrant, nowMs: number): boolean { - if (!grant.expiresAt) return false - const expiresAt = Date.parse(grant.expiresAt) - return Number.isFinite(expiresAt) && expiresAt <= nowMs -} - -function compactEvidence( - envelope: WorkflowTraceEnvelope, - summary: WorkflowExecutionSummary, -): WorkflowTraceCompactEvidence { - return { - eventKinds: summary.eventKinds, - phases: summary.phases, - toolNames: toolNames(envelope.events), - redactedHashes: redactedHashes(envelope), - artifacts: (envelope.artifacts ?? []).map((artifact) => ({ - kind: artifact.kind, - uri: artifact.uri, - ...(artifact.contentType ? { contentType: artifact.contentType } : {}), - ...(artifact.sha256 ? { sha256: artifact.sha256 } : {}), - })), - ...(summary.failureMessage ? { failureMessage: summary.failureMessage } : {}), - } -} - -function toolNames(events: readonly WorkflowTraceEvent[]): string[] { - const names = new Set() - for (const event of events) collectToolNames(event.payload, names) - return [...names].sort() -} - -function collectToolNames(value: unknown, names: Set): void { - if (Array.isArray(value)) { - value.forEach((item) => { - collectToolNames(item, names) - }) - return - } - if (!isRecord(value)) return - const direct = stringValue(value.toolName) ?? stringValue(value.name) - if (direct && looksLikeToolRecord(value)) names.add(direct) - if (isRecord(value.byTool)) { - for (const key of Object.keys(value.byTool)) names.add(key) - } - for (const child of Object.values(value)) collectToolNames(child, names) -} - -function looksLikeToolRecord(value: Record): boolean { - return ( - 'toolName' in value || - 'toolArgs' in value || - 'args' in value || - 'status' in value || - 'error' in value || - 'success' in value - ) -} - -function redactedHashes(envelope: WorkflowTraceEnvelope): WorkflowTraceHashEvidence[] { - const out: WorkflowTraceHashEvidence[] = [] - envelope.events.forEach((event, index) => { - collectHashEvidence(event.payload, `events[${index}].payload`, out) - }) - const artifacts = envelope.artifacts ?? [] - artifacts.forEach((artifact, index) => { - collectHashEvidence(artifact.metadata, `artifacts[${index}].metadata`, out) - }) - collectHashEvidence(envelope.metadata, 'metadata', out) - return out -} - -function collectHashEvidence(value: unknown, path: string, out: WorkflowTraceHashEvidence[]): void { - if (Array.isArray(value)) { - value.forEach((item, index) => { - collectHashEvidence(item, `${path}[${index}]`, out) - }) - return - } - if (!isRecord(value)) return - if (value.redacted === true && typeof value.sha256 === 'string') { - out.push({ - path, - sha256: value.sha256, - ...(value.shape !== undefined ? { shape: value.shape } : {}), - }) - return - } - for (const [key, child] of Object.entries(value)) { - collectHashEvidence(child, `${path}.${key}`, out) - } -} - -function validateCompactEvidence(value: unknown): WorkflowTraceCompactEvidence { - const obj = expectRecord(value, 'compactEvidence') - return { - eventKinds: expectNumberRecord(obj.eventKinds, 'compactEvidence.eventKinds'), - phases: expectStringArray(obj.phases, 'compactEvidence.phases'), - toolNames: expectStringArray(obj.toolNames, 'compactEvidence.toolNames'), - redactedHashes: expectArray(obj.redactedHashes, 'compactEvidence.redactedHashes').map( - (item, index) => { - const record = expectRecord(item, `compactEvidence.redactedHashes[${index}]`) - return { - path: expectString(record.path, `compactEvidence.redactedHashes[${index}].path`), - sha256: expectString(record.sha256, `compactEvidence.redactedHashes[${index}].sha256`), - ...(record.shape !== undefined ? { shape: record.shape } : {}), - } - }, - ), - artifacts: expectArray(obj.artifacts, 'compactEvidence.artifacts').map((item, index) => { - const record = expectRecord(item, `compactEvidence.artifacts[${index}]`) - return { - kind: expectString(record.kind, `compactEvidence.artifacts[${index}].kind`), - uri: expectString(record.uri, `compactEvidence.artifacts[${index}].uri`), - ...(record.contentType !== undefined - ? { - contentType: expectString( - record.contentType, - `compactEvidence.artifacts[${index}].contentType`, - ), - } - : {}), - ...(record.sha256 !== undefined - ? { - sha256: expectString(record.sha256, `compactEvidence.artifacts[${index}].sha256`), - } - : {}), - } - }), - ...(obj.failureMessage !== undefined - ? { failureMessage: expectString(obj.failureMessage, 'compactEvidence.failureMessage') } - : {}), - } -} - -function assertCompactEvidenceEqual( - actual: WorkflowTraceCompactEvidence, - expected: WorkflowTraceCompactEvidence, -): void { - assertNumberRecordEqual(actual.eventKinds, expected.eventKinds, 'compactEvidence.eventKinds') - assertStringArrayEqual(actual.phases, expected.phases, 'compactEvidence.phases') - assertStringArrayEqual(actual.toolNames, expected.toolNames, 'compactEvidence.toolNames') - assertJsonArrayEqual( - actual.redactedHashes, - expected.redactedHashes, - 'compactEvidence.redactedHashes', - ) - assertJsonArrayEqual(actual.artifacts, expected.artifacts, 'compactEvidence.artifacts') - if (actual.failureMessage !== expected.failureMessage) { - throw new ValidationError('compactEvidence.failureMessage does not match trace envelope') - } -} - -function validateSanitizationReport(value: unknown): WorkflowTraceSanitizationReport { - const obj = expectRecord(value, 'sanitization') - return { - redactionCount: expectNonNegativeNumber(obj.redactionCount, 'sanitization.redactionCount'), - byRule: expectNumberRecord(obj.byRule, 'sanitization.byRule'), - hashedArgs: expectNonNegativeNumber(obj.hashedArgs, 'sanitization.hashedArgs'), - truncatedStrings: expectNonNegativeNumber( - obj.truncatedStrings, - 'sanitization.truncatedStrings', - ), - droppedPayloadKeys: expectNumberRecord( - obj.droppedPayloadKeys, - 'sanitization.droppedPayloadKeys', - ), - droppedArtifactContents: expectNonNegativeNumber( - obj.droppedArtifactContents, - 'sanitization.droppedArtifactContents', - ), - } -} - -function validateLinks(value: unknown): WorkflowTraceExportLinks { - const obj = expectRecord(value, 'links') - return { - ...(obj.traceArtifactUri !== undefined - ? { traceArtifactUri: expectString(obj.traceArtifactUri, 'links.traceArtifactUri') } - : {}), - ...(obj.exportBundleUri !== undefined - ? { exportBundleUri: expectString(obj.exportBundleUri, 'links.exportBundleUri') } - : {}), - ...(obj.partnerReportUri !== undefined - ? { partnerReportUri: expectString(obj.partnerReportUri, 'links.partnerReportUri') } - : {}), - ...(obj.intelligenceRunUri !== undefined - ? { intelligenceRunUri: expectString(obj.intelligenceRunUri, 'links.intelligenceRunUri') } - : {}), - } -} - -function requireNonEmpty(value: string, field: string): string { - if (typeof value !== 'string' || value.length === 0) { - throw new ValidationError(`workflow intelligence ${field} must be a non-empty string`) - } - return value -} - -function expectRecord(value: unknown, path: string): Record { - if (!isRecord(value)) throw new ValidationError(`${path}: expected object`) - return value -} - -function expectArray(value: unknown, path: string): unknown[] { - if (!Array.isArray(value)) throw new ValidationError(`${path}: expected array`) - return value -} - -function expectStringArray(value: unknown, path: string): string[] { - return expectArray(value, path).map((item, index) => expectString(item, `${path}[${index}]`)) -} - -function expectNumberRecord(value: unknown, path: string): Record { - const record = expectRecord(value, path) - const out: Record = {} - for (const [key, item] of Object.entries(record)) { - out[key] = expectNonNegativeNumber(item, `${path}.${key}`) - } - return out -} - -function expectString(value: unknown, path: string): string { - if (typeof value !== 'string' || value.length === 0) { - throw new ValidationError(`${path}: expected non-empty string`) - } - return value -} - -function expectNonNegativeNumber(value: unknown, path: string): number { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - throw new ValidationError(`${path}: expected non-negative number`) - } - return value -} - -function assertNumberRecordEqual( - actual: Record, - expected: Record, - path: string, -): void { - const actualKeys = Object.keys(actual).sort() - const expectedKeys = Object.keys(expected).sort() - assertStringArrayEqual(actualKeys, expectedKeys, path) - for (const key of expectedKeys) { - if (actual[key] !== expected[key]) { - throw new ValidationError(`${path}.${key} does not match trace envelope`) - } - } -} - -function assertStringArrayEqual( - actual: readonly string[], - expected: readonly string[], - path: string, -): void { - if (actual.length !== expected.length || actual.some((item, index) => item !== expected[index])) { - throw new ValidationError(`${path} does not match trace envelope`) - } -} - -function assertJsonArrayEqual( - actual: readonly unknown[], - expected: readonly unknown[], - path: string, -): void { - if ( - actual.length !== expected.length || - actual.some((item, index) => JSON.stringify(item) !== JSON.stringify(expected[index])) - ) { - throw new ValidationError(`${path} does not match trace envelope`) - } -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} - -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value) -} diff --git a/src/workflow/partner-report.ts b/src/workflow/partner-report.ts deleted file mode 100644 index c57bdfe3..00000000 --- a/src/workflow/partner-report.ts +++ /dev/null @@ -1,410 +0,0 @@ -import type { AnalystSeverity, EvidenceRef } from '../analyst/types' -import { ValidationError } from '../errors' -import { type RunRecord, validateRunRecord } from '../run-record' -import { - type BuildWorkflowAnalystFeedbackPackOptions, - buildWorkflowAnalystFeedbackPack, -} from './feedback-pack' -import { type WorkflowTraceRunRecordOptions, workflowTraceToRunRecord } from './run-record' -import { - type SanitizedWorkflowTraceEnvelopeResult, - type SanitizeWorkflowTraceEnvelopeOptions, - sanitizeWorkflowTraceEnvelope, -} from './sanitize' -import { summarizeWorkflowTrace, validateWorkflowTraceEnvelope } from './schema' -import { - type WorkflowTraceTrajectoryOptions, - workflowTraceToFeedbackTrajectory, -} from './trajectory' -import type { WorkflowTraceEnvelope, WorkflowTraceExportLinks, WorkflowTraceSummary } from './types' - -export type WorkflowPartnerReportVersion = 'workflow-partner-report-v1' - -export interface WorkflowPartnerFinding { - source: 'analyst' | 'verifier' | 'failure-cluster' - severity: AnalystSeverity - area: string - claim: string - evidence: EvidenceRef[] - recommendedAction?: string - metadata?: Record -} - -export interface WorkflowPartnerReport { - schemaVersion: WorkflowPartnerReportVersion - runId: string - generatedAt: string - summary: WorkflowTraceSummary - docsApiGaps: WorkflowPartnerFinding[] - prReadyFindings: WorkflowPartnerFinding[] - failureClusters: ReturnType['failureClusters'] - recommendations: string[] - traceArtifacts: WorkflowTraceEnvelope['artifacts'] - links?: WorkflowTraceExportLinks - exportBundle: { - traceEnvelope: WorkflowTraceEnvelope - sanitization: SanitizedWorkflowTraceEnvelopeResult['report'] - feedbackPack: ReturnType - trajectory: ReturnType - runRecord?: RunRecord - } -} - -export interface BuildWorkflowPartnerReportOptions - extends Omit { - envelope: WorkflowTraceEnvelope | unknown - sanitize?: SanitizeWorkflowTraceEnvelopeOptions - trajectory: WorkflowTraceTrajectoryOptions - runRecord?: WorkflowTraceRunRecordOptions - links?: WorkflowTraceExportLinks -} - -const REPORT_VERSION: WorkflowPartnerReportVersion = 'workflow-partner-report-v1' - -export function buildWorkflowPartnerReport( - options: BuildWorkflowPartnerReportOptions, -): WorkflowPartnerReport { - const sanitized = sanitizeWorkflowTraceEnvelope(options.envelope, options.sanitize) - const feedbackPack = buildWorkflowAnalystFeedbackPack({ - ...options, - envelope: sanitized.envelope, - }) - const trajectory = workflowTraceToFeedbackTrajectory(sanitized.envelope, options.trajectory) - const runRecord = options.runRecord - ? workflowTraceToRunRecord(sanitized.envelope, { - ...options.runRecord, - runId: sanitized.envelope.runId, - }) - : undefined - const analystFindings: WorkflowPartnerFinding[] = feedbackPack.findings.map((finding) => ({ - source: 'analyst' as const, - severity: finding.severity, - area: finding.area, - claim: finding.claim, - evidence: finding.evidenceRefs, - ...(finding.recommendedAction ? { recommendedAction: finding.recommendedAction } : {}), - metadata: { - findingId: finding.findingId, - analystId: finding.analystId, - confidence: finding.confidence, - ...(finding.subject ? { subject: finding.subject } : {}), - }, - })) - const verifierFindings: WorkflowPartnerFinding[] = (feedbackPack.verifier?.layers ?? []).flatMap( - (layer) => - layer.findings.map((finding) => ({ - source: 'verifier' as const, - severity: finding.severity, - area: layer.layer, - claim: finding.message, - evidence: finding.evidence - ? ([{ kind: 'artifact', uri: finding.evidence }] satisfies EvidenceRef[]) - : [], - ...(layer.reason - ? { recommendedAction: `Fix verifier layer "${layer.layer}": ${layer.reason}` } - : {}), - metadata: { - status: layer.status, - score: layer.score, - }, - })), - ) - const clusterFindings: WorkflowPartnerFinding[] = feedbackPack.failureClusters.map((cluster) => ({ - source: 'failure-cluster' as const, - severity: 'medium' as const, - area: 'failure-cluster', - claim: `${cluster.name} affects ${(cluster.share * 100).toFixed(1)}% of failed workflow runs`, - evidence: cluster.exemplars.map((runId) => ({ - kind: 'artifact' as const, - uri: `run://${runId}`, - })), - ...(cluster.suggestedFix ? { recommendedAction: cluster.suggestedFix } : {}), - metadata: { - clusterId: cluster.id, - runCount: cluster.runCount, - source: cluster.source, - }, - })) - const allFindings = [...analystFindings, ...verifierFindings, ...clusterFindings].sort( - comparePartnerFindings, - ) - - return { - schemaVersion: REPORT_VERSION, - runId: sanitized.envelope.runId, - generatedAt: options.generatedAt ?? feedbackPack.generatedAt, - summary: feedbackPack.summary, - docsApiGaps: allFindings.filter(isDocsApiGap), - prReadyFindings: allFindings.filter(isPrReadyFinding), - failureClusters: feedbackPack.failureClusters, - recommendations: feedbackPack.recommendations, - traceArtifacts: sanitized.envelope.artifacts, - ...(options.links ? { links: options.links } : {}), - exportBundle: { - traceEnvelope: sanitized.envelope, - sanitization: sanitized.report, - feedbackPack, - trajectory, - ...(runRecord ? { runRecord } : {}), - }, - } -} - -export function validateWorkflowPartnerReport(input: unknown): WorkflowPartnerReport { - const obj = expectRecord(input, 'workflow partner report') - if (obj.schemaVersion !== REPORT_VERSION) { - throw new ValidationError(`workflow partner report schemaVersion must be ${REPORT_VERSION}`) - } - - const runId = expectString(obj.runId, 'runId') - const generatedAt = expectString(obj.generatedAt, 'generatedAt') - const exportBundle = validateExportBundle(obj.exportBundle, runId) - const expectedSummary = summarizeWorkflowTrace(exportBundle.traceEnvelope) - assertJsonEqual(expectRecord(obj.summary, 'summary'), expectedSummary, 'summary') - assertJsonEqual( - exportBundle.feedbackPack.summary, - expectedSummary, - 'exportBundle.feedbackPack.summary', - ) - - const traceArtifacts = validateOptionalArtifacts(obj.traceArtifacts, 'traceArtifacts') - assertJsonEqual( - traceArtifacts ?? [], - exportBundle.traceEnvelope.artifacts ?? [], - 'traceArtifacts', - ) - - return { - schemaVersion: REPORT_VERSION, - runId, - generatedAt, - summary: expectedSummary, - docsApiGaps: expectArray(obj.docsApiGaps, 'docsApiGaps') as WorkflowPartnerFinding[], - prReadyFindings: expectArray( - obj.prReadyFindings, - 'prReadyFindings', - ) as WorkflowPartnerFinding[], - failureClusters: expectArray( - obj.failureClusters, - 'failureClusters', - ) as WorkflowPartnerReport['failureClusters'], - recommendations: expectStringArray(obj.recommendations, 'recommendations'), - traceArtifacts, - ...(obj.links !== undefined ? { links: validateLinks(obj.links) } : {}), - exportBundle, - } -} - -export function renderWorkflowPartnerReport( - report: WorkflowPartnerReport, - options: { maxFindings?: number } = {}, -): string { - const maxFindings = options.maxFindings ?? 8 - const lines = [ - `Workflow partner report for ${report.runId}`, - `status=${report.summary.failed ? 'failed' : 'completed'} scoreEvidence events=${report.summary.eventCount} agents=${report.summary.agentCalls} verifier=${report.summary.verifierCalls} analyst=${report.summary.analystCalls} reviewer=${report.summary.reviewerCalls} costUsd=${report.summary.costUsd.toFixed(6)}`, - report.failureClusters.length > 0 ? 'Failure clusters:' : undefined, - ...report.failureClusters - .slice(0, maxFindings) - .map( - (cluster) => - `- ${cluster.name} share=${cluster.share.toFixed(3)} exemplars=${cluster.exemplars.join(',')}`, - ), - report.docsApiGaps.length > 0 ? 'Docs/API gaps:' : undefined, - ...report.docsApiGaps - .slice(0, maxFindings) - .map((finding) => `- ${finding.severity} ${finding.area}: ${finding.claim}`), - report.prReadyFindings.length > 0 ? 'PR-ready findings:' : undefined, - ...report.prReadyFindings - .slice(0, maxFindings) - .map((finding) => `- ${finding.severity} ${finding.area}: ${finding.claim}`), - report.recommendations.length > 0 ? 'Recommendations:' : undefined, - ...report.recommendations.slice(0, maxFindings).map((recommendation) => `- ${recommendation}`), - ].filter((line): line is string => Boolean(line)) - return lines.join('\n') -} - -function validateExportBundle( - value: unknown, - runId: string, -): WorkflowPartnerReport['exportBundle'] { - const obj = expectRecord(value, 'exportBundle') - const traceEnvelope = validateWorkflowTraceEnvelope(obj.traceEnvelope) - if (traceEnvelope.runId !== runId) { - throw new ValidationError('exportBundle.traceEnvelope.runId must match report runId') - } - - const feedbackPack = expectRecord(obj.feedbackPack, 'exportBundle.feedbackPack') - if (feedbackPack.schemaVersion !== 'workflow-feedback-pack-v1') { - throw new ValidationError( - 'exportBundle.feedbackPack.schemaVersion must be workflow-feedback-pack-v1', - ) - } - if (feedbackPack.runId !== runId) { - throw new ValidationError('exportBundle.feedbackPack.runId must match report runId') - } - expectRecord(feedbackPack.toolUsage, 'exportBundle.feedbackPack.toolUsage') - expectArray(feedbackPack.failureClusters, 'exportBundle.feedbackPack.failureClusters') - expectArray(feedbackPack.findings, 'exportBundle.feedbackPack.findings') - expectStringArray(feedbackPack.recommendations, 'exportBundle.feedbackPack.recommendations') - expectStringArray(feedbackPack.driverContextLines, 'exportBundle.feedbackPack.driverContextLines') - - const trajectory = expectRecord(obj.trajectory, 'exportBundle.trajectory') - if (trajectory.id !== runId) { - throw new ValidationError('exportBundle.trajectory.id must match report runId') - } - expectArray(trajectory.attempts, 'exportBundle.trajectory.attempts') - expectArray(trajectory.labels, 'exportBundle.trajectory.labels') - - const runRecord = - obj.runRecord === undefined - ? undefined - : validateWorkflowRunRecord(obj.runRecord, traceEnvelope) - expectRecord(obj.sanitization, 'exportBundle.sanitization') - - return { - traceEnvelope, - sanitization: obj.sanitization as WorkflowPartnerReport['exportBundle']['sanitization'], - feedbackPack: obj.feedbackPack as WorkflowPartnerReport['exportBundle']['feedbackPack'], - trajectory: obj.trajectory as WorkflowPartnerReport['exportBundle']['trajectory'], - ...(runRecord ? { runRecord } : {}), - } -} - -function validateWorkflowRunRecord(value: unknown, envelope: WorkflowTraceEnvelope): RunRecord { - const record = validateRunRecord(value) - const summary = summarizeWorkflowTrace(envelope) - if (record.runId !== envelope.runId) { - throw new ValidationError('exportBundle.runRecord.runId must match trace envelope runId') - } - if (record.outcome.raw.workflow_events !== summary.eventCount) { - throw new ValidationError('exportBundle.runRecord outcome does not match trace event count') - } - return record -} - -function validateLinks(value: unknown): WorkflowTraceExportLinks { - const obj = expectRecord(value, 'links') - return { - ...(obj.traceArtifactUri !== undefined - ? { traceArtifactUri: expectString(obj.traceArtifactUri, 'links.traceArtifactUri') } - : {}), - ...(obj.exportBundleUri !== undefined - ? { exportBundleUri: expectString(obj.exportBundleUri, 'links.exportBundleUri') } - : {}), - ...(obj.partnerReportUri !== undefined - ? { partnerReportUri: expectString(obj.partnerReportUri, 'links.partnerReportUri') } - : {}), - ...(obj.intelligenceRunUri !== undefined - ? { intelligenceRunUri: expectString(obj.intelligenceRunUri, 'links.intelligenceRunUri') } - : {}), - } -} - -function validateOptionalArtifacts( - value: unknown, - path: string, -): WorkflowTraceEnvelope['artifacts'] { - if (value === undefined) return undefined - return expectArray(value, path).map((item, index) => { - const itemPath = `${path}[${index}]` - const obj = expectRecord(item, itemPath) - return { - kind: expectString(obj.kind, `${itemPath}.kind`), - uri: expectString(obj.uri, `${itemPath}.uri`), - ...(obj.contentType !== undefined - ? { contentType: expectString(obj.contentType, `${itemPath}.contentType`) } - : {}), - ...(obj.sha256 !== undefined - ? { sha256: expectString(obj.sha256, `${itemPath}.sha256`) } - : {}), - ...(obj.metadata !== undefined - ? { metadata: expectRecord(obj.metadata, `${itemPath}.metadata`) } - : {}), - } - }) -} - -function expectRecord(value: unknown, path: string): Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new ValidationError(`${path}: expected object`) - } - return value as Record -} - -function expectArray(value: unknown, path: string): unknown[] { - if (!Array.isArray(value)) throw new ValidationError(`${path}: expected array`) - return value -} - -function expectString(value: unknown, path: string): string { - if (typeof value !== 'string' || value.length === 0) { - throw new ValidationError(`${path}: expected non-empty string`) - } - return value -} - -function expectStringArray(value: unknown, path: string): string[] { - return expectArray(value, path).map((item, index) => expectString(item, `${path}[${index}]`)) -} - -function assertJsonEqual(actual: unknown, expected: unknown, path: string): void { - if (JSON.stringify(stableJson(actual)) !== JSON.stringify(stableJson(expected))) { - throw new ValidationError(`${path} does not match trace envelope`) - } -} - -function stableJson(value: unknown): unknown { - if (Array.isArray(value)) return value.map(stableJson) - if (!value || typeof value !== 'object') return value - return Object.fromEntries( - Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, child]) => [key, stableJson(child)]), - ) -} - -function isDocsApiGap(finding: WorkflowPartnerFinding): boolean { - const haystack = [ - finding.area, - finding.claim, - finding.recommendedAction, - typeof finding.metadata?.subject === 'string' ? finding.metadata.subject : undefined, - ] - .filter(Boolean) - .join(' ') - .toLowerCase() - return /\b(api|sdk|docs?|documentation|reference|integration|example|quickstart)\b/.test(haystack) -} - -function isPrReadyFinding(finding: WorkflowPartnerFinding): boolean { - return ( - finding.recommendedAction !== undefined || - finding.severity === 'critical' || - finding.severity === 'high' - ) -} - -function comparePartnerFindings( - left: WorkflowPartnerFinding, - right: WorkflowPartnerFinding, -): number { - const severityDelta = severityRank(right.severity) - severityRank(left.severity) - if (severityDelta !== 0) return severityDelta - return left.area.localeCompare(right.area) -} - -function severityRank(severity: AnalystSeverity): number { - switch (severity) { - case 'critical': - return 5 - case 'high': - return 4 - case 'medium': - return 3 - case 'low': - return 2 - case 'info': - return 1 - } -} diff --git a/src/workflow/phase-graph.ts b/src/workflow/phase-graph.ts deleted file mode 100644 index 0f1e3249..00000000 --- a/src/workflow/phase-graph.ts +++ /dev/null @@ -1,281 +0,0 @@ -import type { RunTokenUsage } from '../run-record' -import { numberField, stringField, tokenUsageField } from './trace-event-fields' -import type { WorkflowTraceEvent } from './types' - -export interface WorkflowPhaseGraphNode { - id: string - title: string - startedAt?: number - endedAt?: number - eventCount: number - branchCount: number - failedBranchCount: number - agentCalls: number - loopCalls: number - verifierCalls: number - analystCalls: number - reviewerCalls: number - agentFailures: number - loopFailures: number - verifierFailures: number - analystFailures: number - reviewerFailures: number - costUsd: number - tokenUsage: RunTokenUsage -} - -export interface WorkflowPhaseGraphBranch { - id: string - operation: string - branchIndex: number - phase: string | null - status: 'started' | 'ended' | 'failed' - startedAt?: number - endedAt?: number - durationMs?: number - stageCount?: number - stageIndex?: number - message?: string - code?: string -} - -export interface WorkflowPhaseGraph { - nodes: WorkflowPhaseGraphNode[] - branches: WorkflowPhaseGraphBranch[] -} - -type MutableWorkflowPhaseGraphNode = WorkflowPhaseGraphNode -type MutableWorkflowPhaseGraphBranch = WorkflowPhaseGraphBranch - -export function workflowPhaseGraph(events: readonly WorkflowTraceEvent[]): WorkflowPhaseGraph { - const nodes = new Map() - const branches: MutableWorkflowPhaseGraphBranch[] = [] - - for (const event of events) { - const phaseTitle = phaseTitleForEvent(event) - if (phaseTitle) observePhaseEvent(phaseNode(nodes, phaseTitle), event) - - if (event.kind === 'workflow.branch.started') { - branches.push(branchStarted(event, branches.length)) - continue - } - - if (event.kind === 'workflow.branch.ended' || event.kind === 'workflow.branch.failed') { - const branch = openBranchFor(branches, event) ?? branchStarted(event, branches.length) - if (!branches.includes(branch)) branches.push(branch) - observeBranchTerminal(branch, event) - } - } - - return { - nodes: Array.from(nodes.values()).map(readonlyPhaseNode), - branches: branches.map(readonlyBranch), - } -} - -function phaseNode( - nodes: Map, - title: string, -): MutableWorkflowPhaseGraphNode { - const existing = nodes.get(title) - if (existing) return existing - const node: MutableWorkflowPhaseGraphNode = { - id: `phase-${nodes.size}`, - title, - eventCount: 0, - branchCount: 0, - failedBranchCount: 0, - agentCalls: 0, - loopCalls: 0, - verifierCalls: 0, - analystCalls: 0, - reviewerCalls: 0, - agentFailures: 0, - loopFailures: 0, - verifierFailures: 0, - analystFailures: 0, - reviewerFailures: 0, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - } - nodes.set(title, node) - return node -} - -function observePhaseEvent(node: MutableWorkflowPhaseGraphNode, event: WorkflowTraceEvent): void { - node.eventCount += 1 - node.startedAt = minDefined(node.startedAt, event.timestamp) - node.endedAt = maxDefined(node.endedAt, event.timestamp) - - switch (event.kind) { - case 'workflow.branch.ended': - node.branchCount += 1 - break - case 'workflow.branch.failed': - node.failedBranchCount += 1 - break - case 'workflow.agent.ended': - node.agentCalls += 1 - break - case 'workflow.agent.failed': - node.agentCalls += 1 - node.agentFailures += 1 - break - case 'workflow.loop.ended': - node.loopCalls += 1 - break - case 'workflow.loop.failed': - node.loopCalls += 1 - node.loopFailures += 1 - break - case 'workflow.verifier.ended': - node.verifierCalls += 1 - break - case 'workflow.verifier.failed': - node.verifierCalls += 1 - node.verifierFailures += 1 - break - case 'workflow.analyst.ended': - node.analystCalls += 1 - break - case 'workflow.analyst.failed': - node.analystCalls += 1 - node.analystFailures += 1 - break - case 'workflow.reviewer.ended': - node.reviewerCalls += 1 - break - case 'workflow.reviewer.failed': - node.reviewerCalls += 1 - node.reviewerFailures += 1 - break - } - - const costUsd = numberField(event.payload, 'costUsd') - if (costUsd !== null) node.costUsd += costUsd - const tokenUsage = tokenUsageField(event.payload.tokenUsage) - if (tokenUsage) addTokenUsage(node.tokenUsage, tokenUsage) -} - -function phaseTitleForEvent(event: WorkflowTraceEvent): string | null { - return event.kind === 'workflow.phase' - ? stringField(event.payload, 'title') - : stringField(event.payload, 'phase') -} - -function branchStarted(event: WorkflowTraceEvent, index: number): MutableWorkflowPhaseGraphBranch { - const branch: MutableWorkflowPhaseGraphBranch = { - id: `branch-${index}`, - operation: stringField(event.payload, 'operation') ?? 'unknown', - branchIndex: numberField(event.payload, 'branchIndex') ?? -1, - phase: stringField(event.payload, 'phase'), - status: 'started', - } - const stageCount = numberField(event.payload, 'stageCount') - if (stageCount !== null) branch.stageCount = stageCount - branch.startedAt = event.timestamp - return branch -} - -function openBranchFor( - branches: readonly MutableWorkflowPhaseGraphBranch[], - event: WorkflowTraceEvent, -): MutableWorkflowPhaseGraphBranch | null { - const operation = stringField(event.payload, 'operation') ?? 'unknown' - const branchIndex = numberField(event.payload, 'branchIndex') ?? -1 - const phase = stringField(event.payload, 'phase') - - for (let i = branches.length - 1; i >= 0; i -= 1) { - const branch = branches[i] - if ( - branch?.status === 'started' && - branch.operation === operation && - branch.branchIndex === branchIndex && - branch.phase === phase - ) { - return branch - } - } - return null -} - -function observeBranchTerminal( - branch: MutableWorkflowPhaseGraphBranch, - event: WorkflowTraceEvent, -): void { - branch.status = event.kind === 'workflow.branch.failed' ? 'failed' : 'ended' - branch.endedAt = event.timestamp - - const durationMs = numberField(event.payload, 'durationMs') - if (durationMs !== null) { - branch.durationMs = durationMs - } else if (branch.startedAt !== undefined) { - branch.durationMs = Math.max(0, event.timestamp - branch.startedAt) - } - - const stageCount = numberField(event.payload, 'stageCount') - if (stageCount !== null) branch.stageCount = stageCount - const stageIndex = numberField(event.payload, 'stageIndex') - if (stageIndex !== null) branch.stageIndex = stageIndex - const message = stringField(event.payload, 'message') - if (message) branch.message = message - const code = stringField(event.payload, 'code') - if (code) branch.code = code -} - -function readonlyPhaseNode(node: MutableWorkflowPhaseGraphNode): WorkflowPhaseGraphNode { - const result: WorkflowPhaseGraphNode = { - id: node.id, - title: node.title, - eventCount: node.eventCount, - branchCount: node.branchCount, - failedBranchCount: node.failedBranchCount, - agentCalls: node.agentCalls, - loopCalls: node.loopCalls, - verifierCalls: node.verifierCalls, - analystCalls: node.analystCalls, - reviewerCalls: node.reviewerCalls, - agentFailures: node.agentFailures, - loopFailures: node.loopFailures, - verifierFailures: node.verifierFailures, - analystFailures: node.analystFailures, - reviewerFailures: node.reviewerFailures, - costUsd: node.costUsd, - tokenUsage: node.tokenUsage, - } - if (node.startedAt !== undefined) result.startedAt = node.startedAt - if (node.endedAt !== undefined) result.endedAt = node.endedAt - return result -} - -function readonlyBranch(branch: MutableWorkflowPhaseGraphBranch): WorkflowPhaseGraphBranch { - const result: WorkflowPhaseGraphBranch = { - id: branch.id, - operation: branch.operation, - branchIndex: branch.branchIndex, - phase: branch.phase, - status: branch.status, - } - if (branch.startedAt !== undefined) result.startedAt = branch.startedAt - if (branch.endedAt !== undefined) result.endedAt = branch.endedAt - if (branch.durationMs !== undefined) result.durationMs = branch.durationMs - if (branch.stageCount !== undefined) result.stageCount = branch.stageCount - if (branch.stageIndex !== undefined) result.stageIndex = branch.stageIndex - if (branch.message !== undefined) result.message = branch.message - if (branch.code !== undefined) result.code = branch.code - return result -} - -function minDefined(current: number | undefined, next: number): number { - return current === undefined ? next : Math.min(current, next) -} - -function maxDefined(current: number | undefined, next: number): number { - return current === undefined ? next : Math.max(current, next) -} - -function addTokenUsage(target: RunTokenUsage, value: RunTokenUsage): void { - target.input += value.input - target.output += value.output - if (value.cached !== undefined) target.cached = (target.cached ?? 0) + value.cached -} diff --git a/src/workflow/promotion-gate.ts b/src/workflow/promotion-gate.ts deleted file mode 100644 index 63b3cb85..00000000 --- a/src/workflow/promotion-gate.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { ValidationError } from '../errors' -import { type RunRecord, validateRunRecord } from '../run-record' -import { type PairedBootstrapResult, pairedBootstrap } from '../statistics' - -export type WorkflowDriverPromotionDecisionVersion = 'workflow-driver-promotion-v1' - -export type WorkflowDriverPromotionRejectionCode = - | 'missing_baseline_records' - | 'missing_candidate_records' - | 'missing_holdout_pairs' - | 'few_pairs' - | 'insufficient_lift' - | 'cost_ceiling' - -export interface WorkflowDriverPromotionPair { - key: string - scenarioId: string - seed: number - baselineRunId: string - candidateRunId: string - baselineScore: number - candidateScore: number - delta: number -} - -export interface WorkflowDriverPromotionEvidence { - pairedRuns: number - expectedScenarioIds: string[] - pairedScenarioIds: string[] - missingScenarioIds: string[] - baselineMean: number - candidateMean: number - lift: number - liftCi: { low: number; high: number } - bootstrap: PairedBootstrapResult - confidence: number - resamples: number - statistic: 'mean' | 'median' - deltaThreshold: number - baselineMedianCostUsd: number - candidateMedianCostUsd: number - pairs: WorkflowDriverPromotionPair[] -} - -export interface WorkflowDriverPromotionDecision { - schemaVersion: WorkflowDriverPromotionDecisionVersion - generatedAt: string - baselineStrategyId: string - candidateStrategyId: string - promote: boolean - rejectionCode: WorkflowDriverPromotionRejectionCode | null - reason: string - evidence: WorkflowDriverPromotionEvidence -} - -export interface DecideWorkflowDriverPromotionOptions { - records: readonly RunRecord[] | readonly unknown[] - baselineStrategyId?: string - candidateStrategyId?: string - expectedScenarioIds?: readonly string[] - minPairedHoldoutRuns?: number - deltaThreshold?: number - confidence?: number - resamples?: number - seed?: number - statistic?: 'mean' | 'median' - costPerRunCeiling?: number - generatedAt?: string -} - -const DECISION_VERSION: WorkflowDriverPromotionDecisionVersion = 'workflow-driver-promotion-v1' -const DEFAULT_BASELINE_STRATEGY = 'reviewer-loop-v1' -const DEFAULT_CANDIDATE_STRATEGY = 'workflow-driver-v1' - -export function decideWorkflowDriverPromotion( - options: DecideWorkflowDriverPromotionOptions, -): WorkflowDriverPromotionDecision { - const records = options.records.map(validateRunRecord) - const baselineStrategyId = options.baselineStrategyId ?? DEFAULT_BASELINE_STRATEGY - const candidateStrategyId = options.candidateStrategyId ?? DEFAULT_CANDIDATE_STRATEGY - const confidence = options.confidence ?? 0.95 - const resamples = options.resamples ?? 2000 - const statistic = options.statistic ?? 'mean' - const deltaThreshold = options.deltaThreshold ?? 0 - const minPairedHoldoutRuns = options.minPairedHoldoutRuns ?? 3 - validateOptions({ confidence, resamples, deltaThreshold, minPairedHoldoutRuns }, options) - - const baseline = records.filter((record) => record.candidateId === baselineStrategyId) - const candidate = records.filter((record) => record.candidateId === candidateStrategyId) - const baselineHoldout = baseline.filter(isScoredHoldout) - const candidateHoldout = candidate.filter(isScoredHoldout) - const expectedScenarioIds = expectedScenarios(options.expectedScenarioIds, [ - ...baselineHoldout, - ...candidateHoldout, - ]) - const expectedScenarioIdSet = new Set(expectedScenarioIds) - const pairs = pairHoldoutRuns(baselineHoldout, candidateHoldout).filter( - (pair) => expectedScenarioIdSet.size === 0 || expectedScenarioIdSet.has(pair.scenarioId), - ) - const pairedScenarioIds = [...new Set(pairs.map((pair) => pair.scenarioId))].sort() - const missingScenarioIds = expectedScenarioIds.filter((id) => !pairedScenarioIds.includes(id)) - const evidence = buildEvidence({ - pairs, - expectedScenarioIds, - pairedScenarioIds, - missingScenarioIds, - baseline, - candidate, - confidence, - resamples, - statistic, - deltaThreshold, - seed: options.seed, - }) - - if (baselineHoldout.length === 0) { - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: false, - rejectionCode: 'missing_baseline_records', - reason: `missing_baseline_records: no holdout RunRecords for baseline "${baselineStrategyId}"`, - }) - } - if (candidateHoldout.length === 0) { - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: false, - rejectionCode: 'missing_candidate_records', - reason: `missing_candidate_records: no holdout RunRecords for candidate "${candidateStrategyId}"`, - }) - } - if (missingScenarioIds.length > 0) { - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: false, - rejectionCode: 'missing_holdout_pairs', - reason: `missing_holdout_pairs: no paired baseline/candidate holdout record for scenario(s) [${missingScenarioIds.join(', ')}]`, - }) - } - if (pairs.length < minPairedHoldoutRuns) { - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: false, - rejectionCode: 'few_pairs', - reason: `few_pairs: ${pairs.length} paired holdout run(s) < min ${minPairedHoldoutRuns}`, - }) - } - if (!(evidence.liftCi.low > deltaThreshold)) { - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: false, - rejectionCode: 'insufficient_lift', - reason: - `insufficient_lift: heldout ${statistic} lift=${fmt(evidence.lift)} ` + - `CI=[${fmt(evidence.liftCi.low)}, ${fmt(evidence.liftCi.high)}] does not clear threshold ${fmt(deltaThreshold)}`, - }) - } - if ( - options.costPerRunCeiling !== undefined && - Number.isFinite(evidence.candidateMedianCostUsd) && - evidence.candidateMedianCostUsd > options.costPerRunCeiling - ) { - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: false, - rejectionCode: 'cost_ceiling', - reason: - `cost_ceiling: candidate median cost $${fmt(evidence.candidateMedianCostUsd)} ` + - `exceeds ceiling $${fmt(options.costPerRunCeiling)}`, - }) - } - - return decision({ - options, - baselineStrategyId, - candidateStrategyId, - evidence, - promote: true, - rejectionCode: null, - reason: - `promote: ${candidateStrategyId} beats ${baselineStrategyId} on paired heldout workflows ` + - `lift=${fmt(evidence.lift)} CI=[${fmt(evidence.liftCi.low)}, ${fmt(evidence.liftCi.high)}] ` + - `over ${pairs.length} pair(s)`, - }) -} - -function validateOptions( - normalized: { - confidence: number - resamples: number - deltaThreshold: number - minPairedHoldoutRuns: number - }, - options: DecideWorkflowDriverPromotionOptions, -): void { - if (options.records.length === 0) { - throw new ValidationError('workflow promotion gate requires at least one RunRecord') - } - if (normalized.confidence <= 0 || normalized.confidence >= 1) { - throw new ValidationError('workflow promotion gate confidence must be in (0,1)') - } - if (!Number.isInteger(normalized.resamples) || normalized.resamples <= 0) { - throw new ValidationError('workflow promotion gate resamples must be a positive integer') - } - if (!Number.isFinite(normalized.deltaThreshold)) { - throw new ValidationError('workflow promotion gate deltaThreshold must be finite') - } - if (!Number.isInteger(normalized.minPairedHoldoutRuns) || normalized.minPairedHoldoutRuns < 1) { - throw new ValidationError( - 'workflow promotion gate minPairedHoldoutRuns must be a positive integer', - ) - } - if ( - options.costPerRunCeiling !== undefined && - (!Number.isFinite(options.costPerRunCeiling) || options.costPerRunCeiling <= 0) - ) { - throw new ValidationError('workflow promotion gate costPerRunCeiling must be positive') - } -} - -function isScoredHoldout(record: RunRecord): boolean { - return record.splitTag === 'holdout' && typeof record.outcome.holdoutScore === 'number' -} - -function expectedScenarios( - requested: readonly string[] | undefined, - records: readonly RunRecord[], -): string[] { - const values = (requested ?? records.map((record) => record.scenarioId)).filter(isString) - return [...new Set(values)].sort() -} - -function pairHoldoutRuns( - baseline: readonly RunRecord[], - candidate: readonly RunRecord[], -): WorkflowDriverPromotionPair[] { - const baselineByKey = indexByPairKey(baseline, 'baseline') - const out: WorkflowDriverPromotionPair[] = [] - for (const candidateRun of candidate) { - const key = holdoutPairKey(candidateRun, 'candidate') - const baselineRun = baselineByKey.get(key) - if (!baselineRun) continue - const baselineScore = baselineRun.outcome.holdoutScore! - const candidateScore = candidateRun.outcome.holdoutScore! - out.push({ - key, - scenarioId: candidateRun.scenarioId!, - seed: candidateRun.seed, - baselineRunId: baselineRun.runId, - candidateRunId: candidateRun.runId, - baselineScore, - candidateScore, - delta: candidateScore - baselineScore, - }) - } - return out.sort((a, b) => a.key.localeCompare(b.key)) -} - -function indexByPairKey( - records: readonly RunRecord[], - side: 'baseline' | 'candidate', -): Map { - const out = new Map() - for (const record of records) { - const key = holdoutPairKey(record, side) - if (out.has(key)) { - throw new ValidationError( - `workflow promotion gate duplicate ${side} holdout pair key: ${key}`, - ) - } - out.set(key, record) - } - return out -} - -function holdoutPairKey(record: RunRecord, side: 'baseline' | 'candidate'): string { - if (!record.scenarioId) { - throw new ValidationError( - `workflow promotion gate ${side} holdout RunRecord ${record.runId} is missing scenarioId`, - ) - } - return `${record.scenarioId}::${record.seed}` -} - -function buildEvidence(args: { - pairs: readonly WorkflowDriverPromotionPair[] - expectedScenarioIds: string[] - pairedScenarioIds: string[] - missingScenarioIds: string[] - baseline: readonly RunRecord[] - candidate: readonly RunRecord[] - confidence: number - resamples: number - statistic: 'mean' | 'median' - deltaThreshold: number - seed?: number -}): WorkflowDriverPromotionEvidence { - const before = args.pairs.map((pair) => pair.baselineScore) - const after = args.pairs.map((pair) => pair.candidateScore) - const bootstrap = pairedBootstrap(before, after, { - confidence: args.confidence, - resamples: args.resamples, - statistic: args.statistic, - ...(args.seed !== undefined ? { seed: args.seed } : {}), - }) - return { - pairedRuns: args.pairs.length, - expectedScenarioIds: args.expectedScenarioIds, - pairedScenarioIds: args.pairedScenarioIds, - missingScenarioIds: args.missingScenarioIds, - baselineMean: mean(before), - candidateMean: mean(after), - lift: args.statistic === 'mean' ? bootstrap.mean : bootstrap.median, - liftCi: { low: bootstrap.low, high: bootstrap.high }, - bootstrap, - confidence: args.confidence, - resamples: args.resamples, - statistic: args.statistic, - deltaThreshold: args.deltaThreshold, - baselineMedianCostUsd: medianFinite(args.baseline.map((record) => record.costUsd)), - candidateMedianCostUsd: medianFinite(args.candidate.map((record) => record.costUsd)), - pairs: [...args.pairs], - } -} - -function decision(args: { - options: DecideWorkflowDriverPromotionOptions - baselineStrategyId: string - candidateStrategyId: string - evidence: WorkflowDriverPromotionEvidence - promote: boolean - rejectionCode: WorkflowDriverPromotionRejectionCode | null - reason: string -}): WorkflowDriverPromotionDecision { - return { - schemaVersion: DECISION_VERSION, - generatedAt: args.options.generatedAt ?? new Date().toISOString(), - baselineStrategyId: args.baselineStrategyId, - candidateStrategyId: args.candidateStrategyId, - promote: args.promote, - rejectionCode: args.rejectionCode, - reason: args.reason, - evidence: args.evidence, - } -} - -function mean(values: readonly number[]): number { - if (values.length === 0) return Number.NaN - return values.reduce((sum, value) => sum + value, 0) / values.length -} - -function medianFinite(values: readonly number[]): number { - const finite = values.filter(Number.isFinite).sort((a, b) => a - b) - if (finite.length === 0) return Number.NaN - const mid = Math.floor(finite.length / 2) - return finite.length % 2 === 0 ? (finite[mid - 1]! + finite[mid]!) / 2 : finite[mid]! -} - -function fmt(value: number): string { - if (!Number.isFinite(value)) return String(value) - return value.toFixed(4) -} - -function isString(value: unknown): value is string { - return typeof value === 'string' && value.length > 0 -} diff --git a/src/workflow/run-record.ts b/src/workflow/run-record.ts deleted file mode 100644 index c050fbe7..00000000 --- a/src/workflow/run-record.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { type RunRecord, validateRunRecord } from '../run-record' -import { summarizeWorkflowTrace, validateWorkflowTraceEnvelope } from './schema' -import type { WorkflowTraceEnvelope, WorkflowTraceProjectionMetadata } from './types' - -export interface WorkflowTraceRunRecordOptions extends WorkflowTraceProjectionMetadata { - runId?: string - score?: number - raw?: Record - failureMode?: string - judgeMetadata?: RunRecord['judgeMetadata'] - agentProfile?: RunRecord['agentProfile'] -} - -export function workflowTraceToRunRecord( - input: WorkflowTraceEnvelope | unknown, - options: WorkflowTraceRunRecordOptions, -): RunRecord { - const envelope = validateWorkflowTraceEnvelope(input) - const summary = summarizeWorkflowTrace(envelope) - const score = clampScore(options.score ?? (summary.failed ? 0 : 1)) - const raw = { - ...finiteOnly(options.raw ?? {}), - score, - workflow_failed: summary.failed ? 1 : 0, - workflow_events: summary.eventCount, - workflow_phases: summary.phaseCount, - workflow_branches: summary.branchCount, - workflow_branch_failures: summary.failedBranchCount, - workflow_agent_calls: summary.agentCalls, - workflow_loop_calls: summary.loopCalls, - workflow_verifier_calls: summary.verifierCalls, - workflow_analyst_calls: summary.analystCalls, - workflow_reviewer_calls: summary.reviewerCalls, - workflow_agent_failures: summary.agentFailures, - workflow_loop_failures: summary.loopFailures, - workflow_verifier_failures: summary.verifierFailures, - workflow_analyst_failures: summary.analystFailures, - workflow_reviewer_failures: summary.reviewerFailures, - } - const outcome = - options.splitTag === 'holdout' ? { holdoutScore: score, raw } : { searchScore: score, raw } - - return validateRunRecord({ - runId: options.runId ?? envelope.runId, - experimentId: options.experimentId, - candidateId: options.candidateId, - seed: options.seed, - model: options.model, - promptHash: options.promptHash, - configHash: options.configHash, - commitSha: options.commitSha, - wallMs: summary.durationMs, - costUsd: summary.costUsd, - tokenUsage: summary.tokenUsage, - ...(options.judgeMetadata ? { judgeMetadata: options.judgeMetadata } : {}), - outcome, - failureMode: options.failureMode ?? (summary.failed ? 'workflow_failed' : undefined), - splitTag: options.splitTag, - ...(options.scenarioId ? { scenarioId: options.scenarioId } : {}), - ...(options.agentProfile ? { agentProfile: options.agentProfile } : {}), - }) -} - -function finiteOnly(values: Record): Record { - const out: Record = {} - for (const [key, value] of Object.entries(values)) { - if (Number.isFinite(value)) out[key] = value - } - return out -} - -function clampScore(value: number): number { - if (!Number.isFinite(value)) return 0 - return Math.max(0, Math.min(1, value)) -} diff --git a/src/workflow/runtime-adapter.ts b/src/workflow/runtime-adapter.ts deleted file mode 100644 index 8865914f..00000000 --- a/src/workflow/runtime-adapter.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { ValidationError } from '../errors' -import { validateWorkflowTraceEnvelope, validateWorkflowTraceEvent } from './schema' -import type { WorkflowTopology, WorkflowTraceArtifact, WorkflowTraceEnvelope } from './types' - -export interface WorkflowTraceEnvelopeFromEventsOptions { - runId?: string - topology?: WorkflowTopology - artifacts?: readonly WorkflowTraceArtifact[] - metadata?: Record -} - -export interface WorkflowRuntimeResultLike { - runId?: string - meta?: unknown - output?: unknown - events: readonly unknown[] -} - -export interface WorkflowRuntimeResultToTraceEnvelopeOptions - extends Omit { - runId?: string - includeOutputInMetadata?: boolean -} - -export function workflowEventsToTraceEnvelope( - events: readonly unknown[], - options: WorkflowTraceEnvelopeFromEventsOptions = {}, -): WorkflowTraceEnvelope { - if (!Array.isArray(events) || events.length === 0) { - throw new ValidationError('workflow trace events must be a non-empty array') - } - const first = validateWorkflowTraceEvent(events[0]) - const runId = options.runId ?? first.runId - return validateWorkflowTraceEnvelope({ - traceVersion: 'workflow-trace-v1', - runId, - ...(options.topology ? { topology: options.topology } : {}), - events, - ...(options.artifacts ? { artifacts: options.artifacts } : {}), - ...(options.metadata ? { metadata: options.metadata } : {}), - }) -} - -export function workflowRuntimeResultToTraceEnvelope( - result: WorkflowRuntimeResultLike, - options: WorkflowRuntimeResultToTraceEnvelopeOptions = {}, -): WorkflowTraceEnvelope { - if (!result || typeof result !== 'object') { - throw new ValidationError('workflow runtime result must be an object') - } - const metadata = runtimeResultMetadata(result, options) - return workflowEventsToTraceEnvelope(result.events, { - runId: options.runId ?? result.runId, - ...(options.topology ? { topology: options.topology } : {}), - ...(options.artifacts ? { artifacts: options.artifacts } : {}), - ...(metadata ? { metadata } : {}), - }) -} - -function runtimeResultMetadata( - result: WorkflowRuntimeResultLike, - options: WorkflowRuntimeResultToTraceEnvelopeOptions, -): Record | undefined { - const runtimeResult: Record = {} - if (result.meta !== undefined) runtimeResult.meta = result.meta - if (options.includeOutputInMetadata && result.output !== undefined) { - runtimeResult.output = result.output - } - const hasRuntimeMetadata = Object.keys(runtimeResult).length > 0 - if (!hasRuntimeMetadata && !options.metadata) return undefined - return { - ...(options.metadata ?? {}), - ...(hasRuntimeMetadata ? { runtimeResult } : {}), - } -} diff --git a/src/workflow/sanitize.ts b/src/workflow/sanitize.ts deleted file mode 100644 index eb3cbc45..00000000 --- a/src/workflow/sanitize.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { createHash } from 'node:crypto' - -import { - DEFAULT_REDACTION_RULES, - type RedactionReport, - type RedactionRule, - redactString, -} from '../trace/redact' -import { validateWorkflowTraceEnvelope } from './schema' -import type { WorkflowTraceArtifact, WorkflowTraceEnvelope, WorkflowTraceEvent } from './types' - -export interface WorkflowTraceSanitizationReport extends RedactionReport { - hashedArgs: number - truncatedStrings: number - droppedPayloadKeys: Record - droppedArtifactContents: number -} - -export interface SanitizeWorkflowTraceEnvelopeOptions { - rules?: readonly RedactionRule[] - maxStringLength?: number - hashSalt?: string - approvedArtifactUris?: readonly string[] - approvedArtifactKinds?: readonly string[] -} - -export interface SanitizedWorkflowTraceEnvelopeResult { - envelope: WorkflowTraceEnvelope - report: WorkflowTraceSanitizationReport -} - -const DEFAULT_MAX_STRING_LENGTH = 600 - -const SECRET_KEY_RE = - /^(authorization|cookie|set-cookie|x-api-key|api[-_]?key|token|access[-_]?token|refresh[-_]?token|secret|password|passwd|session|credential|credentials)$/i -const ARG_KEY_RE = /^(args|arguments|rawargs|raw_args|toolargs|tool_args|inputargs|input_args)$/i -const FILE_CONTENT_KEY_RE = /^(filecontent|filecontents|file_content|file_contents|contents)$/i -const FILE_HINT_KEY_RE = /^(path|filepath|file_path|filename|file|uri)$/i - -export function sanitizeWorkflowTraceEnvelope( - input: WorkflowTraceEnvelope | unknown, - options: SanitizeWorkflowTraceEnvelopeOptions = {}, -): SanitizedWorkflowTraceEnvelopeResult { - const envelope = validateWorkflowTraceEnvelope(input) - const report = emptyReport() - const ctx: SanitizeContext = { - rules: [...(options.rules ?? DEFAULT_REDACTION_RULES)], - maxStringLength: options.maxStringLength ?? DEFAULT_MAX_STRING_LENGTH, - hashSalt: options.hashSalt ?? '', - approvedArtifactUris: new Set(options.approvedArtifactUris ?? []), - approvedArtifactKinds: new Set(options.approvedArtifactKinds ?? []), - report, - } - - return { - envelope: { - traceVersion: envelope.traceVersion, - runId: envelope.runId, - ...(envelope.topology ? { topology: sanitizeValue(envelope.topology, ctx) as never } : {}), - events: envelope.events.map((event) => sanitizeWorkflowTraceEvent(event, ctx)), - ...(envelope.artifacts ? { artifacts: sanitizeArtifacts(envelope.artifacts, ctx) } : {}), - ...(envelope.metadata - ? { metadata: sanitizeRecord(envelope.metadata, ctx) as Record } - : {}), - }, - report, - } -} - -function sanitizeWorkflowTraceEvent( - event: WorkflowTraceEvent, - ctx: SanitizeContext, -): WorkflowTraceEvent { - return { - kind: event.kind, - runId: event.runId, - timestamp: event.timestamp, - payload: sanitizeRecord(event.payload, ctx) as Record, - } -} - -function sanitizeArtifacts( - artifacts: readonly WorkflowTraceArtifact[], - ctx: SanitizeContext, -): WorkflowTraceArtifact[] { - return artifacts.map((artifact) => { - const approved = - ctx.approvedArtifactUris.has(artifact.uri) || ctx.approvedArtifactKinds.has(artifact.kind) - const metadata = artifact.metadata - ? sanitizeRecord(artifact.metadata, ctx, { artifactApproved: approved }) - : undefined - return { - kind: artifact.kind, - uri: sanitizeString(artifact.uri, ctx), - ...(artifact.contentType ? { contentType: sanitizeString(artifact.contentType, ctx) } : {}), - ...(artifact.sha256 ? { sha256: artifact.sha256 } : {}), - ...(metadata ? { metadata: metadata as Record } : {}), - } - }) -} - -function sanitizeRecord( - record: Record, - ctx: SanitizeContext, - options: { artifactApproved?: boolean } = {}, -): Record { - const out: Record = {} - const hasFileHint = Object.keys(record).some((key) => FILE_HINT_KEY_RE.test(key)) - for (const [key, value] of Object.entries(record)) { - if (SECRET_KEY_RE.test(key)) { - out[key] = `[redacted:${key}]` - increment(ctx.report.droppedPayloadKeys, key) - continue - } - if (ARG_KEY_RE.test(key)) { - out[key] = hashedValue(value, ctx) - ctx.report.hashedArgs += 1 - continue - } - if (!options.artifactApproved && isFileContentKey(key, hasFileHint)) { - out[key] = hashedValue(value, ctx) - ctx.report.droppedArtifactContents += 1 - continue - } - out[key] = sanitizeValue(value, ctx) - } - return out -} - -function sanitizeValue(value: unknown, ctx: SanitizeContext): unknown { - if (typeof value === 'string') return sanitizeString(value, ctx) - if (Array.isArray(value)) return value.map((item) => sanitizeValue(item, ctx)) - if (isRecord(value)) return sanitizeRecord(value, ctx) - return value -} - -function sanitizeString(value: string, ctx: SanitizeContext): string { - const redacted = redactString(value, ctx.rules) - ctx.report.redactionCount += redacted.report.redactionCount - for (const [rule, count] of Object.entries(redacted.report.byRule)) { - ctx.report.byRule[rule] = (ctx.report.byRule[rule] ?? 0) + count - } - if (redacted.output.length <= ctx.maxStringLength) return redacted.output - ctx.report.truncatedStrings += 1 - return `${redacted.output.slice(0, Math.max(0, ctx.maxStringLength - 1))}…` -} - -function hashedValue(value: unknown, ctx: SanitizeContext): Record { - return { - redacted: true, - sha256: sha256Stable(value, ctx.hashSalt), - shape: valueShape(value), - } -} - -function sha256Stable(value: unknown, salt: string): string { - return createHash('sha256').update(salt).update(stableStringify(value)).digest('hex') -} - -function stableStringify(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` - if (isRecord(value)) { - return `{${Object.keys(value) - .sort() - .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) - .join(',')}}` - } - return JSON.stringify(value) -} - -function valueShape(value: unknown): unknown { - if (Array.isArray(value)) return { type: 'array', length: value.length } - if (isRecord(value)) { - return { - type: 'object', - keys: Object.keys(value).sort(), - } - } - return { type: typeof value } -} - -function isFileContentKey(key: string, hasFileHint: boolean): boolean { - if (FILE_CONTENT_KEY_RE.test(key)) return true - return hasFileHint && /^(content|source|diff)$/i.test(key) -} - -function emptyReport(): WorkflowTraceSanitizationReport { - return { - redactionCount: 0, - byRule: {}, - hashedArgs: 0, - truncatedStrings: 0, - droppedPayloadKeys: {}, - droppedArtifactContents: 0, - } -} - -function increment(record: Record, key: string): void { - record[key] = (record[key] ?? 0) + 1 -} - -function isRecord(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value) -} - -interface SanitizeContext { - rules: RedactionRule[] - maxStringLength: number - hashSalt: string - approvedArtifactUris: Set - approvedArtifactKinds: Set - report: WorkflowTraceSanitizationReport -} diff --git a/src/workflow/schema.ts b/src/workflow/schema.ts deleted file mode 100644 index 0fece6e2..00000000 --- a/src/workflow/schema.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { ValidationError } from '../errors' -import { validateWorkflowTraceEventKind, validateWorkflowTraceEventPayload } from './event-schema' -import type { - WorkflowTraceEnvelope, - WorkflowTraceEvent, - WorkflowTraceSummary, - WorkflowTraceVersion, -} from './types' - -const TRACE_VERSION: WorkflowTraceVersion = 'workflow-trace-v1' - -export function validateWorkflowTraceEvent(input: unknown): WorkflowTraceEvent { - const obj = expectRecord(input, 'event') - const kind = validateWorkflowTraceEventKind(expectString(obj.kind, 'event.kind')) - const runId = expectString(obj.runId, 'event.runId') - const timestamp = expectFinite(obj.timestamp, 'event.timestamp') - const payload = validateWorkflowTraceEventPayload( - kind, - expectRecord(obj.payload, 'event.payload'), - ) - return { kind, runId, timestamp, payload } -} - -export function validateWorkflowTraceEnvelope(input: unknown): WorkflowTraceEnvelope { - const obj = expectRecord(input, 'workflow trace envelope') - if (obj.traceVersion !== TRACE_VERSION) { - throw new ValidationError(`workflow traceVersion must be ${TRACE_VERSION}`) - } - const runId = expectString(obj.runId, 'workflow trace runId') - if (!Array.isArray(obj.events) || obj.events.length === 0) { - throw new ValidationError('workflow trace envelope must include at least one event') - } - const events = obj.events.map(validateWorkflowTraceEvent) - for (const event of events) { - if (event.runId !== runId) { - throw new ValidationError(`workflow trace event runId ${event.runId} does not match ${runId}`) - } - } - validateWorkflowTraceLifecycle(events) - return { - traceVersion: TRACE_VERSION, - runId, - ...(obj.topology !== undefined ? { topology: obj.topology as never } : {}), - events, - ...(obj.artifacts !== undefined ? { artifacts: validateArtifacts(obj.artifacts) } : {}), - ...(obj.metadata !== undefined ? { metadata: expectRecord(obj.metadata, 'metadata') } : {}), - } -} - -function validateWorkflowTraceLifecycle(events: readonly WorkflowTraceEvent[]): void { - const first = events[0] - if (first?.kind !== 'workflow.started') { - throw new ValidationError('workflow trace first event must be workflow.started') - } - for (let index = 1; index < events.length; index += 1) { - if (events[index]!.timestamp < events[index - 1]!.timestamp) { - throw new ValidationError(`workflow trace timestamps must be nondecreasing at event ${index}`) - } - } - const terminalEvents = events.filter( - (event) => event.kind === 'workflow.ended' || event.kind === 'workflow.failed', - ) - if (terminalEvents.length !== 1) { - throw new ValidationError( - `workflow trace must include exactly one terminal event, got ${terminalEvents.length}`, - ) - } - const last = events.at(-1) - if (last?.kind !== terminalEvents[0]?.kind) { - throw new ValidationError('workflow trace terminal event must be last') - } -} - -export function summarizeWorkflowTrace( - input: WorkflowTraceEnvelope | unknown, -): WorkflowTraceSummary { - const envelope = validateWorkflowTraceEnvelope(input) - const started = envelope.events.find((e) => e.kind === 'workflow.started') - const ended = envelope.events.find((e) => e.kind === 'workflow.ended') - const failed = envelope.events.find((e) => e.kind === 'workflow.failed') - const endedPayload = ended?.payload ?? {} - const tokenUsage = tokenUsageOf(endedPayload.tokenUsage) - return { - runId: envelope.runId, - startedAt: started?.timestamp, - endedAt: ended?.timestamp, - durationMs: finiteOr(endedPayload.durationMs, durationFromEvents(envelope.events)), - costUsd: finiteOr(endedPayload.costUsd, 0), - tokenUsage, - phaseCount: envelope.events.filter((e) => e.kind === 'workflow.phase').length, - branchCount: envelope.events.filter((e) => e.kind === 'workflow.branch.ended').length, - failedBranchCount: envelope.events.filter((e) => e.kind === 'workflow.branch.failed').length, - agentCalls: finiteOr( - endedPayload.agentCalls, - countEvents(envelope.events, 'workflow.agent.ended', 'workflow.agent.failed'), - ), - loopCalls: finiteOr( - endedPayload.loopCalls, - countEvents(envelope.events, 'workflow.loop.ended', 'workflow.loop.failed'), - ), - verifierCalls: countEvents( - envelope.events, - 'workflow.verifier.ended', - 'workflow.verifier.failed', - ), - analystCalls: countEvents(envelope.events, 'workflow.analyst.ended', 'workflow.analyst.failed'), - reviewerCalls: countEvents( - envelope.events, - 'workflow.reviewer.ended', - 'workflow.reviewer.failed', - ), - agentFailures: countEvents(envelope.events, 'workflow.agent.failed'), - loopFailures: countEvents(envelope.events, 'workflow.loop.failed'), - verifierFailures: countEvents(envelope.events, 'workflow.verifier.failed'), - analystFailures: countEvents(envelope.events, 'workflow.analyst.failed'), - reviewerFailures: countEvents(envelope.events, 'workflow.reviewer.failed'), - eventCount: envelope.events.length, - failed: failed !== undefined, - failureMessage: - typeof failed?.payload.message === 'string' ? failed.payload.message : undefined, - } -} - -function countEvents( - events: readonly WorkflowTraceEvent[], - ...kinds: readonly WorkflowTraceEvent['kind'][] -): number { - const allowed = new Set(kinds) - return events.filter((event) => allowed.has(event.kind)).length -} - -function validateArtifacts(value: unknown): WorkflowTraceEnvelope['artifacts'] { - if (!Array.isArray(value)) throw new ValidationError('workflow artifacts must be an array') - return value.map((artifact, index) => { - const obj = expectRecord(artifact, `artifacts[${index}]`) - return { - kind: expectString(obj.kind, `artifacts[${index}].kind`), - uri: expectString(obj.uri, `artifacts[${index}].uri`), - ...(obj.contentType !== undefined - ? { contentType: expectString(obj.contentType, `artifacts[${index}].contentType`) } - : {}), - ...(obj.sha256 !== undefined - ? { sha256: expectString(obj.sha256, `artifacts[${index}].sha256`) } - : {}), - ...(obj.metadata !== undefined - ? { metadata: expectRecord(obj.metadata, `artifacts[${index}].metadata`) } - : {}), - } - }) -} - -function durationFromEvents(events: readonly WorkflowTraceEvent[]): number { - const first = events[0]?.timestamp - const last = events.at(-1)?.timestamp - if (!Number.isFinite(first) || !Number.isFinite(last)) return 0 - return Math.max(0, (last as number) - (first as number)) -} - -function tokenUsageOf(value: unknown): { input: number; output: number; cached?: number } { - const obj = value && typeof value === 'object' ? (value as Record) : {} - const cached = finiteOrUndefined(obj.cached) - return { - input: finiteOr(obj.input, 0), - output: finiteOr(obj.output, 0), - ...(cached !== undefined ? { cached } : {}), - } -} - -function finiteOr(value: unknown, fallback: number): number { - return typeof value === 'number' && Number.isFinite(value) ? value : fallback -} - -function finiteOrUndefined(value: unknown): number | undefined { - return typeof value === 'number' && Number.isFinite(value) ? value : undefined -} - -function expectRecord(value: unknown, path: string): Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new ValidationError(`${path}: expected object`) - } - return value as Record -} - -function expectString(value: unknown, path: string): string { - if (typeof value !== 'string' || value.length === 0) { - throw new ValidationError(`${path}: expected non-empty string`) - } - return value -} - -function expectFinite(value: unknown, path: string): number { - if (typeof value !== 'number' || !Number.isFinite(value)) { - throw new ValidationError(`${path}: expected finite number`) - } - return value -} diff --git a/src/workflow/summary.ts b/src/workflow/summary.ts deleted file mode 100644 index 6111b374..00000000 --- a/src/workflow/summary.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { RunTokenUsage } from '../run-record' -import { type WorkflowPhaseGraph, workflowPhaseGraph } from './phase-graph' -import { summarizeWorkflowTrace, validateWorkflowTraceEnvelope } from './schema' -import { numberField, objectRecord, stringField, tokenUsageField } from './trace-event-fields' -import type { WorkflowTraceEnvelope, WorkflowTraceEvent, WorkflowTraceSummary } from './types' - -export interface WorkflowDelegateTraceSummary { - index: number | null - label: string | null - phase: string | null - costUsd: number | null - tokenUsage: RunTokenUsage | null - trace: unknown -} - -export interface WorkflowDelegateFailureSummary { - index: number | null - label: string | null - phase: string | null - durationMs: number | null - message: string | null - code: string | null -} - -export interface WorkflowCheckpointTraceSummary extends WorkflowDelegateTraceSummary { - output: unknown -} - -export interface WorkflowExecutionSummary extends WorkflowTraceSummary { - source?: string - eventKinds: Record - phases: string[] - phaseGraph: WorkflowPhaseGraph - agentRuns: WorkflowDelegateTraceSummary[] - loopRuns: WorkflowDelegateTraceSummary[] - verifierOutputs: WorkflowCheckpointTraceSummary[] - analystOutputs: WorkflowCheckpointTraceSummary[] - reviewerOutputs: WorkflowCheckpointTraceSummary[] - agentFailureDetails: WorkflowDelegateFailureSummary[] - loopFailureDetails: WorkflowDelegateFailureSummary[] - verifierFailureDetails: WorkflowDelegateFailureSummary[] - analystFailureDetails: WorkflowDelegateFailureSummary[] - reviewerFailureDetails: WorkflowDelegateFailureSummary[] -} - -export interface SummarizeWorkflowExecutionOptions { - source?: string -} - -export function summarizeWorkflowExecution( - input: WorkflowTraceEnvelope | unknown, - options: SummarizeWorkflowExecutionOptions = {}, -): WorkflowExecutionSummary { - const envelope = validateWorkflowTraceEnvelope(input) - const base = summarizeWorkflowTrace(envelope) - return { - ...base, - ...(options.source !== undefined ? { source: options.source } : {}), - eventKinds: workflowEventKinds(envelope.events), - phases: workflowPhaseTitles(envelope.events), - phaseGraph: workflowPhaseGraph(envelope.events), - agentRuns: workflowDelegateTraceSummaries(envelope.events, 'workflow.agent.ended'), - loopRuns: workflowDelegateTraceSummaries(envelope.events, 'workflow.loop.ended'), - verifierOutputs: workflowCheckpointTraceSummaries(envelope.events, 'workflow.verifier.ended'), - analystOutputs: workflowCheckpointTraceSummaries(envelope.events, 'workflow.analyst.ended'), - reviewerOutputs: workflowCheckpointTraceSummaries(envelope.events, 'workflow.reviewer.ended'), - agentFailureDetails: workflowDelegateFailureSummaries(envelope.events, 'workflow.agent.failed'), - loopFailureDetails: workflowDelegateFailureSummaries(envelope.events, 'workflow.loop.failed'), - verifierFailureDetails: workflowDelegateFailureSummaries( - envelope.events, - 'workflow.verifier.failed', - ), - analystFailureDetails: workflowDelegateFailureSummaries( - envelope.events, - 'workflow.analyst.failed', - ), - reviewerFailureDetails: workflowDelegateFailureSummaries( - envelope.events, - 'workflow.reviewer.failed', - ), - } -} - -function workflowEventKinds(events: readonly WorkflowTraceEvent[]): Record { - return events.reduce>((acc, event) => { - acc[event.kind] = (acc[event.kind] ?? 0) + 1 - return acc - }, {}) -} - -function workflowPhaseTitles(events: readonly WorkflowTraceEvent[]): string[] { - const titles: string[] = [] - const seen = new Set() - for (const event of events) { - const title = - event.kind === 'workflow.phase' - ? stringField(event.payload, 'title') - : stringField(event.payload, 'phase') - if (!title || seen.has(title)) continue - seen.add(title) - titles.push(title) - } - return titles -} - -function workflowDelegateTraceSummaries( - events: readonly WorkflowTraceEvent[], - endedKind: WorkflowTraceEvent['kind'], -): WorkflowDelegateTraceSummary[] { - return events - .filter((event) => event.kind === endedKind) - .map((event) => ({ - index: numberField(event.payload, 'index'), - label: stringField(event.payload, 'label'), - phase: stringField(event.payload, 'phase'), - costUsd: numberField(event.payload, 'costUsd'), - tokenUsage: tokenUsageField(event.payload.tokenUsage), - trace: event.payload.trace ?? null, - })) -} - -function workflowCheckpointTraceSummaries( - events: readonly WorkflowTraceEvent[], - endedKind: WorkflowTraceEvent['kind'], -): WorkflowCheckpointTraceSummary[] { - return workflowDelegateTraceSummaries(events, endedKind).map((summary) => ({ - ...summary, - output: checkpointOutput(summary.trace), - })) -} - -function workflowDelegateFailureSummaries( - events: readonly WorkflowTraceEvent[], - failedKind: WorkflowTraceEvent['kind'], -): WorkflowDelegateFailureSummary[] { - return events - .filter((event) => event.kind === failedKind) - .map((event) => ({ - index: numberField(event.payload, 'index'), - label: stringField(event.payload, 'label'), - phase: stringField(event.payload, 'phase'), - durationMs: numberField(event.payload, 'durationMs'), - message: stringField(event.payload, 'message'), - code: stringField(event.payload, 'code'), - })) -} - -function checkpointOutput(trace: unknown): unknown { - const record = objectRecord(trace) - if (record && Object.hasOwn(record, 'checkpointOutput')) { - return record.checkpointOutput - } - if (record && Object.hasOwn(record, 'output')) return record.output - return trace -} diff --git a/src/workflow/trace-event-fields.ts b/src/workflow/trace-event-fields.ts deleted file mode 100644 index 779cd849..00000000 --- a/src/workflow/trace-event-fields.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { RunTokenUsage } from '../run-record' - -export function objectRecord(value: unknown): Record | null { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : null -} - -export function stringField(record: Record, key: string): string | null { - const value = record[key] - return typeof value === 'string' ? value : null -} - -export function numberField(record: Record, key: string): number | null { - const value = record[key] - return typeof value === 'number' && Number.isFinite(value) ? value : null -} - -export function tokenUsageField(value: unknown): RunTokenUsage | null { - const record = objectRecord(value) - if (!record) return null - const input = numberField(record, 'input') - const output = numberField(record, 'output') - if (input === null || output === null) return null - const cached = numberField(record, 'cached') - return { - input, - output, - ...(cached !== null ? { cached } : {}), - } -} diff --git a/src/workflow/trajectory.ts b/src/workflow/trajectory.ts deleted file mode 100644 index 68c0a8ab..00000000 --- a/src/workflow/trajectory.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { DatasetSplit } from '../dataset' -import type { FeedbackAttempt, FeedbackTrajectory } from '../feedback-trajectory' -import { summarizeWorkflowTrace, validateWorkflowTraceEnvelope } from './schema' -import type { WorkflowTraceEnvelope, WorkflowTraceEvent } from './types' - -export interface WorkflowTraceTrajectoryOptions { - projectId?: string - scenarioId?: string - task: string - split?: DatasetSplit - tags?: Record - score?: number - success?: boolean - metadata?: Record -} - -export function workflowTraceToFeedbackTrajectory( - input: WorkflowTraceEnvelope | unknown, - options: WorkflowTraceTrajectoryOptions, -): FeedbackTrajectory { - const envelope = validateWorkflowTraceEnvelope(input) - const summary = summarizeWorkflowTrace(envelope) - const createdAt = iso(envelope.events[0]?.timestamp) - const updatedAt = iso(envelope.events.at(-1)?.timestamp) - return { - id: envelope.runId, - projectId: options.projectId, - scenarioId: options.scenarioId, - task: { - intent: options.task, - context: { - topology: envelope.topology, - metadata: envelope.metadata, - }, - }, - attempts: workflowEventsToAttempts(envelope.events), - labels: [], - outcome: { - success: options.success ?? !summary.failed, - score: options.score, - costUsd: summary.costUsd, - observedAt: updatedAt, - detail: summary.failureMessage, - metrics: { - workflow_events: summary.eventCount, - workflow_phases: summary.phaseCount, - workflow_branches: summary.branchCount, - workflow_branch_failures: summary.failedBranchCount, - workflow_agent_calls: summary.agentCalls, - workflow_loop_calls: summary.loopCalls, - workflow_verifier_calls: summary.verifierCalls, - workflow_analyst_calls: summary.analystCalls, - workflow_reviewer_calls: summary.reviewerCalls, - workflow_agent_failures: summary.agentFailures, - workflow_loop_failures: summary.loopFailures, - workflow_verifier_failures: summary.verifierFailures, - workflow_analyst_failures: summary.analystFailures, - workflow_reviewer_failures: summary.reviewerFailures, - workflow_tokens_input: summary.tokenUsage.input, - workflow_tokens_output: summary.tokenUsage.output, - }, - metadata: { - durationMs: summary.durationMs, - ...(options.metadata ?? {}), - }, - }, - split: options.split, - tags: options.tags, - createdAt, - updatedAt, - metadata: { - traceVersion: envelope.traceVersion, - artifacts: envelope.artifacts, - }, - } -} - -function workflowEventsToAttempts(events: readonly WorkflowTraceEvent[]): FeedbackAttempt[] { - const attempts: FeedbackAttempt[] = [] - for (const event of events) { - const artifactType = artifactTypeForWorkflowEvent(event.kind) - if (!artifactType) continue - const stepIndex = attempts.length - attempts.push({ - id: `${event.runId}:${stepIndex}`, - stepIndex, - artifactType, - artifact: event.payload.trace ?? event.payload, - createdAt: iso(event.timestamp), - metadata: { - eventKind: event.kind, - label: event.payload.label, - phase: event.payload.phase, - costUsd: event.payload.costUsd, - tokenUsage: event.payload.tokenUsage, - ...(event.kind.endsWith('.failed') - ? { failed: true, message: event.payload.message, code: event.payload.code } - : {}), - }, - }) - } - return attempts -} - -function artifactTypeForWorkflowEvent( - kind: WorkflowTraceEvent['kind'], -): FeedbackAttempt['artifactType'] | null { - switch (kind) { - case 'workflow.agent.ended': - case 'workflow.agent.failed': - return 'action' - case 'workflow.analyst.ended': - case 'workflow.analyst.failed': - return 'data' - case 'workflow.loop.ended': - case 'workflow.verifier.ended': - case 'workflow.reviewer.ended': - case 'workflow.loop.failed': - case 'workflow.verifier.failed': - case 'workflow.reviewer.failed': - return 'decision' - default: - return null - } -} - -function iso(timestamp: number | undefined): string { - return new Date(Number.isFinite(timestamp) ? timestamp! : 0).toISOString() -} diff --git a/src/workflow/types.ts b/src/workflow/types.ts deleted file mode 100644 index 037a38c1..00000000 --- a/src/workflow/types.ts +++ /dev/null @@ -1,103 +0,0 @@ -import type { WorkflowTopology } from '../harness-optimizer' -import type { RunSplitTag, RunTokenUsage } from '../run-record' - -export type { WorkflowTopology } - -export type WorkflowTraceVersion = 'workflow-trace-v1' - -export type WorkflowTraceEventKind = - | 'workflow.started' - | 'workflow.phase' - | 'workflow.log' - | 'workflow.parallel.started' - | 'workflow.parallel.ended' - | 'workflow.pipeline.started' - | 'workflow.pipeline.ended' - | 'workflow.branch.started' - | 'workflow.branch.ended' - | 'workflow.branch.failed' - | 'workflow.agent.started' - | 'workflow.agent.ended' - | 'workflow.agent.failed' - | 'workflow.loop.started' - | 'workflow.loop.ended' - | 'workflow.loop.failed' - | 'workflow.verifier.started' - | 'workflow.verifier.ended' - | 'workflow.verifier.failed' - | 'workflow.analyst.started' - | 'workflow.analyst.ended' - | 'workflow.analyst.failed' - | 'workflow.reviewer.started' - | 'workflow.reviewer.ended' - | 'workflow.reviewer.failed' - | 'workflow.failed' - | 'workflow.ended' - -export interface WorkflowTraceEvent { - kind: WorkflowTraceEventKind - runId: string - timestamp: number - payload: Record -} - -export interface WorkflowTraceArtifact { - kind: string - uri: string - contentType?: string - sha256?: string - metadata?: Record -} - -export interface WorkflowTraceExportLinks { - traceArtifactUri?: string - exportBundleUri?: string - partnerReportUri?: string - intelligenceRunUri?: string -} - -export interface WorkflowTraceEnvelope { - traceVersion: WorkflowTraceVersion - runId: string - topology?: WorkflowTopology - events: WorkflowTraceEvent[] - artifacts?: WorkflowTraceArtifact[] - metadata?: Record -} - -export interface WorkflowTraceSummary { - runId: string - startedAt?: number - endedAt?: number - durationMs: number - costUsd: number - tokenUsage: RunTokenUsage - phaseCount: number - branchCount: number - failedBranchCount: number - agentCalls: number - loopCalls: number - verifierCalls: number - analystCalls: number - reviewerCalls: number - agentFailures: number - loopFailures: number - verifierFailures: number - analystFailures: number - reviewerFailures: number - eventCount: number - failed: boolean - failureMessage?: string -} - -export interface WorkflowTraceProjectionMetadata { - experimentId: string - candidateId: string - seed: number - model: string - promptHash: string - configHash: string - commitSha: string - splitTag: RunSplitTag - scenarioId?: string -} diff --git a/tests/diagnose.test.ts b/tests/diagnose.test.ts deleted file mode 100644 index 14bf9c66..00000000 --- a/tests/diagnose.test.ts +++ /dev/null @@ -1,490 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { CounterfactualMutation, CounterfactualRunner } from '../src/counterfactual' -import { - causalSweep, - DIAGNOSE_ANALYST_ID, - describeMutation, - prescribeRepair, - suggestInvariant, - toAnalystFindings, - toCorpusRecord, - type ValidatedRepair, -} from '../src/diagnose' -import type { RunRecord } from '../src/run-record' -import type { ToolSpan } from '../src/trace' -import { InMemoryTraceStore, TraceEmitter } from '../src/trace' - -function mulberry32(seed: number): () => number { - let s = seed | 0 - return () => { - s = (s + 0x6d2b79f5) | 0 - let t = s - t = Math.imul(t ^ (t >>> 15), t | 1) - t ^= t + Math.imul(t ^ (t >>> 7), t | 61) - return ((t ^ (t >>> 14)) >>> 0) / 4294967296 - } -} - -async function seedRun( - store: InMemoryTraceStore, - outputScore: number, - shape: Array<{ kind: 'llm' | 'tool'; name: string; model?: string; toolName?: string }>, -): Promise { - const e = new TraceEmitter(store) - await e.startRun({ scenarioId: 's' }) - for (const s of shape) { - if (s.kind === 'llm') { - const h = await e.span({ - kind: 'llm', - name: s.name, - model: s.model ?? 'm', - messages: [], - output: 'x', - }) - await h.end() - } else { - const h = await e.span({ - kind: 'tool', - name: s.name, - toolName: s.toolName ?? s.name, - args: {}, - }) - await h.end({ result: 'rate=WRONG' } as Partial) - } - } - await e.endRun({ pass: false, score: outputScore }) - return e.runId -} - -const SHAPE = [ - { kind: 'llm' as const, name: 'plan' }, - { kind: 'tool' as const, name: 'fetch-rates' }, - { kind: 'tool' as const, name: 'format' }, - { kind: 'llm' as const, name: 'answer' }, -] - -/** - * Deterministic fake of the execution seam (same pattern as the - * runCounterfactual tests in tier2.test.ts): knocking out the faulty - * fetch-rates step (index 1) flips the run to ~0.8; every other - * intervention reproduces the original ~0.2 plus seeded noise. - */ -function makeRunner(opts: { seed: number; scoreFor?: (m: CounterfactualMutation) => number }): { - runner: CounterfactualRunner - calls: CounterfactualMutation[] -} { - const rng = mulberry32(opts.seed) - const calls: CounterfactualMutation[] = [] - const runner: CounterfactualRunner = { - async executeFrom(ctx, emitter) { - calls.push(ctx.mutation) - // Symmetric two-draw noise so per-rep deltas straddle the mean. - const noise = (rng() - 0.5) * 0.02 + (rng() - 0.5) * 0.02 - const base = - opts.scoreFor?.(ctx.mutation) ?? - (ctx.mutation.kind === 'swap-tool-result' && ctx.mutation.at === 1 ? 0.8 : 0.2) - await emitter.endRun({ pass: base >= 0.5, score: base + noise }) - }, - } - return { runner, calls } -} - -describe('causalSweep', () => { - it('ranks the injected-fault step #1 with CI excluding zero; no-effect step CI includes zero', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner } = makeRunner({ seed: 42 }) - - const report = await causalSweep({ - store, - runId, - runner, - candidateSteps: [1, 2], - reps: 5, - budget: 100, - ciSeed: 7, - }) - - expect(report.steps).toHaveLength(2) - const [top, rest] = report.steps - expect(top!.stepRef.index).toBe(1) - expect(top!.stepRef.name).toBe('fetch-rates') - expect(top!.mutationKind).toBe('swap-tool-result') - expect(top!.meanEffect).toBeGreaterThan(0.5) - expect(top!.ciExcludesZero).toBe(true) - expect(top!.ci.lower).toBeGreaterThan(0) - expect(top!.deltas).toHaveLength(5) - - expect(rest!.stepRef.index).toBe(2) - expect(rest!.ciExcludesZero).toBe(false) - expect(rest!.ci.lower).toBeLessThanOrEqual(0) - expect(rest!.ci.upper).toBeGreaterThanOrEqual(0) - - expect(report.replaysUsed).toBe(10) - expect(report.uncovered).toHaveLength(0) - expect(report.originalScore).toBeCloseTo(0.2) - expect(report.byMutationKind[0]!.mutationKind).toBe('swap-tool-result') - expect(report.byMutationKind[0]!.n).toBe(10) - }) - - it('records counterfactual replays as meta runs parented to the original', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner } = makeRunner({ seed: 1 }) - const report = await causalSweep({ - store, - runId, - runner, - candidateSteps: [1], - reps: 2, - budget: 10, - }) - const cfRun = await store.getRun(report.steps[0]!.counterfactualRunIds[0]!) - expect(cfRun?.parentRunId).toBe(runId) - expect(cfRun?.layer).toBe('meta') - }) - - it('names uncovered steps under a tight budget instead of silently dropping them', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner } = makeRunner({ seed: 9 }) - - const report = await causalSweep({ - store, - runId, - runner, - candidateSteps: [1, 2], - reps: 4, - budget: 6, - }) - - expect(report.steps).toHaveLength(1) - expect(report.steps[0]!.stepRef.index).toBe(1) - expect(report.replaysUsed).toBe(4) - expect(report.uncovered).toHaveLength(1) - expect(report.uncovered[0]!.index).toBe(2) - expect(report.uncovered[0]!.name).toBe('format') - }) - - it('covers nothing when budget < reps — everything uncovered, zero replays', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner, calls } = makeRunner({ seed: 9 }) - const report = await causalSweep({ - store, - runId, - runner, - candidateSteps: [1, 2], - reps: 4, - budget: 3, - }) - expect(report.steps).toHaveLength(0) - expect(report.replaysUsed).toBe(0) - expect(calls).toHaveLength(0) - expect(report.uncovered.map((s) => s.index)).toEqual([1, 2]) - }) - - it('defaults candidate steps to llm + tool spans with the payload-free probe kinds', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner, calls } = makeRunner({ seed: 3 }) - await causalSweep({ store, runId, runner, reps: 2, budget: 100 }) - // 4 steps × 1 default mutation × 2 reps - expect(calls).toHaveLength(8) - const kinds = new Set(calls.map((c) => c.kind)) - expect(kinds).toEqual(new Set(['truncate-after', 'swap-tool-result'])) - }) - - it('rejects reps < 2 — a single intervention delta is noise, not measurement', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner } = makeRunner({ seed: 3 }) - await expect(causalSweep({ store, runId, runner, reps: 1, budget: 10 })).rejects.toThrow( - /reps must be an integer >= 2/, - ) - }) - - it('fails loud when the original run has no numeric score', async () => { - const store = new InMemoryTraceStore() - const e = new TraceEmitter(store) - await e.startRun({ scenarioId: 's' }) - await e.endRun({ pass: false }) - const { runner } = makeRunner({ seed: 3 }) - await expect( - causalSweep({ store, runId: e.runId, runner, reps: 2, budget: 10 }), - ).rejects.toThrow(/no numeric outcome\.score/) - }) - - it('fails loud when a replay omits the score instead of recording a bogus delta', async () => { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const runner: CounterfactualRunner = { - async executeFrom(_ctx, emitter) { - await emitter.endRun({ pass: true }) - }, - } - await expect( - causalSweep({ store, runId, runner, candidateSteps: [1], reps: 2, budget: 10 }), - ).rejects.toThrow(/runner must endRun with a numeric outcome\.score/) - }) -}) - -describe('prescribeRepair', () => { - async function diagnosedSetup() { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const { runner } = makeRunner({ seed: 42 }) - const report = await causalSweep({ - store, - runId, - runner, - candidateSteps: [1, 2], - reps: 5, - budget: 100, - ciSeed: 7, - }) - return { store, runId, report } - } - - const goodFix: CounterfactualMutation = { - kind: 'swap-tool-result', - at: 1, - newResult: { rate: 4.5 }, - } - const badFix: CounterfactualMutation = { kind: 'swap-tool-result', at: 1, newResult: 'garbage' } - - it('emits only flipping mutations; non-flippers land in rejected with reason', async () => { - const { store, runId, report } = await diagnosedSetup() - const { runner } = makeRunner({ - seed: 11, - scoreFor: (m) => - m.kind === 'swap-tool-result' && - JSON.stringify(m.newResult) === JSON.stringify(goodFix.newResult) - ? 0.9 - : 0.3, - }) - - const repair = await prescribeRepair({ - store, - runId, - runner, - blamed: report.steps.slice(0, 1), - proposeFix: async () => [badFix, goodFix], - flipThreshold: 0.5, - repsToValidate: 3, - }) - - expect(repair.repairs).toHaveLength(1) - const validated = repair.repairs[0]! - expect(validated.validated).toBe(true) - expect(validated.mutation).toEqual(goodFix) - expect(validated.stepRef.index).toBe(1) - expect(validated.meanScore).toBeGreaterThanOrEqual(0.5) - expect(validated.deltaScore).toBeCloseTo(validated.meanScore - 0.2, 10) - expect(validated.reps).toBe(3) - expect(validated.counterfactualRunIds).toHaveLength(3) - - expect(repair.rejected).toHaveLength(1) - expect(repair.rejected[0]!.reason).toBe('did-not-flip') - expect(repair.rejected[0]!.mutation).toEqual(badFix) - expect(repair.rejected[0]!.deltaScore).toBeCloseTo(0.1, 1) - expect(repair.replaysUsed).toBe(6) - }) - - it('a repair must flip on EVERY validation rep, not on average', async () => { - const { store, runId, report } = await diagnosedSetup() - // Scores alternate 0.9 / 0.4: mean 0.65 crosses the threshold but rep 2 does not. - let call = 0 - const runner: CounterfactualRunner = { - async executeFrom(_ctx, emitter) { - call++ - await emitter.endRun({ pass: true, score: call % 2 === 1 ? 0.9 : 0.4 }) - }, - } - const repair = await prescribeRepair({ - store, - runId, - runner, - blamed: report.steps.slice(0, 1), - proposeFix: async () => [goodFix], - repsToValidate: 3, - }) - expect(repair.repairs).toHaveLength(0) - expect(repair.rejected[0]!.reason).toBe('did-not-flip') - }) - - it('replay errors become typed rejections, never silent drops', async () => { - const { store, runId, report } = await diagnosedSetup() - const { runner } = makeRunner({ seed: 5 }) - const explosive: CounterfactualMutation = { - kind: 'custom', - at: 1, - describe: 'patch the parser', - apply: () => { - throw new Error('boom: parser patch unapplicable') - }, - } - const repair = await prescribeRepair({ - store, - runId, - runner, - blamed: report.steps.slice(0, 1), - proposeFix: async () => [explosive], - }) - expect(repair.repairs).toHaveLength(0) - expect(repair.rejected).toHaveLength(1) - expect(repair.rejected[0]!.reason).toBe('error') - expect(repair.rejected[0]!.error).toMatch(/boom/) - }) - - it('respects maxAttemptsPerStep', async () => { - const { store, runId, report } = await diagnosedSetup() - const { runner } = makeRunner({ seed: 5, scoreFor: () => 0.3 }) - const repair = await prescribeRepair({ - store, - runId, - runner, - blamed: report.steps.slice(0, 1), - proposeFix: async () => [badFix, goodFix], - maxAttemptsPerStep: 1, - }) - expect(repair.repairs).toHaveLength(0) - expect(repair.rejected).toHaveLength(1) - expect(repair.rejected[0]!.mutation).toEqual(badFix) - }) - - it('rejects a stale report whose stepRef does not match the run', async () => { - const { store, runId, report } = await diagnosedSetup() - const { runner } = makeRunner({ seed: 5 }) - const stale = { ...report.steps[0]!, stepRef: { ...report.steps[0]!.stepRef, spanId: 'nope' } } - await expect( - prescribeRepair({ - store, - runId, - runner, - blamed: [stale], - proposeFix: async () => [goodFix], - }), - ).rejects.toThrow(/does not match run/) - }) -}) - -describe('remediation adapters', () => { - async function fullChain() { - const store = new InMemoryTraceStore() - const runId = await seedRun(store, 0.2, SHAPE) - const sweep = makeRunner({ seed: 42 }) - const report = await causalSweep({ - store, - runId, - runner: sweep.runner, - candidateSteps: [1, 2], - reps: 5, - budget: 100, - ciSeed: 7, - }) - const fix: CounterfactualMutation = { - kind: 'swap-tool-result', - at: 1, - newResult: { rate: 4.5 }, - } - const validate = makeRunner({ seed: 11, scoreFor: () => 0.9 }) - const repairs = await prescribeRepair({ - store, - runId, - runner: validate.runner, - blamed: report.steps.slice(0, 1), - proposeFix: async () => [fix], - }) - return { report, repairs } - } - - it('toAnalystFindings emits schema-valid findings with effect-scaled severity', async () => { - const { report, repairs } = await fullChain() - const findings = toAnalystFindings(report, repairs) - expect(findings).toHaveLength(2) - - for (const f of findings) { - expect(f.schema_version).toBe('1.0.0') - expect(f.finding_id).toMatch(/^f_[0-9a-f]{20}$/) - expect(f.analyst_id).toBe(DIAGNOSE_ANALYST_ID) - expect(f.area).toBe('causal-attribution') - expect(f.evidence_refs.length).toBeGreaterThanOrEqual(2) - expect(f.derived_from_judge).toBeUndefined() - } - - const blamed = findings.find((f) => f.subject === report.steps[0]!.stepRef.spanId)! - expect(blamed.severity).toBe('critical') - expect(blamed.confidence).toBe(0.95) - expect(blamed.recommended_action).toBe(describeMutation(repairs.repairs[0]!.mutation)) - expect(blamed.validation_plan).toMatch(/replay-validated: 3\/3 reps scored >= 0\.5/) - expect(blamed.evidence_refs[0]!.uri).toBe(`span://${report.steps[0]!.stepRef.spanId}`) - expect(blamed.evidence_refs[1]!.excerpt).toContain('deltas=[') - - const noise = findings.find((f) => f.subject === report.steps[1]!.stepRef.spanId)! - expect(noise.severity).toBe('info') - expect(noise.confidence).toBe(0.3) - expect(noise.recommended_action).toBeUndefined() - }) - - it('toCorpusRecord pins the failure as a fresh, schema-valid corpus scenario', async () => { - const { repairs } = await fullChain() - const original: RunRecord = { - runId: 'run-original', - experimentId: 'exp-1', - candidateId: 'cand-1', - seed: 42, - model: 'test-model@2026-01-01', - promptHash: 'p'.repeat(8), - configHash: 'c'.repeat(8), - commitSha: 'deadbeef', - wallMs: 1200, - costUsd: 0.01, - tokenUsage: { input: 100, output: 50 }, - outcome: { searchScore: 0.2, raw: {} }, - splitTag: 'search', - } - const repair = repairs.repairs[0]! - const pinned = toCorpusRecord(original, repair, { prompt: 'fetch the current rates' }) - - expect(pinned.runId).toBe(`run-original#repair:${repair.stepRef.spanId}`) - expect(pinned.runId).not.toBe(original.runId) - expect(pinned.prompt).toBe('fetch the current rates') - expect(pinned.completion).toBe(describeMutation(repair.mutation)) - expect(pinned.outcome.raw.diagnose_blamed_step_index).toBe(1) - expect(pinned.outcome.raw.diagnose_repair_mean_score).toBeCloseTo(repair.meanScore, 10) - expect(pinned.outcome.raw.diagnose_repair_delta_score).toBeCloseTo(repair.deltaScore, 10) - // Original record untouched. - expect(original.outcome.raw.diagnose_blamed_step_index).toBeUndefined() - }) - - it('suggestInvariant derives never/without clauses per mutation kind', async () => { - const { repairs } = await fullChain() - const toolHint = suggestInvariant(repairs.repairs[0]!) - expect(toolHint.description).toContain('fetch-rates') - expect(toolHint.never).toContain("tool 'fetch-rates'") - expect(toolHint.without).toContain("tool 'fetch-rates'") - - const base = repairs.repairs[0]! - const truncate: ValidatedRepair = { - ...base, - mutation: { kind: 'truncate-after', at: 1 }, - } - const truncateHint = suggestInvariant(truncate) - expect(truncateHint.never).toContain('after') - expect(truncateHint.without).toBeUndefined() - - const inject: ValidatedRepair = { - ...base, - mutation: { kind: 'inject-system-message', at: 1, content: 'always validate rates' }, - } - const injectHint = suggestInvariant(inject) - expect(injectHint.without).toContain('always validate rates') - - const swapModel: ValidatedRepair = { - ...base, - mutation: { kind: 'swap-model', at: 1, newModel: 'better-model@2026-01-01' }, - } - expect(suggestInvariant(swapModel).never).toContain('better-model@2026-01-01') - }) -}) diff --git a/tests/governance.test.ts b/tests/governance.test.ts deleted file mode 100644 index 88b16b6e..00000000 --- a/tests/governance.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { Dataset } from '../src/dataset' -import { - classifyEuAiRisk, - euAiActReport, - type GovernanceContext, - nistAiRmfReport, - renderMarkdown, - soc2Report, -} from '../src/governance' -import { InMemoryOutcomeStore } from '../src/meta-eval' -import { redTeamReport } from '../src/red-team' -import { InMemoryTraceStore, TraceEmitter } from '../src/trace' - -async function makeContext(partial: Partial = {}): Promise { - const traceStore = partial.traceStore ?? new InMemoryTraceStore() - const dataset = new Dataset({ - name: 'default', - provenance: { version: '1.0.0', createdAt: '2026-04-20T00:00:00Z' }, - scenarios: [{ id: 's1', payload: {}, difficulty: 'easy' }], - }) - const manifest = await dataset.manifest() - return { - organization: 'Tangle Technologies', - systemName: 'agent-builder', - periodStart: '2026-04-01T00:00:00Z', - periodEnd: '2026-04-30T23:59:59Z', - datasets: [manifest], - traceStore, - owner: { role: 'founder', name: 'Drew Stone', email: 'drew@webb.tools' }, - ...partial, - } -} - -describe('NIST AI RMF', () => { - it('flags missing red-team + missing outcomes + missing calibration', async () => { - const ctx = await makeContext() - const report = await nistAiRmfReport(ctx) - const controls = report.findings.map((f) => f.control) - expect(controls).toContain('NIST-AI-RMF:MEASURE-2.6') - expect(controls).toContain('NIST-AI-RMF:MEASURE-2.11') - expect(controls).toContain('NIST-AI-RMF:MANAGE-1.1') - expect(report.summary.overall).not.toBe('compliant') - }) - - it('compliant when red-team, calibration, outcomes all present — regression: we must reward full-stack posture', async () => { - const trace = new InMemoryTraceStore() - const outcome = new InMemoryOutcomeStore() - // Add one passing run during the period - const e = new TraceEmitter(trace, { now: () => Date.parse('2026-04-15T00:00:00Z') }) - await e.startRun({ scenarioId: 's1' }) - await e.endRun({ pass: true, score: 0.9 }) - await outcome.append({ - runId: e.runId, - capturedAt: Date.parse('2026-04-16T00:00:00Z'), - metrics: { retention_7d: 0.8 }, - }) - const rt = redTeamReport([ - { scenarioId: 'rt1', category: 'prompt_injection_direct', passed: true, reason: '' }, - { scenarioId: 'rt2', category: 'pii_leak', passed: true, reason: '' }, - ]) - const ctx = await makeContext({ - traceStore: trace, - outcomeStore: outcome, - redTeam: rt, - judgeCalibration: [{ n: 50, pearson: 0.85, kappa: 0.7, mae: 0.3, worstItems: [] }], - }) - const report = await nistAiRmfReport(ctx) - expect(report.summary.overall).toBe('compliant') - }) -}) - -describe('SOC2', () => { - it('flags high failure rate and absent codeSha', async () => { - const trace = new InMemoryTraceStore() - for (let i = 0; i < 10; i++) { - const e = new TraceEmitter(trace, { now: () => Date.parse('2026-04-15T00:00:00Z') + i }) - await e.startRun({ scenarioId: 's' }) - await e.endRun({ pass: i > 6 }) // 30% pass → 70% fail - } - const ctx = await makeContext({ traceStore: trace }) - const report = await soc2Report(ctx) - const controls = report.findings.map((f) => f.control) - expect(controls).toContain('SOC2:CC7.1') - expect(controls).toContain('SOC2:CC7.4') - }) -}) - -describe('EU AI Act', () => { - it('classifies biometric public as unacceptable', () => { - expect(classifyEuAiRisk({ biometricPublic: true })).toBe('unacceptable') - }) - - it('classifies Annex III as high', () => { - expect(classifyEuAiRisk({ annexIII: true })).toBe('high') - }) - - it('classifies chatbot as limited', () => { - expect(classifyEuAiRisk({ chatbot: true })).toBe('limited') - }) - - it('defaults to minimal', () => { - expect(classifyEuAiRisk({})).toBe('minimal') - }) - - it('high-risk report flags Article 9/10/11/14/15 gaps when evidence absent — regression: silent gaps in high-risk deployments are unshippable in the EU', async () => { - const ctx = await makeContext({ datasets: [] }) - const report = await euAiActReport(ctx, { annexIII: true }) - const controls = report.findings.map((f) => f.control) - expect(controls).toContain('EU-AI-ACT:Article-9') - expect(controls).toContain('EU-AI-ACT:Article-10') - expect(controls).toContain('EU-AI-ACT:Article-11') - }) - - it('unacceptable risk produces critical Article-5 finding', async () => { - const ctx = await makeContext() - const report = await euAiActReport(ctx, { socialScoring: true }) - expect( - report.findings.some((f) => f.control === 'EU-AI-ACT:Article-5' && f.severity === 'critical'), - ).toBe(true) - expect(report.summary.overall).toBe('non-compliant') - }) -}) - -describe('renderMarkdown', () => { - it('produces a human-readable header + summary + findings list', async () => { - const ctx = await makeContext() - const report = await nistAiRmfReport(ctx) - const md = renderMarkdown(report) - expect(md).toMatch(/# NIST-AI-RMF report/) - expect(md).toMatch(/Tangle Technologies/) - expect(md).toMatch(/## Summary/) - }) -}) diff --git a/tests/perf.test.ts b/tests/perf.test.ts deleted file mode 100644 index 0c04c434..00000000 --- a/tests/perf.test.ts +++ /dev/null @@ -1,272 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - assertRecordIntegrity, - checkRecordIntegrity, - expandMatrix, - gatePerf, - type JourneySpec, - type PerfBaseline, - scenarioKey, - summarizeRecords, -} from '../src/perf' - -function journey(overrides: Partial): JourneySpec { - return { - id: 'provision.cold', - description: 'cold-start provision', - requiresLLM: false, - requiredFields: ['total_ms'], - ...overrides, - } -} - -describe('expandMatrix', () => { - const journeys = [ - journey({ id: 'provision.cold' }), - journey({ id: 'chat.ttft', requiresLLM: true }), - ] - const axes = { - region: ['us', 'eu'], - driver: ['docker', 'firecracker'], - } - - it('expands the full cartesian product (catches dropped journeys or axis values)', () => { - const scenarios = expandMatrix(journeys, axes) - expect(scenarios).toHaveLength(2 * 2 * 2) - const keys = scenarios.map((s) => s.key) - expect(new Set(keys).size).toBe(8) - expect(keys).toContain('chat.ttft|driver=firecracker|region=eu') - }) - - it('sorts dimensions in the key regardless of axes object key order (catches insertion-order keys that break baseline lookup)', () => { - const [a] = expandMatrix([journeys[0]], { region: ['us'], driver: ['docker'] }) - const [b] = expandMatrix([journeys[0]], { driver: ['docker'], region: ['us'] }) - expect(a.key).toBe('provision.cold|driver=docker|region=us') - expect(b.key).toBe(a.key) - expect(scenarioKey('provision.cold', { region: 'us', driver: 'docker' })).toBe(a.key) - }) - - it('filter drops invalid combos (catches a filter that is ignored or inverted)', () => { - const scenarios = expandMatrix( - journeys, - axes, - (journeyId, combo) => !(journeyId === 'chat.ttft' && combo.driver === 'firecracker'), - ) - expect(scenarios).toHaveLength(6) - expect( - scenarios.some((s) => s.journey.id === 'chat.ttft' && s.axes.driver === 'firecracker'), - ).toBe(false) - }) - - it('produces one keyed scenario per journey when axes are empty (catches zero-combo expansion)', () => { - const scenarios = expandMatrix(journeys, {}) - expect(scenarios.map((s) => s.key)).toEqual(['provision.cold', 'chat.ttft']) - }) -}) - -describe('checkRecordIntegrity', () => { - const spec = journey({ - id: 'chat.stream', - requiredFields: ['total_ms', 'ttft_ms'], - minimums: [{ field: 'event_count', min: 1 }], - phaseFields: ['connect_ms'], - }) - const resolve = () => spec - - it('flags a passing record with a null required field (catches integrity checks that trust pass=true)', () => { - const result = checkRecordIntegrity( - [{ pass: true, total_ms: 1200, ttft_ms: null, event_count: 4, connect_ms: 10 }], - resolve, - ) - expect(result.succeeded).toBe(false) - expect(result.violations).toHaveLength(1) - expect(result.violations[0]).toMatchObject({ - recordIndex: 0, - journeyId: 'chat.stream', - field: 'ttft_ms', - reason: 'null-required-field', - }) - }) - - it('does NOT flag a failing record with nulls (catches over-eager checks that punish honest failures)', () => { - const result = checkRecordIntegrity( - [{ pass: false, total_ms: null, ttft_ms: null, event_count: null, connect_ms: null }], - resolve, - ) - expect(result.succeeded).toBe(true) - expect(result.violations).toEqual([]) - }) - - it('flags a passing streaming record with event_count 0 under min 1 (catches minimums treated as null checks)', () => { - const result = checkRecordIntegrity( - [{ pass: true, total_ms: 1200, ttft_ms: 80, event_count: 0, connect_ms: 10 }], - resolve, - ) - expect(result.succeeded).toBe(false) - expect(result.violations[0]).toMatchObject({ field: 'event_count', reason: 'below-minimum' }) - }) - - it('flags a null phase field on a passing record (catches phaseFields being ignored)', () => { - const result = checkRecordIntegrity( - [{ pass: true, total_ms: 1200, ttft_ms: 80, event_count: 4, connect_ms: null }], - resolve, - ) - expect(result.succeeded).toBe(false) - expect(result.violations[0]).toMatchObject({ - field: 'connect_ms', - reason: 'null-required-field', - }) - expect(result.violations[0].detail).toContain('phase field') - }) - - it('skips records whose journey resolves to null (catches resolveJourney null being treated as a violation)', () => { - const result = checkRecordIntegrity([{ pass: true, total_ms: null }], () => null) - expect(result.succeeded).toBe(true) - }) - - it('reports the violating record index across a batch (catches index drift when failing records are skipped)', () => { - const result = checkRecordIntegrity( - [ - { pass: false, total_ms: null, ttft_ms: null, event_count: null, connect_ms: null }, - { pass: true, total_ms: 900, ttft_ms: null, event_count: 2, connect_ms: 5 }, - ], - resolve, - ) - expect(result.violations[0].recordIndex).toBe(1) - }) - - it('assertRecordIntegrity throws listing every violation (catches an assert that swallows the detail)', () => { - expect(() => - assertRecordIntegrity( - [{ pass: true, total_ms: null, ttft_ms: 80, event_count: 0, connect_ms: 1 }], - resolve, - ), - ).toThrowError(/2 violation\(s\)[\s\S]*total_ms[\s\S]*event_count/) - }) - - it('assertRecordIntegrity returns silently on clean records (catches an assert that always throws)', () => { - expect(() => - assertRecordIntegrity( - [{ pass: true, total_ms: 900, ttft_ms: 80, event_count: 3, connect_ms: 5 }], - resolve, - ), - ).not.toThrow() - }) -}) - -describe('summarizeRecords', () => { - const keyOf = (r: Record) => (typeof r.key === 'string' ? r.key : null) - - it('computes nearest-rank p50/p90 on a known array (catches interpolated or off-by-one percentiles)', () => { - const records = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100].map((total_ms) => ({ - key: 's', - total_ms, - })) - const summary = summarizeRecords(records, keyOf, ['total_ms']) - // Nearest-rank: rank = ceil(p/100 * 10) → p50 = 5th value, p90 = 9th value. - expect(summary.scenarios.s.total_ms).toEqual({ p50: 50, p90: 90, n: 10 }) - }) - - it('rounds the rank up on fractional ranks (catches floor-based nearest-rank on odd n)', () => { - // n = 7: p50 rank = ceil(3.5) = 4th value, p90 rank = ceil(6.3) = 7th value. - const records = [70, 10, 30, 50, 20, 60, 40].map((total_ms) => ({ key: 's', total_ms })) - const summary = summarizeRecords(records, keyOf, ['total_ms']) - expect(summary.scenarios.s.total_ms).toEqual({ p50: 40, p90: 70, n: 7 }) - }) - - it('excludes null and non-numeric metric values from n (catches nulls coerced to fake zeros)', () => { - const records = [ - { key: 's', total_ms: 100 }, - { key: 's', total_ms: null }, - { key: 's', total_ms: 'broken' }, - { key: 's', total_ms: 300 }, - ] - const summary = summarizeRecords(records, keyOf, ['total_ms']) - expect(summary.scenarios.s.total_ms.n).toBe(2) - expect(summary.scenarios.s.total_ms.p50).toBe(100) - }) - - it('omits a field with zero real samples instead of emitting a fake stat (catches all-null fields summarized as 0)', () => { - const summary = summarizeRecords([{ key: 's', total_ms: null }], keyOf, ['total_ms']) - expect(summary.scenarios.s).toEqual({}) - }) - - it('skips records keyOf maps to null (catches unkeyed records leaking into a scenario)', () => { - const summary = summarizeRecords([{ total_ms: 100 }, { key: 's', total_ms: 200 }], keyOf, [ - 'total_ms', - ]) - expect(summary.scenarios.s.total_ms.n).toBe(1) - }) -}) - -describe('gatePerf', () => { - function baselineOf(p50: number, p90: number, n = 10): PerfBaseline { - return { version: 1, scenarios: { s: { total_ms: { p50, p90, n } } } } - } - - it('trips a regression beyond tolerance (catches a gate that always passes)', () => { - const result = gatePerf(baselineOf(120, 200), baselineOf(100, 180), { tolerancePct: 10 }) - expect(result.succeeded).toBe(false) - expect(result.regressions).toHaveLength(1) - expect(result.regressions[0].overBy.p50Pct).toBeCloseTo(20) - }) - - it('does not trip within tolerance (catches a zero-tolerance gate that blocks noise)', () => { - const result = gatePerf(baselineOf(105, 185), baselineOf(100, 180), { tolerancePct: 10 }) - expect(result.succeeded).toBe(true) - expect(result.regressions).toEqual([]) - }) - - it('trips on p90 alone even when p50 holds (catches gates that only watch the median)', () => { - const result = gatePerf(baselineOf(100, 250), baselineOf(100, 180), { tolerancePct: 10 }) - expect(result.succeeded).toBe(false) - expect(result.regressions[0].overBy.p90Pct).toBeCloseTo((250 - 180) / 1.8) - }) - - it('records strict improvements with negative overBy (catches improvements misfiled as regressions)', () => { - const result = gatePerf(baselineOf(80, 150), baselineOf(100, 180)) - expect(result.succeeded).toBe(true) - expect(result.improvements).toHaveLength(1) - expect(result.improvements[0].overBy.p50Pct).toBeLessThan(0) - expect(result.improvements[0].overBy.p90Pct).toBeLessThan(0) - }) - - it('detects scenarios missing from current and new in current (catches silently dropped coverage)', () => { - const current: PerfBaseline = { - version: 1, - scenarios: { added: { total_ms: { p50: 1, p90: 2, n: 5 } } }, - } - const baseline: PerfBaseline = { - version: 1, - scenarios: { s: { total_ms: { p50: 100, p90: 180, n: 10 } } }, - } - const result = gatePerf(current, baseline) - expect(result.missingScenarios).toEqual(['s']) - expect(result.newScenarios).toEqual(['added']) - }) - - it('never gates a scenario with n < minSamples — reports it missing instead (catches one noisy sample tripping the gate)', () => { - const result = gatePerf(baselineOf(500, 900, 2), baselineOf(100, 180, 10), { - tolerancePct: 10, - minSamples: 3, - }) - expect(result.regressions).toEqual([]) - expect(result.missingScenarios).toEqual(['s']) - expect(result.succeeded).toBe(true) - }) - - it('gates a scenario with exactly n = minSamples (catches an off-by-one that drops the boundary)', () => { - const result = gatePerf(baselineOf(500, 900, 3), baselineOf(100, 180, 10), { - tolerancePct: 10, - minSamples: 3, - }) - expect(result.regressions).toHaveLength(1) - expect(result.missingScenarios).toEqual([]) - }) - - it('a zero baseline with a grown current trips instead of dividing to NaN (catches 0-baseline division holes)', () => { - const result = gatePerf(baselineOf(5, 5), baselineOf(0, 0)) - expect(result.succeeded).toBe(false) - expect(result.regressions[0].overBy.p50Pct).toBe(Number.POSITIVE_INFINITY) - }) -}) diff --git a/tests/telemetry.test.ts b/tests/telemetry.test.ts deleted file mode 100644 index 2f47459a..00000000 --- a/tests/telemetry.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { beforeEach, describe, expect, it } from 'vitest' -import type { TelemetryEnvelope, TelemetrySink, TelemetrySource } from '../src/telemetry/index' -import { - FanoutTelemetrySink, - HttpTelemetrySink, - InMemoryTelemetrySink, - NullTelemetrySink, - sanitiseArgv, - TELEMETRY_SCHEMA_VERSION, - TelemetryClient, -} from '../src/telemetry/index' - -const defaultSource: TelemetrySource = { - repo: 'agent-eval-tests', - cwd: '/test', - cliVersion: 'test', - invocation: 'unit-test', -} - -describe('TelemetryClient', () => { - let captured: InMemoryTelemetrySink - - beforeEach(() => { - captured = new InMemoryTelemetrySink() - }) - - it('emits a fully-shaped envelope', () => { - const client = new TelemetryClient(captured, defaultSource) - client.emit({ - kind: 'design-audit-page', - runId: 'r1', - ok: true, - durationMs: 123, - data: { url: 'https://x' }, - metrics: { score: 7.5 }, - }) - expect(captured.envelopes).toHaveLength(1) - const env = captured.envelopes[0]! - expect(env.schemaVersion).toBe(TELEMETRY_SCHEMA_VERSION) - expect(env.runId).toBe('r1') - expect(env.kind).toBe('design-audit-page') - expect(env.metrics.score).toBe(7.5) - expect(env.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/) - expect(env.source.repo).toBe('agent-eval-tests') - }) - - it('honours per-emit source override', () => { - const client = new TelemetryClient(captured, defaultSource) - client.emit({ - kind: 'agent-run', - runId: 'r1', - ok: true, - durationMs: 1, - source: { ...defaultSource, repo: 'overridden', tenantId: 'workspace-xyz' }, - }) - expect(captured.envelopes[0]!.source.repo).toBe('overridden') - expect(captured.envelopes[0]!.source.tenantId).toBe('workspace-xyz') - }) - - it('never throws when sink throws', () => { - const blowing: TelemetrySink = { - emit() { - throw new Error('disk full') - }, - } - const client = new TelemetryClient(blowing, defaultSource) - expect(() => - client.emit({ kind: 'agent-run', runId: 'r', ok: true, durationMs: 0 }), - ).not.toThrow() - }) -}) - -describe('FanoutTelemetrySink', () => { - it('continues fanout when one sink throws', () => { - const good = new InMemoryTelemetrySink() - const bad: TelemetrySink = { - emit() { - throw new Error('boom') - }, - } - const fan = new FanoutTelemetrySink([bad, good]) - const client = new TelemetryClient(fan, defaultSource) - client.emit({ kind: 'agent-run', runId: 'r', ok: true, durationMs: 0 }) - expect(good.envelopes).toHaveLength(1) - }) - - it('forwards close() to all child sinks', async () => { - let closed = 0 - const a: TelemetrySink = { - emit() {}, - close() { - closed++ - }, - } - const b: TelemetrySink = { - emit() {}, - close: async () => { - closed++ - }, - } - await new FanoutTelemetrySink([a, b]).close?.() - expect(closed).toBe(2) - }) -}) - -describe('NullTelemetrySink', () => { - it('drops envelopes silently', () => { - const sink = new NullTelemetrySink() - const client = new TelemetryClient(sink, defaultSource) - expect(() => - client.emit({ kind: 'agent-run', runId: 'r', ok: true, durationMs: 0 }), - ).not.toThrow() - }) -}) - -describe('HttpTelemetrySink', () => { - it('POSTs JSON with bearer when set', async () => { - const calls: Array<{ url: string; init?: RequestInit }> = [] - const origFetch = globalThis.fetch - globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => { - calls.push({ url: String(url), init }) - return new Response('ok', { status: 200 }) - }) as typeof fetch - try { - const sink = new HttpTelemetrySink('https://collector.test/v1', 'tok') - const client = new TelemetryClient(sink, defaultSource) - client.emit({ kind: 'agent-run', runId: 'r', ok: true, durationMs: 1 }) - await sink.close() - } finally { - globalThis.fetch = origFetch - } - expect(calls.length).toBe(1) - expect(calls[0]!.url).toBe('https://collector.test/v1') - expect((calls[0]!.init?.headers as Record).authorization).toBe('Bearer tok') - const env = JSON.parse(String(calls[0]!.init?.body)) as TelemetryEnvelope - expect(env.schemaVersion).toBe(TELEMETRY_SCHEMA_VERSION) - }) - - it('swallows fetch errors (best-effort)', async () => { - const origFetch = globalThis.fetch - globalThis.fetch = (async () => { - throw new Error('unreachable') - }) as typeof fetch - try { - const sink = new HttpTelemetrySink('https://collector.test/v1') - const client = new TelemetryClient(sink, defaultSource) - expect(() => - client.emit({ kind: 'agent-run', runId: 'r', ok: true, durationMs: 0 }), - ).not.toThrow() - await sink.close() - } finally { - globalThis.fetch = origFetch - } - }) -}) - -describe('sanitiseArgv', () => { - it('redacts secret-bearing flags', () => { - expect(sanitiseArgv(['run', '--api-key', 'sk', '--url', 'http://x', '--token=abc'])).toEqual([ - 'run', - '--api-key', - '', - '--url', - 'http://x', - '--token=', - ]) - }) - - it('passes through clean argv unchanged', () => { - expect(sanitiseArgv(['run', '--goal', 'hello', '--max-turns', '10'])).toEqual([ - 'run', - '--goal', - 'hello', - '--max-turns', - '10', - ]) - }) -}) diff --git a/tests/workflow-trace.test.ts b/tests/workflow-trace.test.ts deleted file mode 100644 index 60bf64b5..00000000 --- a/tests/workflow-trace.test.ts +++ /dev/null @@ -1,1049 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { makeFinding } from '../src/analyst/types' -import type { VerificationReport } from '../src/multi-layer-verifier' -import type { FailureClusterReport } from '../src/pipelines' -import type { RunRecord } from '../src/run-record' -import { - buildWorkflowAnalystFeedbackPack, - buildWorkflowPartnerReport, - buildWorkflowTraceIntelligenceEnvelope, - decideWorkflowDriverPromotion, - renderWorkflowFeedbackPack, - renderWorkflowPartnerReport, - sanitizeWorkflowTraceEnvelope, - summarizeWorkflowExecution, - summarizeWorkflowTrace, - validateWorkflowPartnerReport, - validateWorkflowTraceEnvelope, - validateWorkflowTraceIntelligenceEnvelope, - type WorkflowTraceEnvelope, - workflowEventsToTraceEnvelope, - workflowRuntimeResultToTraceEnvelope, - workflowTraceToFeedbackTrajectory, - workflowTraceToRunRecord, -} from '../src/workflow' - -const envelope: WorkflowTraceEnvelope = { - traceVersion: 'workflow-trace-v1', - runId: 'wf-1', - topology: { - id: 'driver-authored', - interventions: ['plan', 'verify'], - maxParallelBranches: 2, - }, - events: [ - { - kind: 'workflow.started', - runId: 'wf-1', - timestamp: 1000, - payload: { - meta: { name: 'driver-authored', description: 'Runtime-generated workflow' }, - depth: 0, - caps: { maxFanout: 2, maxDepth: 1 }, - }, - }, - { kind: 'workflow.phase', runId: 'wf-1', timestamp: 1010, payload: { title: 'Plan' } }, - { - kind: 'workflow.branch.started', - runId: 'wf-1', - timestamp: 1020, - payload: { operation: 'parallel', branchIndex: 0, phase: 'Plan' }, - }, - { - kind: 'workflow.agent.ended', - runId: 'wf-1', - timestamp: 1200, - payload: { - index: 0, - label: 'planner', - phase: 'Plan', - durationMs: 180, - costUsd: 0.01, - tokenUsage: { input: 10, output: 20 }, - trace: { text: 'plan' }, - }, - }, - { - kind: 'workflow.branch.ended', - runId: 'wf-1', - timestamp: 1210, - payload: { operation: 'parallel', branchIndex: 0, durationMs: 190, phase: 'Plan' }, - }, - { - kind: 'workflow.loop.ended', - runId: 'wf-1', - timestamp: 1500, - payload: { - index: 0, - label: 'implement', - durationMs: 250, - costUsd: 0.02, - tokenUsage: { input: 30, output: 40 }, - trace: { winner: 1 }, - }, - }, - { - kind: 'workflow.verifier.ended', - runId: 'wf-1', - timestamp: 1550, - payload: { - index: 0, - label: 'acceptance', - durationMs: 20, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - trace: { checkpointOutput: { allPass: true } }, - }, - }, - { - kind: 'workflow.analyst.ended', - runId: 'wf-1', - timestamp: 1560, - payload: { - index: 0, - label: 'trace-analyst', - durationMs: 10, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - trace: { output: { findings: [] } }, - }, - }, - { - kind: 'workflow.reviewer.ended', - runId: 'wf-1', - timestamp: 1570, - payload: { - index: 0, - label: 'next-shot', - durationMs: 10, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - trace: { shouldContinue: false }, - }, - }, - { - kind: 'workflow.ended', - runId: 'wf-1', - timestamp: 1600, - payload: { - durationMs: 600, - costUsd: 0.03, - tokenUsage: { input: 40, output: 60 }, - agentCalls: 1, - loopCalls: 1, - }, - }, - ], -} - -const projection = { - experimentId: 'exp-1', - candidateId: 'workflow-driver-v1', - seed: 7, - model: 'claude-sonnet-4-6@2025-04-15', - promptHash: 'p'.repeat(64), - configHash: 'c'.repeat(64), - commitSha: 'cafebabe', - splitTag: 'search' as const, - scenarioId: 'scenario-1', -} - -function workflowRunRecord(args: { - candidateId: string - scenarioId?: string - seed: number - score: number - costUsd?: number -}): RunRecord { - return { - runId: `${args.candidateId}-${args.scenarioId ?? 'missing'}-${args.seed}`, - experimentId: 'workflow-driver-promotion', - candidateId: args.candidateId, - seed: args.seed, - model: 'claude-sonnet-4-6@2025-04-15', - promptHash: 'p'.repeat(64), - configHash: 'c'.repeat(64), - commitSha: 'cafebabe', - wallMs: 1000, - costUsd: args.costUsd ?? 0.01, - tokenUsage: { input: 10, output: 5 }, - outcome: { - holdoutScore: args.score, - raw: { workflow_driver_score: args.score }, - }, - splitTag: 'holdout', - ...(args.scenarioId ? { scenarioId: args.scenarioId } : {}), - } -} - -describe('workflow trace substrate', () => { - it('validates and summarizes workflow trace envelopes', () => { - const valid = validateWorkflowTraceEnvelope(envelope) - expect(valid.runId).toBe('wf-1') - const summary = summarizeWorkflowTrace(valid) - expect(summary).toMatchObject({ - runId: 'wf-1', - durationMs: 600, - costUsd: 0.03, - tokenUsage: { input: 40, output: 60 }, - phaseCount: 1, - branchCount: 1, - failedBranchCount: 0, - agentCalls: 1, - loopCalls: 1, - verifierCalls: 1, - analystCalls: 1, - reviewerCalls: 1, - failed: false, - }) - }) - - it('summarizes failed workflow branches as first-class trace evidence', () => { - const failedEnvelope: WorkflowTraceEnvelope = { - traceVersion: 'workflow-trace-v1', - runId: 'wf-failed-branch', - events: [ - { - kind: 'workflow.started', - runId: 'wf-failed-branch', - timestamp: 1, - payload: { - meta: { name: 'failed-branch', description: 'Failed branch trace' }, - depth: 0, - caps: { maxFanout: 1, maxDepth: 1 }, - }, - }, - { - kind: 'workflow.branch.started', - runId: 'wf-failed-branch', - timestamp: 2, - payload: { operation: 'parallel', branchIndex: 0, phase: 'Build' }, - }, - { - kind: 'workflow.branch.failed', - runId: 'wf-failed-branch', - timestamp: 3, - payload: { - operation: 'parallel', - branchIndex: 0, - phase: 'Build', - durationMs: 1, - message: 'worker failed', - }, - }, - { - kind: 'workflow.failed', - runId: 'wf-failed-branch', - timestamp: 4, - payload: { message: 'worker failed', phase: 'Build' }, - }, - ], - } - - const summary = summarizeWorkflowTrace(failedEnvelope) - expect(summary.branchCount).toBe(0) - expect(summary.failedBranchCount).toBe(1) - expect(summary.failed).toBe(true) - - const executionSummary = summarizeWorkflowExecution(failedEnvelope) - expect(executionSummary.phaseGraph.nodes[0]).toMatchObject({ - title: 'Build', - eventCount: 3, - failedBranchCount: 1, - }) - expect(executionSummary.phaseGraph.branches[0]).toMatchObject({ - operation: 'parallel', - branchIndex: 0, - phase: 'Build', - status: 'failed', - durationMs: 1, - message: 'worker failed', - }) - - const record = workflowTraceToRunRecord(failedEnvelope, projection) - expect(record.outcome.searchScore).toBe(0) - expect(record.outcome.raw.workflow_branch_failures).toBe(1) - }) - - it('projects failed delegate events as first-class workflow evidence', () => { - const failedDelegateEnvelope: WorkflowTraceEnvelope = { - traceVersion: 'workflow-trace-v1', - runId: 'wf-failed-agent', - events: [ - { - kind: 'workflow.started', - runId: 'wf-failed-agent', - timestamp: 1, - payload: { - meta: { name: 'failed-agent', description: 'Failed delegate trace' }, - depth: 0, - caps: { maxFanout: 1, maxDepth: 1 }, - }, - }, - { - kind: 'workflow.phase', - runId: 'wf-failed-agent', - timestamp: 2, - payload: { title: 'Build' }, - }, - { - kind: 'workflow.agent.started', - runId: 'wf-failed-agent', - timestamp: 3, - payload: { - index: 0, - label: 'implementation', - promptChars: 64, - phase: 'Build', - }, - }, - { - kind: 'workflow.agent.failed', - runId: 'wf-failed-agent', - timestamp: 13, - payload: { - index: 0, - label: 'implementation', - durationMs: 10, - message: 'sandbox worker crashed', - code: 'SandboxExit', - phase: 'Build', - }, - }, - { - kind: 'workflow.failed', - runId: 'wf-failed-agent', - timestamp: 14, - payload: { message: 'sandbox worker crashed', code: 'SandboxExit', phase: 'Build' }, - }, - ], - } - - expect(validateWorkflowTraceEnvelope(failedDelegateEnvelope).events[3]?.kind).toBe( - 'workflow.agent.failed', - ) - const summary = summarizeWorkflowTrace(failedDelegateEnvelope) - expect(summary).toMatchObject({ - agentCalls: 1, - agentFailures: 1, - loopFailures: 0, - verifierFailures: 0, - analystFailures: 0, - reviewerFailures: 0, - failed: true, - failureMessage: 'sandbox worker crashed', - }) - - const executionSummary = summarizeWorkflowExecution(failedDelegateEnvelope) - expect(executionSummary.agentFailureDetails[0]).toMatchObject({ - index: 0, - label: 'implementation', - phase: 'Build', - durationMs: 10, - message: 'sandbox worker crashed', - code: 'SandboxExit', - }) - expect(executionSummary.phaseGraph.nodes[0]).toMatchObject({ - title: 'Build', - agentCalls: 1, - agentFailures: 1, - }) - - const trajectory = workflowTraceToFeedbackTrajectory(failedDelegateEnvelope, { - projectId: 'blueprint-agent', - scenarioId: 'scenario-1', - task: 'Build the app', - }) - expect(trajectory.attempts).toHaveLength(1) - expect(trajectory.attempts[0]).toMatchObject({ - artifactType: 'action', - artifact: { - index: 0, - label: 'implementation', - message: 'sandbox worker crashed', - }, - metadata: { - eventKind: 'workflow.agent.failed', - failed: true, - message: 'sandbox worker crashed', - code: 'SandboxExit', - }, - }) - expect(trajectory.outcome?.metrics?.workflow_agent_failures).toBe(1) - - const record = workflowTraceToRunRecord(failedDelegateEnvelope, projection) - expect(record.outcome.searchScore).toBe(0) - expect(record.outcome.raw.workflow_agent_calls).toBe(1) - expect(record.outcome.raw.workflow_agent_failures).toBe(1) - - const pack = buildWorkflowAnalystFeedbackPack({ - envelope: failedDelegateEnvelope, - generatedAt: '2026-06-01T00:00:02.000Z', - }) - expect(pack.driverContextLines).toContain('delegateFailures=agent:1') - }) - - it('builds rich execution summaries from runtime workflow events', () => { - const summary = summarizeWorkflowExecution(envelope, { source: 'export const meta = {}' }) - - expect(summary.source).toBe('export const meta = {}') - expect(summary.eventKinds).toMatchObject({ - 'workflow.started': 1, - 'workflow.phase': 1, - 'workflow.branch.started': 1, - 'workflow.branch.ended': 1, - 'workflow.agent.ended': 1, - 'workflow.verifier.ended': 1, - 'workflow.ended': 1, - }) - expect(summary.phases).toEqual(['Plan']) - expect(summary.phaseGraph.nodes[0]).toMatchObject({ - id: 'phase-0', - title: 'Plan', - startedAt: 1010, - endedAt: 1210, - eventCount: 4, - branchCount: 1, - failedBranchCount: 0, - agentCalls: 1, - loopCalls: 0, - costUsd: 0.01, - tokenUsage: { input: 10, output: 20 }, - }) - expect(summary.phaseGraph.branches[0]).toMatchObject({ - id: 'branch-0', - operation: 'parallel', - branchIndex: 0, - phase: 'Plan', - status: 'ended', - startedAt: 1020, - endedAt: 1210, - durationMs: 190, - }) - expect(summary.agentRuns[0]).toMatchObject({ - index: 0, - label: 'planner', - phase: 'Plan', - costUsd: 0.01, - tokenUsage: { input: 10, output: 20 }, - trace: { text: 'plan' }, - }) - expect(summary.loopRuns[0]).toMatchObject({ - index: 0, - label: 'implement', - costUsd: 0.02, - }) - expect(summary.verifierOutputs[0]?.output).toEqual({ allPass: true }) - expect(summary.analystOutputs[0]?.output).toEqual({ findings: [] }) - expect(summary.reviewerOutputs[0]?.output).toEqual({ shouldContinue: false }) - }) - - it('projects workflow traces into canonical RunRecords', () => { - const record = workflowTraceToRunRecord(envelope, { ...projection, score: 0.82 }) - expect(record.runId).toBe('wf-1') - expect(record.candidateId).toBe('workflow-driver-v1') - expect(record.outcome.searchScore).toBe(0.82) - expect(record.outcome.raw.workflow_agent_calls).toBe(1) - expect(record.outcome.raw.workflow_branches).toBe(1) - expect(record.outcome.raw.workflow_branch_failures).toBe(0) - expect(record.outcome.raw.workflow_loop_calls).toBe(1) - expect(record.outcome.raw.workflow_verifier_calls).toBe(1) - expect(record.outcome.raw.workflow_analyst_calls).toBe(1) - expect(record.outcome.raw.workflow_reviewer_calls).toBe(1) - expect(record.tokenUsage).toEqual({ input: 40, output: 60 }) - }) - - it('projects workflow traces into feedback trajectories for RL/export consumers', () => { - const trajectory = workflowTraceToFeedbackTrajectory(envelope, { - projectId: 'blueprint-agent', - scenarioId: 'scenario-1', - task: 'Build the app', - score: 0.82, - tags: { driver: 'workflow-driver-v1' }, - }) - expect(trajectory.id).toBe('wf-1') - expect(trajectory.attempts).toHaveLength(5) - expect(trajectory.attempts.map((a) => a.artifactType)).toEqual([ - 'action', - 'decision', - 'decision', - 'data', - 'decision', - ]) - expect(trajectory.outcome?.metrics?.workflow_tokens_output).toBe(60) - expect(trajectory.outcome?.metrics?.workflow_branches).toBe(1) - expect(trajectory.outcome?.metrics?.workflow_branch_failures).toBe(0) - expect(trajectory.outcome?.metrics?.workflow_analyst_calls).toBe(1) - expect(trajectory.tags?.driver).toBe('workflow-driver-v1') - }) - - it('wraps agent-runtime WorkflowResult objects into eval trace envelopes', () => { - const runtimeResult = { - runId: envelope.runId, - meta: { name: 'driver-authored', description: 'Runtime-generated workflow' }, - output: { files: ['src/App.tsx'] }, - events: envelope.events, - } - - const wrapped = workflowRuntimeResultToTraceEnvelope(runtimeResult, { - topology: envelope.topology, - metadata: { productId: 'blueprint-agent' }, - }) - const record = workflowTraceToRunRecord(wrapped, projection) - - expect(wrapped.traceVersion).toBe('workflow-trace-v1') - expect(wrapped.metadata).toMatchObject({ - productId: 'blueprint-agent', - runtimeResult: { meta: runtimeResult.meta }, - }) - expect(JSON.stringify(wrapped.metadata)).not.toContain('src/App.tsx') - expect(record.outcome.searchScore).toBe(1) - }) - - it('wraps emitted runtime events from failed workflows without needing a WorkflowResult', () => { - const failed = workflowEventsToTraceEnvelope([ - { - kind: 'workflow.started', - runId: 'wf-failed', - timestamp: 1000, - payload: { - meta: { name: 'failed-workflow', description: 'Failed workflow trace' }, - depth: 0, - caps: { maxFanout: 1, maxDepth: 1 }, - }, - }, - { - kind: 'workflow.failed', - runId: 'wf-failed', - timestamp: 1100, - payload: { message: 'worker exhausted its budget' }, - }, - ]) - - expect(summarizeWorkflowTrace(failed)).toMatchObject({ - runId: 'wf-failed', - durationMs: 100, - failed: true, - failureMessage: 'worker exhausted its budget', - }) - expect(workflowTraceToRunRecord(failed, projection).failureMode).toBe('workflow_failed') - }) - - it('rejects events from mixed run ids', () => { - expect(() => - validateWorkflowTraceEnvelope({ - ...envelope, - events: [ - ...envelope.events, - { - kind: 'workflow.log', - runId: 'other', - timestamp: 1700, - payload: { message: 'other run' }, - }, - ], - }), - ).toThrow(/does not match/) - }) - - it('rejects non-canonical workflow event kinds', () => { - expect(() => - validateWorkflowTraceEnvelope({ - ...envelope, - events: envelope.events.map((event, index) => - index === 3 ? { ...event, kind: 'workflow.agent.done' } : event, - ), - }), - ).toThrow(/unknown workflow trace event kind/) - }) - - it('rejects malformed typed workflow event payloads', () => { - expect(() => - validateWorkflowTraceEnvelope({ - ...envelope, - events: envelope.events.map((event, index) => - index === 3 - ? { - ...event, - payload: { - ...event.payload, - costUsd: -1, - }, - } - : event, - ), - }), - ).toThrow(/workflow\.agent\.ended\.payload\.costUsd/) - }) - - it('rejects workflow events emitted after terminal completion', () => { - expect(() => - validateWorkflowTraceEnvelope({ - ...envelope, - events: [ - ...envelope.events, - { - kind: 'workflow.log', - runId: 'wf-1', - timestamp: 1700, - payload: { message: 'late log' }, - }, - ], - }), - ).toThrow(/terminal event must be last/) - }) - - it('rejects empty emitted runtime event buffers', () => { - expect(() => workflowEventsToTraceEnvelope([])).toThrow(/non-empty array/) - }) - - it('builds a bounded analyst feedback pack for the next workflow driver shot', () => { - const feedbackEnvelope: WorkflowTraceEnvelope = { - ...envelope, - events: [ - ...envelope.events.slice(0, 3), - { - kind: 'workflow.agent.ended', - runId: 'wf-1', - timestamp: 1190, - payload: { - index: 1, - label: 'implementation', - durationMs: 90, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - toolUsage: { - byTool: { - read: { calls: 2, errors: 1 }, - }, - }, - toolCalls: [ - { toolName: 'write', status: 'ok' }, - { toolName: 'test', error: 'vitest failed' }, - ], - }, - }, - ...envelope.events.slice(3), - ], - } - const verifier: VerificationReport = { - layers: [ - { - layer: 'typecheck', - status: 'fail', - score: 0.2, - durationMs: 1200, - findings: [ - { - severity: 'major', - message: 'Type error in Auction.ts', - evidence: 'Auction.ts:12', - }, - ], - reason: 'tsc failed', - diagnostics: { errors: 1 }, - }, - ], - passCount: 0, - failCount: 1, - skippedCount: 0, - errorCount: 0, - allPass: false, - blendedScore: 0.2, - durationMs: 1200, - startedAt: '2026-06-01T00:00:00.000Z', - finishedAt: '2026-06-01T00:00:01.200Z', - } - const failureClusters: FailureClusterReport = { - totalRuns: 3, - totalFailures: 2, - clusters: [ - { - failureClass: 'tool_recovery_failure', - toolName: 'test', - argPrefix: 'abc123', - runCount: 2, - scenarioIds: ['scenario-1'], - exampleRunId: 'wf-1', - exampleError: 'vitest failed', - }, - ], - } - const finding = makeFinding({ - analyst_id: 'failure-mode', - severity: 'high', - area: 'verification', - claim: 'The implementation loop keeps proceeding after typecheck failure', - confidence: 0.9, - evidence_refs: [{ kind: 'event', uri: 'workflow://wf-1/typecheck' }], - recommended_action: 'Stop the next workflow at typecheck until Auction.ts is fixed', - validation_plan: 'Run pnpm typecheck before fanout review', - }) - - const pack = buildWorkflowAnalystFeedbackPack({ - envelope: feedbackEnvelope, - verifier, - failureClusters, - analystFindings: [finding], - generatedAt: '2026-06-01T00:00:02.000Z', - }) - - expect(pack.schemaVersion).toBe('workflow-feedback-pack-v1') - expect(pack.summary.agentCalls).toBe(1) - expect(pack.verifier?.failedLayers).toEqual(['typecheck']) - expect(pack.verifier?.layers[0]?.findings[0]?.severity).toBe('high') - expect(pack.toolUsage).toEqual({ - totalCalls: 4, - erroredCalls: 2, - byTool: { - read: { calls: 2, errors: 1 }, - write: { calls: 1, errors: 0 }, - test: { calls: 1, errors: 1 }, - }, - }) - expect(pack.failureClusters[0]).toMatchObject({ - id: 'tool_recovery_failure|test|abc123|', - share: 1, - runCount: 2, - source: 'failure-cluster-view', - }) - expect(pack.findings[0]?.findingId).toBe(finding.finding_id) - expect(pack.recommendations).toContain('Fix verifier layer "typecheck": tsc failed') - expect(pack.recommendations).toContain( - 'Stop the next workflow at typecheck until Auction.ts is fixed', - ) - expect(pack.driverContextLines.join('\n')).toContain('verifier=fail') - }) - - it('renders feedback packs into a capped driver context block', () => { - const pack = buildWorkflowAnalystFeedbackPack({ - envelope, - generatedAt: '2026-06-01T00:00:02.000Z', - }) - - expect(renderWorkflowFeedbackPack(pack)).toContain('Workflow feedback pack for wf-1') - expect(renderWorkflowFeedbackPack(pack, { maxChars: 12 })).toHaveLength(12) - }) - - it('sanitizes workflow traces for intelligence export without losing clustering evidence', () => { - const sensitiveEnvelope: WorkflowTraceEnvelope = { - ...envelope, - events: [ - ...envelope.events.slice(0, 2), - { - kind: 'workflow.agent.ended', - runId: 'wf-1', - timestamp: 1200, - payload: { - index: 1, - label: 'implementation', - durationMs: 90, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - apiKey: 'sk-test-secret-value', - toolArgs: { prompt: 'Use Bearer abcdefghijklmnop', count: 2 }, - filePath: 'src/App.tsx', - content: 'const apiKey = "sk-another-secret-value"', - }, - }, - ...envelope.events.slice(3), - ], - artifacts: [ - { - kind: 'source-file', - uri: 'artifact://src/App.tsx', - metadata: { - path: 'src/App.tsx', - contents: 'export const secret = "sk-file-secret-value"', - }, - }, - ], - } - - const sanitized = sanitizeWorkflowTraceEnvelope(sensitiveEnvelope, { hashSalt: 'test' }) - - const agentPayload = sanitized.envelope.events[2]?.payload as Record - expect(agentPayload.apiKey).toBe('[redacted:apiKey]') - expect(agentPayload.toolArgs).toMatchObject({ redacted: true, shape: { type: 'object' } }) - expect(agentPayload.content).toMatchObject({ redacted: true, shape: { type: 'string' } }) - expect(JSON.stringify(sanitized.envelope)).not.toContain('sk-test-secret-value') - expect(JSON.stringify(sanitized.envelope)).not.toContain('sk-file-secret-value') - expect(sanitized.report.hashedArgs).toBe(1) - expect(sanitized.report.droppedArtifactContents).toBe(2) - expect(sanitized.report.droppedPayloadKeys.apiKey).toBe(1) - }) - - it('builds a grant-gated intelligence export envelope from the sanitized workflow trace', () => { - const sensitiveEnvelope: WorkflowTraceEnvelope = { - ...envelope, - metadata: { - requestId: 'req-1', - authorization: 'Bearer metadata-secret-token', - }, - events: [ - ...envelope.events.slice(0, 2), - { - kind: 'workflow.agent.ended', - runId: 'wf-1', - timestamp: 1200, - payload: { - index: 1, - label: 'implementation', - durationMs: 90, - costUsd: 0, - tokenUsage: { input: 0, output: 0 }, - apiKey: 'sk-test-secret-value', - toolCalls: [ - { - toolName: 'write_file', - toolArgs: { path: 'src/App.tsx', content: 'const key = "sk-tool-secret-value"' }, - status: 'ok', - }, - ], - filePath: 'src/App.tsx', - content: 'const apiKey = "sk-file-secret-value"', - }, - }, - ...envelope.events.slice(3), - ], - artifacts: [ - { - kind: 'source-file', - uri: 'artifact://src/App.tsx', - sha256: 'a'.repeat(64), - metadata: { - path: 'src/App.tsx', - contents: 'export const secret = "sk-artifact-secret-value"', - }, - }, - ], - } - - expect(() => - buildWorkflowTraceIntelligenceEnvelope({ - envelope: sensitiveEnvelope, - productId: 'blueprint-agent', - grants: [], - }), - ).toThrow(/requires at least one opt-in grant/) - - const exported = buildWorkflowTraceIntelligenceEnvelope({ - envelope: sensitiveEnvelope, - productId: 'blueprint-agent', - partnerId: 'partner-acme', - generatedAt: '2026-06-01T00:00:03.000Z', - grants: [ - { - grantId: 'grant-product-export', - subject: 'product', - subjectId: 'blueprint-agent', - scopes: ['workflow-trace:export'], - grantedAt: '2026-06-01T00:00:00.000Z', - }, - ], - sanitize: { hashSalt: 'intelligence-test' }, - links: { - traceArtifactUri: 'artifact://wf-1/trace.json', - exportBundleUri: 'artifact://wf-1/export-bundle.json', - partnerReportUri: 'artifact://wf-1/partner-report.md', - }, - metadata: { - route: 'vb.workflow-driver-v1', - cookie: 'session=secret', - }, - }) - - expect(validateWorkflowTraceIntelligenceEnvelope(exported).schemaVersion).toBe( - 'workflow-trace-intelligence-envelope-v1', - ) - expect(exported.destination).toBe('intelligence.tangle.tools') - expect(exported.grantIds).toEqual(['grant-product-export']) - expect(exported.summary.agentRuns[0]?.label).toBe('implementation') - expect(exported.compactEvidence.toolNames).toEqual(['write_file']) - expect(exported.compactEvidence.artifacts[0]).toMatchObject({ - kind: 'source-file', - uri: 'artifact://src/App.tsx', - sha256: 'a'.repeat(64), - }) - expect(exported.compactEvidence.redactedHashes).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - path: 'events[2].payload.toolCalls[0].toolArgs', - shape: { type: 'object', keys: ['content', 'path'] }, - }), - expect.objectContaining({ - path: 'events[2].payload.content', - shape: { type: 'string' }, - }), - expect.objectContaining({ - path: 'artifacts[0].metadata.contents', - shape: { type: 'string' }, - }), - ]), - ) - expect(exported.links?.exportBundleUri).toBe('artifact://wf-1/export-bundle.json') - const serialized = JSON.stringify(exported) - expect(serialized).not.toContain('sk-test-secret-value') - expect(serialized).not.toContain('sk-tool-secret-value') - expect(serialized).not.toContain('sk-file-secret-value') - expect(serialized).not.toContain('sk-artifact-secret-value') - expect(serialized).not.toContain('metadata-secret-token') - expect(serialized).not.toContain('session=secret') - expect(exported.sanitization.hashedArgs).toBe(1) - expect(exported.sanitization.droppedArtifactContents).toBeGreaterThanOrEqual(2) - - const tampered = structuredClone(exported) - tampered.compactEvidence.toolNames = [] - expect(() => validateWorkflowTraceIntelligenceEnvelope(tampered)).toThrow( - /compactEvidence.toolNames/, - ) - }) - - it('builds a shareable workflow report with sanitized trace, RL trajectory, and PR-ready findings', () => { - const docsFinding = makeFinding({ - analyst_id: 'knowledge-gap', - severity: 'high', - area: 'api-docs', - claim: 'The generated app guessed an SDK method that is missing from the partner docs', - confidence: 0.88, - evidence_refs: [{ kind: 'event', uri: 'workflow://wf-1/analyst' }], - recommended_action: 'Open a docs PR adding the SDK method and a runnable example', - validation_plan: 'Regenerate the workflow against the updated docs', - }) - - const report = buildWorkflowPartnerReport({ - envelope, - analystFindings: [docsFinding], - generatedAt: '2026-06-01T00:00:02.000Z', - trajectory: { - projectId: 'blueprint-agent', - scenarioId: 'scenario-1', - task: 'Build the app', - score: 0.82, - tags: { driver: 'workflow-driver-v1' }, - }, - runRecord: { ...projection, runId: 'caller-override', score: 0.82 }, - links: { - traceArtifactUri: 'artifact://wf-1/trace.json', - exportBundleUri: 'artifact://wf-1/export-bundle.json', - partnerReportUri: 'artifact://wf-1/partner-report.md', - }, - }) - - expect(report.schemaVersion).toBe('workflow-partner-report-v1') - expect(report.docsApiGaps[0]?.claim).toContain('partner docs') - expect(report.prReadyFindings[0]?.recommendedAction).toContain('docs PR') - expect(report.links?.traceArtifactUri).toBe('artifact://wf-1/trace.json') - expect(report.links?.exportBundleUri).toBe('artifact://wf-1/export-bundle.json') - expect(report.exportBundle.trajectory.attempts).toHaveLength(5) - expect(report.exportBundle.runRecord?.runId).toBe('wf-1') - expect(report.exportBundle.runRecord?.outcome.searchScore).toBe(0.82) - expect(validateWorkflowPartnerReport(report).runId).toBe('wf-1') - expect(renderWorkflowPartnerReport(report)).toContain('Docs/API gaps') - - const tampered = structuredClone(report) - tampered.summary.eventCount = 999 - expect(() => validateWorkflowPartnerReport(tampered)).toThrow(/summary/) - }) - - it('gates workflow-driver promotion against reviewer-loop baseline on paired heldout scenarios', () => { - const records = ['auction', 'dex', 'payroll'].flatMap((scenarioId) => [ - workflowRunRecord({ - candidateId: 'reviewer-loop-v1', - scenarioId, - seed: 7, - score: 0.6, - }), - workflowRunRecord({ - candidateId: 'workflow-driver-v1', - scenarioId, - seed: 7, - score: 0.82, - }), - ]) - records.push( - workflowRunRecord({ - candidateId: 'reviewer-loop-v1', - scenarioId: 'out-of-scope', - seed: 7, - score: 1, - }), - workflowRunRecord({ - candidateId: 'workflow-driver-v1', - scenarioId: 'out-of-scope', - seed: 7, - score: 0, - }), - ) - - const decision = decideWorkflowDriverPromotion({ - records, - expectedScenarioIds: ['auction', 'dex', 'payroll'], - minPairedHoldoutRuns: 3, - resamples: 200, - seed: 1, - generatedAt: '2026-06-01T00:00:04.000Z', - }) - - expect(decision.schemaVersion).toBe('workflow-driver-promotion-v1') - expect(decision.promote).toBe(true) - expect(decision.rejectionCode).toBeNull() - expect(decision.baselineStrategyId).toBe('reviewer-loop-v1') - expect(decision.candidateStrategyId).toBe('workflow-driver-v1') - expect(decision.evidence.pairedRuns).toBe(3) - expect(decision.evidence.pairedScenarioIds).toEqual(['auction', 'dex', 'payroll']) - expect(decision.evidence.lift).toBeCloseTo(0.22, 6) - expect(decision.evidence.liftCi.low).toBeGreaterThan(0) - expect(decision.evidence.pairs.map((pair) => pair.key)).toEqual([ - 'auction::7', - 'dex::7', - 'payroll::7', - ]) - }) - - it('fails closed when a workflow promotion gate is missing a heldout scenario pair', () => { - const decision = decideWorkflowDriverPromotion({ - records: [ - workflowRunRecord({ - candidateId: 'reviewer-loop-v1', - scenarioId: 'auction', - seed: 1, - score: 0.7, - }), - workflowRunRecord({ - candidateId: 'workflow-driver-v1', - scenarioId: 'auction', - seed: 1, - score: 0.9, - }), - workflowRunRecord({ - candidateId: 'reviewer-loop-v1', - scenarioId: 'dex', - seed: 1, - score: 0.7, - }), - ], - expectedScenarioIds: ['auction', 'dex'], - minPairedHoldoutRuns: 2, - resamples: 50, - seed: 2, - }) - - expect(decision.promote).toBe(false) - expect(decision.rejectionCode).toBe('missing_holdout_pairs') - expect(decision.evidence.missingScenarioIds).toEqual(['dex']) - expect(decision.reason).toContain('no paired baseline/candidate holdout record') - }) - - it('rejects workflow promotion records without scenarioId instead of pairing by seed only', () => { - expect(() => - decideWorkflowDriverPromotion({ - records: [ - workflowRunRecord({ candidateId: 'reviewer-loop-v1', seed: 1, score: 0.7 }), - workflowRunRecord({ candidateId: 'workflow-driver-v1', seed: 1, score: 0.9 }), - ], - minPairedHoldoutRuns: 1, - }), - ).toThrow(/missing scenarioId/) - }) -}) diff --git a/tsup.config.ts b/tsup.config.ts index ef2db4df..7d0b0d42 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -7,35 +7,21 @@ export default defineConfig({ control: 'src/control.ts', reporting: 'src/reporting.ts', rl: 'src/rl/index.ts', - diagnose: 'src/diagnose/index.ts', fuzz: 'src/fuzz/index.ts', traces: 'src/traces.ts', - 'telemetry/index': 'src/telemetry/index.ts', - 'telemetry/file': 'src/telemetry/sink-file.ts', 'wire/index': 'src/wire/index.ts', 'benchmarks/index': 'src/benchmarks/index.ts', 'pipelines/index': 'src/pipelines/index.ts', 'meta-eval/index': 'src/meta-eval/index.ts', - 'prm/index': 'src/prm/index.ts', 'builder-eval/index': 'src/builder-eval/index.ts', - 'governance/index': 'src/governance/index.ts', - 'knowledge/index': 'src/knowledge/index.ts', 'matrix/index': 'src/matrix/index.ts', 'multishot/index': 'src/multishot/index.ts', - 'perf/index': 'src/perf/index.ts', - 'product-benchmark/index': 'src/product-benchmark/index.ts', 'campaign/index': 'src/campaign/index.ts', 'storyboard/index': 'src/storyboard/index.ts', 'authenticity/index': 'src/authenticity/index.ts', - 'groundedness/index': 'src/groundedness/index.ts', 'belief-state/index': 'src/belief-state/index.ts', - 'workflow/index': 'src/workflow/index.ts', 'contract/index': 'src/contract/index.ts', - 'adapters/langchain': 'src/adapters/langchain.ts', - 'adapters/http': 'src/adapters/http.ts', - 'adapters/otel': 'src/adapters/otel.ts', 'hosted/index': 'src/hosted/index.ts', - testing: 'src/testing.ts', cli: 'src/cli.ts', }, format: ['esm'],