From eeac9c2364434c34791bd9aade7446b737f0a0fb Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:05:41 -0600 Subject: [PATCH 1/5] feat(statistics): add clustered paired binary inference --- .claude/skills/agent-eval/SKILL.md | 1 + .../2026-07-09T21-47-00Z/findings.jsonl | 3 + .../2026-07-09T21-47-00Z/manifest.json | 19 + .../2026-07-09T21-47-00Z/summary.md | 16 + .../2026-07-09T21-54-37Z/findings.jsonl | 3 + .../2026-07-09T21-54-37Z/manifest.json | 17 + .../2026-07-09T21-54-37Z/summary.md | 14 + .evolve/skill-runs.jsonl | 1 + src/clustered-paired-binary.ts | 433 ++++++++++++++++++ src/index.ts | 12 + src/statistics.ts | 37 ++ tests/clustered-paired-binary.test.ts | 302 ++++++++++++ 12 files changed, 858 insertions(+) create mode 100644 .evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl create mode 100644 .evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json create mode 100644 .evolve/critical-audit/2026-07-09T21-47-00Z/summary.md create mode 100644 .evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl create mode 100644 .evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json create mode 100644 .evolve/critical-audit/2026-07-09T21-54-37Z/summary.md create mode 100644 src/clustered-paired-binary.ts create mode 100644 tests/clustered-paired-binary.test.ts diff --git a/.claude/skills/agent-eval/SKILL.md b/.claude/skills/agent-eval/SKILL.md index 6b8d26f3..5e8278c6 100644 --- a/.claude/skills/agent-eval/SKILL.md +++ b/.claude/skills/agent-eval/SKILL.md @@ -145,6 +145,7 @@ Extend, don't fork — see §"Extend, don't duplicate." | `HeldOutGate` | `held-out-gate.ts` | Paired-delta + overfit-gap gate. Three rejection codes: `few_runs`, `negative_delta`, `overfit_gap`. Use before promoting an optimizer's top-1. Pairs with `promotion-gate.ts` (bootstrap CI for "is this real?") — use both. | | `RunRecord` | `run-record.ts` | Typed run schema. `validateRunRecord` throws on missing fields and on bare model aliases — record the snapshot (`claude-sonnet-4-6@2025-04-15`). | | `pairedBootstrap`, `pairedWilcoxon`, `bhAdjust` | `paired-stats.ts` | Stats primitives. Pass `seed` to `pairedBootstrap` when the result feeds a CI / promotion decision. | +| `clusteredPairedBinary`, `holm` | `clustered-paired-binary.ts`, `statistics.ts` | Repository- or subject-clustered paired binary inference plus strong family-wise multiple-comparison adjustment. The clustered result preserves original paired/unpaired rows, resamples whole clusters for a task-weighted interval, and sign-flips whole-cluster totals for the same estimand with a required deterministic seed. | | `runCanaries` | `canary.ts` | Silent fallback (constant confidence), calibration drift (KS), distribution shift (chi-square). Returns a report; doesn't fail tests — wire it to a notification. | | `summaryTable`, `paretoChart`, `gainHistogram` | `summary-report.ts` | A/B reporting. `summaryTable` emits markdown with bootstrap CIs + paired Wilcoxon p (BH-adjusted) + Cohen's d. The other two return vega-lite-friendly specs. | | `researchReport` | `summary-report.ts` | Async, launch-decision-grade artifact: paired-evidence-only verdicts (`promote` / `hold` / `equivalent` / `reject` / `needs_more_data`), ROPE, Pr(Δ>0), per-candidate MDE via `pairedMde`, SHA-256 `runFingerprint`, optional `preregistrationHash`, embedded methodology. See [`docs/research-report-methodology.md`](../../../docs/research-report-methodology.md). | diff --git a/.evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl new file mode 100644 index 00000000..4a881cf5 --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl @@ -0,0 +1,3 @@ +{"severity":"medium","file":"src/clustered-paired-binary.ts","line":366,"issue":"The sign-flip p-value tested the equal-cluster mean while the headline interval measured task-weighted risk difference, allowing contradictory significance claims for different estimands.","action":"Sign-flip whole-cluster outcome totals and divide by total matched pairs so the test statistic equals taskWeightedRiskDifference.","verification":"The 30 one-task positive plus 10 two-task negative repository construction must report RD 0.20 and a one-sided p-value above 0.05, not the pre-fix 0.00118."} +{"severity":"medium","file":"src/clustered-paired-binary.ts","line":180,"issue":"A single independent cluster produced a nominal zero-width 95% cluster-bootstrap interval, presenting unestimable cluster uncertainty as certainty.","action":"Return bootstrap null when fewer than two matched clusters exist while retaining the observed descriptive effect and sign-flip result.","verification":"A single improving repository must return taskWeightedRiskDifference 1 and bootstrap null."} +{"severity":"medium","file":"src/clustered-paired-binary.ts","line":228,"issue":"The API accepted one bootstrap draw, which could produce a nominal 95% interval [1,1] from two opposing repositories and falsely satisfy a lower-bound-above-zero promotion rule.","action":"Require at least ceil(2/(1-confidence)) draws so both requested interval tails are representable.","verification":"At confidence 0.95, bootstrapResamples 1 must fail with a minimum of 40 before sampling."} diff --git a/.evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json b/.evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json new file mode 100644 index 00000000..205955ce --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json @@ -0,0 +1,19 @@ +{ + "scope": [ + "src/clustered-paired-binary.ts", + "src/statistics.ts", + "src/index.ts", + "tests/clustered-paired-binary.test.ts", + ".claude/skills/agent-eval/SKILL.md" + ], + "base": "9c7fc3056b9effddda6a5b91f1467fd3dc1af225", + "head": "cf51ffd63b53daa66e2d8d8d8003ceec5123231d", + "project_type": "typescript-node", + "flags": ["diff-only"], + "findings_count_by_severity": { + "critical": 0, + "high": 0, + "medium": 3, + "low": 0 + } +} diff --git a/.evolve/critical-audit/2026-07-09T21-47-00Z/summary.md b/.evolve/critical-audit/2026-07-09T21-47-00Z/summary.md new file mode 100644 index 00000000..d6eb09c3 --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T21-47-00Z/summary.md @@ -0,0 +1,16 @@ +# Critical audit: clustered paired binary inference + +Score: 6/10. + +Three medium correctness findings block using this unreleased API as the DeepSWE promotion decision. + +## Fix plan + +1. Align the sign-flip statistic with the task-weighted interval. + Verification: reproduce the 40-repository unequal-size counterexample and require the p-value to test the reported `+0.20` effect. +2. Suppress cluster-bootstrap intervals below two independent repositories. + Verification: one improving repository returns `bootstrap: null`. +3. Reject a bootstrap draw count too small to represent both requested tails. + Verification: one draw at 95% confidence fails with minimum `40`. + +REQUEST_CHANGES — the original API could certify significance for a different effect and could fabricate certainty from one repository or one draw. diff --git a/.evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl new file mode 100644 index 00000000..84e20c1e --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl @@ -0,0 +1,3 @@ +{"severity":"medium","file":"src/clustered-paired-binary.ts","line":374,"issue":"Sign-flip estimand mismatch.","action":"Resolved by sign-flipping cluster totals for taskWeightedRiskDifference.","verification":"resolved: 40-repository counterexample now returns p=0.1413 for RD 0.20."} +{"severity":"medium","file":"src/clustered-paired-binary.ts","line":180,"issue":"One-cluster zero-width interval.","action":"Resolved by returning bootstrap null below two clusters.","verification":"resolved: single-repository adversarial test passes."} +{"severity":"medium","file":"src/clustered-paired-binary.ts","line":236,"issue":"Too few bootstrap draws accepted.","action":"Resolved by a confidence-dependent minimum draw count.","verification":"resolved: one draw at 95% fails with minimum 40."} diff --git a/.evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json b/.evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json new file mode 100644 index 00000000..e74df4ed --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json @@ -0,0 +1,17 @@ +{ + "priorRun": ".evolve/critical-audit/2026-07-09T21-47-00Z", + "scope": [ + "src/clustered-paired-binary.ts", + "tests/clustered-paired-binary.test.ts" + ], + "base": "9c7fc3056b9effddda6a5b91f1467fd3dc1af225", + "head": "cf51ffd63b53daa66e2d8d8d8003ceec5123231d", + "project_type": "typescript-node", + "flags": ["diff-only", "reaudit"], + "findings_count_by_severity": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0 + } +} diff --git a/.evolve/critical-audit/2026-07-09T21-54-37Z/summary.md b/.evolve/critical-audit/2026-07-09T21-54-37Z/summary.md new file mode 100644 index 00000000..b2a018db --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T21-54-37Z/summary.md @@ -0,0 +1,14 @@ +# Critical re-audit: clustered paired binary inference + +Score: 8/10. + +All three medium findings are resolved. + +The task-weighted interval and sign-flip test now answer the same question. +One repository reports no cluster interval. +Too few bootstrap draws fail before sampling. +The 40-repository counterexample, one-repository case, one-draw case, exact enumeration, Monte Carlo determinism, whole-cluster bootstrap, unpaired rows, and Holm adjustment are covered by executable tests. + +Residual requirement: consumers must reject unpaired rows before promotion, because the utility exposes them and intentionally computes complete-case statistics. + +APPROVE — no correctness blocker remains in the unreleased API. diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 0e538d31..ba0524d7 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -15,3 +15,4 @@ {"ts":"2026-06-01","skill":"pursue","generation":6,"goal":"empirical proof of substrate lift","outcome":"HONEST-NEGATIVE: mechanism proven e2e on AppWorld d3 (real backend), but NO significant prompt-opt lift on a competent baseline; lift needs a weak-baseline config","files":["src/campaign/presets/compare-drivers.ts","examples/benchmarks/gsm8k/compare-drivers.ts","examples/benchmarks/appworld/run-bench.ts","examples/benchmarks/appworld/repl_agent.py"],"branch":"pursue/empirical-proof","cost":"~$3.50 live"} {"skill":"/research","ts":"2026-06-05T13:21:01Z","project":"agent-eval-belief-roadmap","target":"belief-state-code-agent-corpus","operatorPrompt":"","durationMin":null,"verdict":"SHIP","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"critical-audit","ts":"2026-06-21T12:57:05Z","project":"agent-eval","target":"PR #264 review comment 4762012574","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"1 medium, 9 low; 8 fixed, 1 not_actionable, 1 deferred","files":[".gitignore","CHANGELOG.md","src/agent-profile.ts","src/agent-profile.test.ts","src/contract/define-agent-eval.ts","src/contract/index.ts","tests/contract-define-agent-eval.test.ts"]} +{"skill":"critical-audit","ts":"2026-07-09T21:54:37Z","project":"agent-eval","target":"clustered paired binary inference for repository-disjoint coding benchmarks","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"3 medium found and resolved: estimand mismatch, one-cluster interval, insufficient bootstrap draws","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} diff --git a/src/clustered-paired-binary.ts b/src/clustered-paired-binary.ts new file mode 100644 index 00000000..6cf0a8ff --- /dev/null +++ b/src/clustered-paired-binary.ts @@ -0,0 +1,433 @@ +/** + * Paired binary comparison for work items nested inside independent clusters. + * + * Pairing is delegated to {@link pairArms}; this module adds the cluster-aware + * estimands and inference that task-level McNemar/bootstrap utilities cannot + * provide. Callers keep their own row shape through accessors, and every + * matched or unpaired result returns the original row object unchanged. + */ + +import { ValidationError } from './errors' +import { type PairedArmRow, pairArms } from './paired-arms' +import { mulberry32 } from './statistics' + +export type ClusterSignFlipAlternative = 'two-sided' | 'greater' | 'less' + +export interface ClusteredPairedBinaryOptions { + /** Arm treated as the control side of every pair. */ + baselineArm: string + /** Arm treated as the treatment side of every pair. */ + treatmentArm: string + /** Stable work-item identity shared by both arms. */ + pairKey: (row: TRow) => string + /** Independent-cluster identity, shared by both rows in a matched pair. */ + clusterKey: (row: TRow) => string + /** Arm identity for this row. Rows from other arms are ignored. */ + arm: (row: TRow) => string + /** Binary outcome. */ + pass: (row: TRow) => boolean + /** Replicate identity when a work item has multiple rows in either arm. */ + repKey?: (row: TRow) => string | undefined + /** Deterministic seed for bootstrap and Monte Carlo sign-flip draws. */ + seed: number + /** Percentile confidence level. Default 0.95. */ + confidence?: number + /** Whole-cluster bootstrap draws. Default 10,000. */ + bootstrapResamples?: number + /** Sign-flip alternative. Default 'two-sided'. */ + alternative?: ClusterSignFlipAlternative + /** + * Enumerate every sign assignment at or below this many non-zero clusters; + * otherwise use Monte Carlo. Default 20; maximum 20. + */ + exactClusterLimit?: number + /** Monte Carlo sign-flip draws when exact enumeration is not used. Default 100,000. */ + signFlipResamples?: number +} + +export interface ClusteredMatchedPair { + pairKey: string + repIndex: number + clusterKey: string + baseline: TRow + treatment: TRow + baselinePass: boolean + treatmentPass: boolean +} + +export interface ClusteredBinaryCluster { + clusterKey: string + nPairs: number + /** Treatment passes and baseline fails. */ + b10: number + /** Baseline passes and treatment fails. */ + b01: number + /** Mean (treatment - baseline) binary outcome within this cluster. */ + meanDifference: number +} + +export interface ClusterBootstrapInterval { + /** The interval resamples clusters and recomputes this task-weighted statistic. */ + statistic: 'task-weighted-risk-difference' + lower: number + upper: number + confidence: number + resamples: number + seed: number +} + +export interface ClusterSignFlipResult { + /** Task-weighted paired risk difference, matching the reported bootstrap estimand. */ + statistic: number + /** + * Randomization p-value under whole-cluster arm-label exchangeability. + * `method: 'exact'` means every cluster-level sign assignment was enumerated; + * it does not make the exchangeability assumption unnecessary. + */ + pValue: number + alternative: ClusterSignFlipAlternative + method: 'exact' | 'monte-carlo' + /** Exact assignments enumerated or Monte Carlo assignments drawn. */ + assignments: number + nClusters: number + nNonZeroClusters: number + /** Null for exact enumeration, which has no random draws. */ + seed: number | null +} + +export interface ClusteredPairedBinaryStatistics { + nPairs: number + nClusters: number + /** Treatment passes and baseline fails, across all matched pairs. */ + b10: number + /** Baseline passes and treatment fails, across all matched pairs. */ + b01: number + /** Mean paired difference across tasks, so every task has equal weight. */ + taskWeightedRiskDifference: number + /** Mean of cluster-level paired differences, so every cluster has equal weight. */ + equalClusterMean: number + clusters: ClusteredBinaryCluster[] + /** Null below two independent clusters; one cluster cannot estimate cluster uncertainty. */ + bootstrap: ClusterBootstrapInterval | null + signFlip: ClusterSignFlipResult +} + +export interface ClusteredPairedBinaryResult { + matchedPairs: ClusteredMatchedPair[] + unpairedBaseline: TRow[] + unpairedTreatment: TRow[] + /** Null when there are no matched rows; absence is never reported as a zero effect. */ + statistics: ClusteredPairedBinaryStatistics | null +} + +interface ProjectedRow extends PairedArmRow { + clusterKey: string + original: TRow +} + +const DEFAULT_BOOTSTRAP_RESAMPLES = 10_000 +const DEFAULT_SIGN_FLIP_RESAMPLES = 100_000 +const DEFAULT_EXACT_CLUSTER_LIMIT = 20 +const SIGN_FLIP_SEED_SALT = 0x9e3779b9 + +/** + * Compare binary outcomes on matched work items while respecting independent + * clusters. The confidence interval resamples whole clusters and recomputes the + * task-weighted risk difference. The sign-flip test flips whole-cluster outcome + * totals and tests that same task-weighted estimand. + */ +export function clusteredPairedBinary( + rows: readonly TRow[], + options: ClusteredPairedBinaryOptions, +): ClusteredPairedBinaryResult { + const config = validateOptions(options) + const projected = projectSelectedRows(rows, options) + const paired = pairArms(projected, { + baselineArm: options.baselineArm, + treatmentArm: options.treatmentArm, + }) + + const matchedPairs = paired.pairs.map((pair): ClusteredMatchedPair => { + const baseline = pair.baseline as ProjectedRow + const treatment = pair.treatment as ProjectedRow + if (baseline.clusterKey !== treatment.clusterKey) { + throw new ValidationError( + `clusteredPairedBinary: pairKey '${pair.pairKey}' rep ${pair.repIndex} crosses clusters ` + + `('${baseline.clusterKey}' vs '${treatment.clusterKey}')`, + ) + } + return { + pairKey: pair.pairKey, + repIndex: pair.repIndex, + clusterKey: baseline.clusterKey, + baseline: baseline.original, + treatment: treatment.original, + baselinePass: baseline.pass!, + treatmentPass: treatment.pass!, + } + }) + + const unpairedBaseline = paired.unpairedBaseline.map( + (row) => (row as ProjectedRow).original, + ) + const unpairedTreatment = paired.unpairedTreatment.map( + (row) => (row as ProjectedRow).original, + ) + + if (matchedPairs.length === 0) { + return { matchedPairs, unpairedBaseline, unpairedTreatment, statistics: null } + } + + const clusters = summarizeClusters(matchedPairs) + const b10 = clusters.reduce((sum, cluster) => sum + cluster.b10, 0) + const b01 = clusters.reduce((sum, cluster) => sum + cluster.b01, 0) + const taskWeightedRiskDifference = (b10 - b01) / matchedPairs.length + const equalClusterMean = mean(clusters.map((cluster) => cluster.meanDifference)) + const bootstrap = clusters.length < 2 ? null : clusterBootstrap(clusters, config) + const signFlip = clusterSignFlip(clusters, config) + + return { + matchedPairs, + unpairedBaseline, + unpairedTreatment, + statistics: { + nPairs: matchedPairs.length, + nClusters: clusters.length, + b10, + b01, + taskWeightedRiskDifference, + equalClusterMean, + clusters, + bootstrap, + signFlip, + }, + } +} + +interface ValidatedConfig { + seed: number + confidence: number + bootstrapResamples: number + alternative: ClusterSignFlipAlternative + exactClusterLimit: number + signFlipResamples: number +} + +function validateOptions(options: ClusteredPairedBinaryOptions): ValidatedConfig { + assertNonEmptyString('baselineArm', options.baselineArm) + assertNonEmptyString('treatmentArm', options.treatmentArm) + if (options.baselineArm === options.treatmentArm) { + throw new ValidationError( + `clusteredPairedBinary: baselineArm and treatmentArm are both '${options.baselineArm}'`, + ) + } + if (!Number.isInteger(options.seed)) { + throw new ValidationError(`clusteredPairedBinary: seed must be an integer, got ${options.seed}`) + } + const confidence = options.confidence ?? 0.95 + if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) { + throw new ValidationError( + `clusteredPairedBinary: confidence must be in (0,1), got ${confidence}`, + ) + } + const bootstrapResamples = options.bootstrapResamples ?? DEFAULT_BOOTSTRAP_RESAMPLES + assertPositiveInteger('bootstrapResamples', bootstrapResamples) + const rawMinimumBootstrapResamples = 2 / (1 - confidence) + const minimumBootstrapResamples = Math.ceil( + rawMinimumBootstrapResamples - Number.EPSILON * Math.max(1, rawMinimumBootstrapResamples) * 8, + ) + if (bootstrapResamples < minimumBootstrapResamples) { + throw new ValidationError( + `clusteredPairedBinary: bootstrapResamples must be at least ${minimumBootstrapResamples} for confidence ${confidence} so both interval tails are represented, got ${bootstrapResamples}`, + ) + } + const signFlipResamples = options.signFlipResamples ?? DEFAULT_SIGN_FLIP_RESAMPLES + assertPositiveInteger('signFlipResamples', signFlipResamples) + const exactClusterLimit = options.exactClusterLimit ?? DEFAULT_EXACT_CLUSTER_LIMIT + if ( + !Number.isInteger(exactClusterLimit) || + exactClusterLimit < 0 || + exactClusterLimit > DEFAULT_EXACT_CLUSTER_LIMIT + ) { + throw new ValidationError( + `clusteredPairedBinary: exactClusterLimit must be an integer in [0,${DEFAULT_EXACT_CLUSTER_LIMIT}], got ${exactClusterLimit}`, + ) + } + const alternative = options.alternative ?? 'two-sided' + if (alternative !== 'two-sided' && alternative !== 'greater' && alternative !== 'less') { + throw new ValidationError( + `clusteredPairedBinary: alternative must be 'two-sided', 'greater', or 'less', got ${String(alternative)}`, + ) + } + return { + seed: options.seed, + confidence, + bootstrapResamples, + alternative, + exactClusterLimit, + signFlipResamples, + } +} + +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value <= 0) { + throw new ValidationError( + `clusteredPairedBinary: ${name} must be a positive integer, got ${value}`, + ) + } +} + +function projectSelectedRows( + rows: readonly TRow[], + options: ClusteredPairedBinaryOptions, +): ProjectedRow[] { + const projected: ProjectedRow[] = [] + for (const original of rows) { + const arm = options.arm(original) + assertNonEmptyString('arm', arm) + if (arm !== options.baselineArm && arm !== options.treatmentArm) continue + + const pairKey = options.pairKey(original) + const clusterKey = options.clusterKey(original) + const pass = options.pass(original) + const repKey = options.repKey?.(original) + assertNonEmptyString('pairKey', pairKey) + assertNonEmptyString('clusterKey', clusterKey) + if (typeof pass !== 'boolean') { + throw new ValidationError( + `clusteredPairedBinary: pass accessor must return boolean for pairKey '${pairKey}'`, + ) + } + if (repKey !== undefined) assertNonEmptyString('repKey', repKey) + projected.push({ pairKey, clusterKey, arm, pass, repKey, original }) + } + return projected +} + +function assertNonEmptyString(name: string, value: string): void { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new ValidationError( + `clusteredPairedBinary: ${name} accessor must return a non-empty string`, + ) + } +} + +function summarizeClusters( + pairs: readonly ClusteredMatchedPair[], +): ClusteredBinaryCluster[] { + const byCluster = new Map() + for (const pair of pairs) { + const summary = byCluster.get(pair.clusterKey) ?? { nPairs: 0, b10: 0, b01: 0 } + summary.nPairs++ + if (pair.treatmentPass && !pair.baselinePass) summary.b10++ + else if (pair.baselinePass && !pair.treatmentPass) summary.b01++ + byCluster.set(pair.clusterKey, summary) + } + return [...byCluster.entries()] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([clusterKey, summary]) => ({ + clusterKey, + ...summary, + meanDifference: (summary.b10 - summary.b01) / summary.nPairs, + })) +} + +function clusterBootstrap( + clusters: readonly ClusteredBinaryCluster[], + config: ValidatedConfig, +): ClusterBootstrapInterval { + const rng = mulberry32(config.seed) + const samples = new Array(config.bootstrapResamples) + for (let draw = 0; draw < config.bootstrapResamples; draw++) { + let differenceSum = 0 + let pairCount = 0 + for (let index = 0; index < clusters.length; index++) { + const cluster = clusters[Math.floor(rng() * clusters.length)]! + differenceSum += cluster.b10 - cluster.b01 + pairCount += cluster.nPairs + } + samples[draw] = differenceSum / pairCount + } + samples.sort((a, b) => a - b) + const alpha = 1 - config.confidence + const lowerIndex = Math.floor((alpha / 2) * config.bootstrapResamples) + const upperIndex = Math.min( + config.bootstrapResamples - 1, + Math.ceil((1 - alpha / 2) * config.bootstrapResamples) - 1, + ) + return { + statistic: 'task-weighted-risk-difference', + lower: samples[lowerIndex]!, + upper: samples[Math.max(lowerIndex, upperIndex)]!, + confidence: config.confidence, + resamples: config.bootstrapResamples, + seed: config.seed, + } +} + +function clusterSignFlip( + clusters: readonly ClusteredBinaryCluster[], + config: ValidatedConfig, +): ClusterSignFlipResult { + const clusterTotals = clusters.map((cluster) => cluster.b10 - cluster.b01) + const nonZero = clusterTotals.filter((delta) => delta !== 0) + const totalPairs = clusters.reduce((sum, cluster) => sum + cluster.nPairs, 0) + const statistic = clusterTotals.reduce((sum, delta) => sum + delta, 0) / totalPairs + if (nonZero.length <= config.exactClusterLimit) { + const assignments = 2 ** nonZero.length + let extreme = 0 + for (let mask = 0; mask < assignments; mask++) { + let sum = 0 + for (let index = 0; index < nonZero.length; index++) { + sum += (mask & (2 ** index) ? 1 : -1) * nonZero[index]! + } + const permuted = sum / totalPairs + if (isExtreme(permuted, statistic, config.alternative)) extreme++ + } + return { + statistic, + pValue: extreme / assignments, + alternative: config.alternative, + method: 'exact', + assignments, + nClusters: clusters.length, + nNonZeroClusters: nonZero.length, + seed: null, + } + } + + const signFlipSeed = (config.seed ^ SIGN_FLIP_SEED_SALT) >>> 0 + const rng = mulberry32(signFlipSeed) + let extreme = 0 + for (let draw = 0; draw < config.signFlipResamples; draw++) { + let sum = 0 + for (const delta of nonZero) sum += (rng() < 0.5 ? -1 : 1) * delta + const permuted = sum / totalPairs + if (isExtreme(permuted, statistic, config.alternative)) extreme++ + } + return { + statistic, + pValue: (extreme + 1) / (config.signFlipResamples + 1), + alternative: config.alternative, + method: 'monte-carlo', + assignments: config.signFlipResamples, + nClusters: clusters.length, + nNonZeroClusters: nonZero.length, + seed: signFlipSeed, + } +} + +function isExtreme( + candidate: number, + observed: number, + alternative: ClusterSignFlipAlternative, +): boolean { + const tolerance = Number.EPSILON * Math.max(1, Math.abs(observed)) * 16 + if (alternative === 'greater') return candidate >= observed - tolerance + if (alternative === 'less') return candidate <= observed + tolerance + return Math.abs(candidate) >= Math.abs(observed) - tolerance +} + +function mean(values: readonly number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length +} diff --git a/src/index.ts b/src/index.ts index a1e1e0c0..ff68d6c2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -170,6 +170,17 @@ export type { export { assertCapabilityHeadroom, capabilityHeadroom } from './capability-headroom' // ── Client / driver / judges / executor / benchmark / registry / reporter ─ export { ProductClient, runE2EWorkflow } from './client' +export type { + ClusterBootstrapInterval, + ClusteredBinaryCluster, + ClusteredMatchedPair, + ClusteredPairedBinaryOptions, + ClusteredPairedBinaryResult, + ClusteredPairedBinaryStatistics, + ClusterSignFlipAlternative, + ClusterSignFlipResult, +} from './clustered-paired-binary' +export { clusteredPairedBinary } from './clustered-paired-binary' export type { ControlActionFailureMode, ControlActionOutcome, @@ -402,6 +413,7 @@ export { confidenceInterval, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, + holm, interpretCliffs, interRaterReliability, mannWhitneyU, diff --git a/src/statistics.ts b/src/statistics.ts index 82f72885..d27e1ae6 100644 --- a/src/statistics.ts +++ b/src/statistics.ts @@ -845,6 +845,43 @@ export function bonferroni( return { adjusted, significant } } +/** + * Holm step-down family-wise error adjustment. + * + * P-values are sorted from smallest to largest, multiplied by their remaining + * hypothesis count, and made monotonically non-decreasing before being mapped + * back to input order. This uniformly dominates plain Bonferroni while keeping + * strong family-wise error control under arbitrary dependence. + */ +export function holm( + pValues: readonly number[], + alpha = 0.05, +): { adjusted: number[]; significant: boolean[] } { + if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) { + throw new Error(`holm: alpha must be in (0,1), got ${alpha}`) + } + for (const [index, pValue] of pValues.entries()) { + if (!Number.isFinite(pValue) || pValue < 0 || pValue > 1) { + throw new Error(`holm: pValues[${index}] must be in [0,1], got ${pValue}`) + } + } + const count = pValues.length + if (count === 0) return { adjusted: [], significant: [] } + + const ordered = pValues + .map((pValue, index) => ({ pValue, index })) + .sort((a, b) => a.pValue - b.pValue || a.index - b.index) + const adjusted = new Array(count) + let previous = 0 + for (let rank = 0; rank < count; rank++) { + const entry = ordered[rank]! + const stepAdjusted = Math.min(1, entry.pValue * (count - rank)) + previous = Math.max(previous, stepAdjusted) + adjusted[entry.index] = previous + } + return { adjusted, significant: adjusted.map((pValue) => pValue <= alpha) } +} + /** * Benjamini–Hochberg false discovery rate. Returns adjusted q-values and * significance at the target FDR; handles ties and preserves q monotonicity. diff --git a/tests/clustered-paired-binary.test.ts b/tests/clustered-paired-binary.test.ts new file mode 100644 index 00000000..e3187749 --- /dev/null +++ b/tests/clustered-paired-binary.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, it } from 'vitest' +import { type ClusteredPairedBinaryOptions, clusteredPairedBinary, holm } from '../src/index' + +interface BinaryRow { + task: string + cluster: string + arm: string + pass: boolean + rep?: string + note?: string +} + +function row(task: string, cluster: string, arm: string, pass: boolean, note?: string): BinaryRow { + return { task, cluster, arm, pass, note } +} + +function options( + overrides: Partial> = {}, +): ClusteredPairedBinaryOptions { + return { + baselineArm: 'baseline', + treatmentArm: 'candidate', + pairKey: (value) => value.task, + clusterKey: (value) => value.cluster, + arm: (value) => value.arm, + pass: (value) => value.pass, + repKey: (value) => value.rep, + seed: 1337, + bootstrapResamples: 2_000, + signFlipResamples: 5_000, + ...overrides, + } +} + +describe('clusteredPairedBinary', () => { + it('retains original matched and unpaired rows while reporting both weighting schemes', () => { + const a1Base = row('a1', 'cluster-a', 'baseline', false, 'original-a1-base') + const a1Candidate = row('a1', 'cluster-a', 'candidate', true, 'original-a1-candidate') + const cOnly = row('c-only', 'cluster-c', 'baseline', false) + const dOnly = row('d-only', 'cluster-d', 'candidate', true) + const rows = [ + a1Base, + a1Candidate, + row('a2', 'cluster-a', 'baseline', false), + row('a2', 'cluster-a', 'candidate', true), + row('a3', 'cluster-a', 'baseline', true), + row('a3', 'cluster-a', 'candidate', false), + row('b1', 'cluster-b', 'baseline', true), + row('b1', 'cluster-b', 'candidate', false), + cOnly, + dOnly, + ] + + const result = clusteredPairedBinary(rows, options()) + + expect(result.matchedPairs).toHaveLength(4) + expect(result.matchedPairs[0]!.baseline).toBe(a1Base) + expect(result.matchedPairs[0]!.treatment).toBe(a1Candidate) + expect(result.unpairedBaseline).toEqual([cOnly]) + expect(result.unpairedBaseline[0]).toBe(cOnly) + expect(result.unpairedTreatment).toEqual([dOnly]) + expect(result.unpairedTreatment[0]).toBe(dOnly) + + expect(result.statistics).not.toBeNull() + expect(result.statistics!.b10).toBe(2) + expect(result.statistics!.b01).toBe(2) + expect(result.statistics!.taskWeightedRiskDifference).toBe(0) + expect(result.statistics!.equalClusterMean).toBeCloseTo(-1 / 3, 12) + expect(result.statistics!.clusters).toEqual([ + { + clusterKey: 'cluster-a', + nPairs: 3, + b10: 2, + b01: 1, + meanDifference: 1 / 3, + }, + { + clusterKey: 'cluster-b', + nPairs: 1, + b10: 0, + b01: 1, + meanDifference: -1, + }, + ]) + }) + + it('resamples whole unequal-sized clusters rather than individual tasks', () => { + const rows: BinaryRow[] = [] + for (let index = 0; index < 10; index++) { + rows.push(row(`large-${index}`, 'large', 'baseline', false)) + rows.push(row(`large-${index}`, 'large', 'candidate', true)) + } + rows.push(row('small-0', 'small', 'baseline', true)) + rows.push(row('small-0', 'small', 'candidate', false)) + + const result = clusteredPairedBinary( + rows, + options({ seed: 7, bootstrapResamples: 2_000, alternative: 'greater' }), + ) + + expect(result.statistics!.taskWeightedRiskDifference).toBeCloseTo(9 / 11, 12) + expect(result.statistics!.equalClusterMean).toBe(0) + // A whole-cluster draw can select the small regressing cluster twice or + // the large improving cluster twice. Task-IID resampling would hide this. + expect(result.statistics!.bootstrap!.lower).toBe(-1) + expect(result.statistics!.bootstrap!.upper).toBe(1) + expect(result.statistics!.signFlip.statistic).toBeCloseTo(9 / 11, 12) + expect(result.statistics!.signFlip.pValue).toBe(0.5) + }) + + it('uses exact sign flips when the non-zero cluster count is within the limit', () => { + const rows: BinaryRow[] = [] + for (let index = 0; index < 3; index++) { + rows.push(row(`task-${index}`, `cluster-${index}`, 'baseline', false)) + rows.push(row(`task-${index}`, `cluster-${index}`, 'candidate', true)) + } + + const result = clusteredPairedBinary(rows, options({ alternative: 'greater' })) + + expect(result.statistics!.signFlip).toEqual({ + statistic: 1, + pValue: 1 / 8, + alternative: 'greater', + method: 'exact', + assignments: 8, + nClusters: 3, + nNonZeroClusters: 3, + seed: null, + }) + }) + + it('uses deterministic Monte Carlo sign flips above the exact limit', () => { + const rows: BinaryRow[] = [] + for (let index = 0; index < 8; index++) { + const candidatePass = index !== 7 + rows.push(row(`task-${index}`, `cluster-${index}`, 'baseline', !candidatePass)) + rows.push(row(`task-${index}`, `cluster-${index}`, 'candidate', candidatePass)) + } + const config = options({ exactClusterLimit: 2, seed: 42, signFlipResamples: 4_000 }) + + const forward = clusteredPairedBinary(rows, config) + const reversed = clusteredPairedBinary([...rows].reverse(), config) + + expect(forward).toEqual(reversed) + expect(forward.statistics!.signFlip.method).toBe('monte-carlo') + expect(forward.statistics!.signFlip.assignments).toBe(4_000) + expect(forward.statistics!.signFlip.pValue).toBeGreaterThan(0) + expect(forward.statistics!.signFlip.pValue).toBeLessThanOrEqual(1) + }) + + it('tests the task-weighted effect instead of a conflicting equal-cluster estimand', () => { + const rows: BinaryRow[] = [] + for (let cluster = 0; cluster < 30; cluster++) { + rows.push(row(`positive-${cluster}`, `positive-${cluster}`, 'baseline', false)) + rows.push(row(`positive-${cluster}`, `positive-${cluster}`, 'candidate', true)) + } + for (let cluster = 0; cluster < 10; cluster++) { + for (let task = 0; task < 2; task++) { + rows.push(row(`negative-${cluster}-${task}`, `negative-${cluster}`, 'baseline', true)) + rows.push(row(`negative-${cluster}-${task}`, `negative-${cluster}`, 'candidate', false)) + } + } + + const result = clusteredPairedBinary( + rows, + options({ + seed: 36020260709, + bootstrapResamples: 10_000, + signFlipResamples: 200_000, + exactClusterLimit: 20, + alternative: 'greater', + }), + ) + + expect(result.statistics!.taskWeightedRiskDifference).toBeCloseTo(0.2, 12) + expect(result.statistics!.equalClusterMean).toBeCloseTo(0.5, 12) + expect(result.statistics!.bootstrap!.lower).toBeLessThan(0) + expect(result.statistics!.signFlip.statistic).toBeCloseTo( + result.statistics!.taskWeightedRiskDifference, + 12, + ) + expect(result.statistics!.signFlip.pValue).toBeGreaterThan(0.05) + }) + + it('returns null statistics instead of fabricating a zero when no rows match', () => { + const baseline = row('baseline-only', 'cluster-a', 'baseline', false) + const treatment = row('candidate-only', 'cluster-b', 'candidate', true) + + const result = clusteredPairedBinary([baseline, treatment], options()) + + expect(result.matchedPairs).toEqual([]) + expect(result.unpairedBaseline).toEqual([baseline]) + expect(result.unpairedTreatment).toEqual([treatment]) + expect(result.statistics).toBeNull() + }) + + it('distinguishes a measured concordant zero from absent matched evidence', () => { + const rows = [ + row('task-1', 'cluster-a', 'baseline', true), + row('task-1', 'cluster-a', 'candidate', true), + ] + + const result = clusteredPairedBinary(rows, options()) + + expect(result.statistics).not.toBeNull() + expect(result.statistics!.taskWeightedRiskDifference).toBe(0) + expect(result.statistics!.equalClusterMean).toBe(0) + expect(result.statistics!.bootstrap).toBeNull() + expect(result.statistics!.signFlip.pValue).toBe(1) + }) + + it('does not fabricate a zero-width cluster interval from one independent cluster', () => { + const rows = [ + row('task-1', 'only-repository', 'baseline', false), + row('task-1', 'only-repository', 'candidate', true), + ] + + const result = clusteredPairedBinary(rows, options({ alternative: 'greater' })) + + expect(result.statistics!.taskWeightedRiskDifference).toBe(1) + expect(result.statistics!.bootstrap).toBeNull() + expect(result.statistics!.signFlip).toMatchObject({ statistic: 1, pValue: 0.5 }) + }) + + it('rejects too few bootstrap draws before a nominal interval can collapse to one draw', () => { + const rows = [ + row('improves', 'repository-a', 'baseline', false), + row('improves', 'repository-a', 'candidate', true), + row('regresses', 'repository-b', 'baseline', true), + row('regresses', 'repository-b', 'candidate', false), + ] + + expect(() => clusteredPairedBinary(rows, options({ seed: 7, bootstrapResamples: 1 }))).toThrow( + /bootstrapResamples must be at least 40 for confidence 0\.95/, + ) + }) + + it('rejects a matched work item whose arms claim different clusters', () => { + const rows = [ + row('task-1', 'cluster-a', 'baseline', false), + row('task-1', 'cluster-b', 'candidate', true), + ] + expect(() => clusteredPairedBinary(rows, options())).toThrow(/crosses clusters/) + }) + + it('validates accessor outputs and inference configuration', () => { + const rows = [ + row('task-1', 'cluster-a', 'baseline', false), + row('task-1', 'cluster-a', 'candidate', true), + ] + expect(() => clusteredPairedBinary(rows, options({ clusterKey: () => '' }))).toThrow( + /clusterKey accessor must return a non-empty string/, + ) + expect(() => clusteredPairedBinary(rows, options({ seed: 1.5 }))).toThrow( + /seed must be an integer/, + ) + expect(() => clusteredPairedBinary(rows, options({ confidence: 1 }))).toThrow( + /confidence must be in \(0,1\)/, + ) + expect(() => clusteredPairedBinary(rows, options({ bootstrapResamples: 0 }))).toThrow( + /bootstrapResamples must be a positive integer/, + ) + expect(() => clusteredPairedBinary(rows, options({ exactClusterLimit: 21 }))).toThrow( + /exactClusterLimit must be an integer in \[0,20\]/, + ) + expect(() => + clusteredPairedBinary( + rows, + options({ alternative: 'up' as ClusteredPairedBinaryOptions['alternative'] }), + ), + ).toThrow(/alternative must be/) + expect(() => + clusteredPairedBinary( + rows, + options({ pass: (() => 'yes') as unknown as (value: BinaryRow) => boolean }), + ), + ).toThrow(/pass accessor must return boolean/) + }) +}) + +describe('holm', () => { + it('applies monotone step-down adjustment and restores input order', () => { + expect(holm([0.04, 0.01, 0.03])).toEqual({ + adjusted: [0.06, 0.03, 0.06], + significant: [false, true, false], + }) + }) + + it('handles ties, the alpha boundary, and an empty family', () => { + expect(holm([0.025, 0.025])).toEqual({ + adjusted: [0.05, 0.05], + significant: [true, true], + }) + expect(holm([])).toEqual({ adjusted: [], significant: [] }) + }) + + it('rejects invalid p-values and alpha', () => { + expect(() => holm([0.1, Number.NaN])).toThrow(/pValues\[1\] must be in \[0,1\]/) + expect(() => holm([-0.1])).toThrow(/pValues\[0\] must be in \[0,1\]/) + expect(() => holm([0.1], 0)).toThrow(/alpha must be in \(0,1\)/) + }) +}) From 0837b2443cc8da37f7bde477b7bbf2ac9f4d623e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:07:15 -0600 Subject: [PATCH 2/5] chore(release): prepare 0.111.0 --- CHANGELOG.md | 9 +++++++++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc18e95e..90e50842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.111.0] — 2026-07-09 — repository-clustered paired inference + +### Added + +- Added `clusteredPairedBinary`, which pairs binary outcomes by work item, exposes every unmatched row, resamples whole repositories for a task-weighted confidence interval, and tests the same effect with whole-repository sign flips. +- Added Holm step-down adjustment for strong family-wise error control across benchmark arms. + +The cluster interval is unavailable below two repositories, and consumers must reject unmatched rows before promotion. + ## [0.110.1] — 2026-07-09 — proposer portfolio export ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index e53c99cf..687c6007 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.110.1" +version = "0.111.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index c58c2fbd..5d63aab6 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.110.1" + __version__ = "0.111.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index b2e96c2c..5af1ec79 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "agent-eval-rpc" -version = "0.110.1" +version = "0.111.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/package.json b/package.json index 932f9e31..bb4041db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.110.1", + "version": "0.111.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { From 09625b21903e9817a749271488f8220a79cd8784 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:33:41 -0600 Subject: [PATCH 3/5] fix(statistics): correct clustered inference review artifacts --- .../2026-07-09T21-47-00Z/findings.jsonl | 3 --- .../2026-07-09T21-47-00Z/manifest.json | 19 ------------------- .../2026-07-09T21-47-00Z/summary.md | 16 ---------------- .../2026-07-09T21-54-37Z/findings.jsonl | 3 --- .../2026-07-09T21-54-37Z/manifest.json | 17 ----------------- .../2026-07-09T21-54-37Z/summary.md | 14 -------------- src/statistics.ts | 5 +++-- tests/clustered-paired-binary.test.ts | 10 +++++++++- 8 files changed, 12 insertions(+), 75 deletions(-) delete mode 100644 .evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl delete mode 100644 .evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json delete mode 100644 .evolve/critical-audit/2026-07-09T21-47-00Z/summary.md delete mode 100644 .evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl delete mode 100644 .evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json delete mode 100644 .evolve/critical-audit/2026-07-09T21-54-37Z/summary.md diff --git a/.evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl deleted file mode 100644 index 4a881cf5..00000000 --- a/.evolve/critical-audit/2026-07-09T21-47-00Z/findings.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"severity":"medium","file":"src/clustered-paired-binary.ts","line":366,"issue":"The sign-flip p-value tested the equal-cluster mean while the headline interval measured task-weighted risk difference, allowing contradictory significance claims for different estimands.","action":"Sign-flip whole-cluster outcome totals and divide by total matched pairs so the test statistic equals taskWeightedRiskDifference.","verification":"The 30 one-task positive plus 10 two-task negative repository construction must report RD 0.20 and a one-sided p-value above 0.05, not the pre-fix 0.00118."} -{"severity":"medium","file":"src/clustered-paired-binary.ts","line":180,"issue":"A single independent cluster produced a nominal zero-width 95% cluster-bootstrap interval, presenting unestimable cluster uncertainty as certainty.","action":"Return bootstrap null when fewer than two matched clusters exist while retaining the observed descriptive effect and sign-flip result.","verification":"A single improving repository must return taskWeightedRiskDifference 1 and bootstrap null."} -{"severity":"medium","file":"src/clustered-paired-binary.ts","line":228,"issue":"The API accepted one bootstrap draw, which could produce a nominal 95% interval [1,1] from two opposing repositories and falsely satisfy a lower-bound-above-zero promotion rule.","action":"Require at least ceil(2/(1-confidence)) draws so both requested interval tails are representable.","verification":"At confidence 0.95, bootstrapResamples 1 must fail with a minimum of 40 before sampling."} diff --git a/.evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json b/.evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json deleted file mode 100644 index 205955ce..00000000 --- a/.evolve/critical-audit/2026-07-09T21-47-00Z/manifest.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "scope": [ - "src/clustered-paired-binary.ts", - "src/statistics.ts", - "src/index.ts", - "tests/clustered-paired-binary.test.ts", - ".claude/skills/agent-eval/SKILL.md" - ], - "base": "9c7fc3056b9effddda6a5b91f1467fd3dc1af225", - "head": "cf51ffd63b53daa66e2d8d8d8003ceec5123231d", - "project_type": "typescript-node", - "flags": ["diff-only"], - "findings_count_by_severity": { - "critical": 0, - "high": 0, - "medium": 3, - "low": 0 - } -} diff --git a/.evolve/critical-audit/2026-07-09T21-47-00Z/summary.md b/.evolve/critical-audit/2026-07-09T21-47-00Z/summary.md deleted file mode 100644 index d6eb09c3..00000000 --- a/.evolve/critical-audit/2026-07-09T21-47-00Z/summary.md +++ /dev/null @@ -1,16 +0,0 @@ -# Critical audit: clustered paired binary inference - -Score: 6/10. - -Three medium correctness findings block using this unreleased API as the DeepSWE promotion decision. - -## Fix plan - -1. Align the sign-flip statistic with the task-weighted interval. - Verification: reproduce the 40-repository unequal-size counterexample and require the p-value to test the reported `+0.20` effect. -2. Suppress cluster-bootstrap intervals below two independent repositories. - Verification: one improving repository returns `bootstrap: null`. -3. Reject a bootstrap draw count too small to represent both requested tails. - Verification: one draw at 95% confidence fails with minimum `40`. - -REQUEST_CHANGES — the original API could certify significance for a different effect and could fabricate certainty from one repository or one draw. diff --git a/.evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl deleted file mode 100644 index 84e20c1e..00000000 --- a/.evolve/critical-audit/2026-07-09T21-54-37Z/findings.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"severity":"medium","file":"src/clustered-paired-binary.ts","line":374,"issue":"Sign-flip estimand mismatch.","action":"Resolved by sign-flipping cluster totals for taskWeightedRiskDifference.","verification":"resolved: 40-repository counterexample now returns p=0.1413 for RD 0.20."} -{"severity":"medium","file":"src/clustered-paired-binary.ts","line":180,"issue":"One-cluster zero-width interval.","action":"Resolved by returning bootstrap null below two clusters.","verification":"resolved: single-repository adversarial test passes."} -{"severity":"medium","file":"src/clustered-paired-binary.ts","line":236,"issue":"Too few bootstrap draws accepted.","action":"Resolved by a confidence-dependent minimum draw count.","verification":"resolved: one draw at 95% fails with minimum 40."} diff --git a/.evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json b/.evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json deleted file mode 100644 index e74df4ed..00000000 --- a/.evolve/critical-audit/2026-07-09T21-54-37Z/manifest.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "priorRun": ".evolve/critical-audit/2026-07-09T21-47-00Z", - "scope": [ - "src/clustered-paired-binary.ts", - "tests/clustered-paired-binary.test.ts" - ], - "base": "9c7fc3056b9effddda6a5b91f1467fd3dc1af225", - "head": "cf51ffd63b53daa66e2d8d8d8003ceec5123231d", - "project_type": "typescript-node", - "flags": ["diff-only", "reaudit"], - "findings_count_by_severity": { - "critical": 0, - "high": 0, - "medium": 0, - "low": 0 - } -} diff --git a/.evolve/critical-audit/2026-07-09T21-54-37Z/summary.md b/.evolve/critical-audit/2026-07-09T21-54-37Z/summary.md deleted file mode 100644 index b2a018db..00000000 --- a/.evolve/critical-audit/2026-07-09T21-54-37Z/summary.md +++ /dev/null @@ -1,14 +0,0 @@ -# Critical re-audit: clustered paired binary inference - -Score: 8/10. - -All three medium findings are resolved. - -The task-weighted interval and sign-flip test now answer the same question. -One repository reports no cluster interval. -Too few bootstrap draws fail before sampling. -The 40-repository counterexample, one-repository case, one-draw case, exact enumeration, Monte Carlo determinism, whole-cluster bootstrap, unpaired rows, and Holm adjustment are covered by executable tests. - -Residual requirement: consumers must reject unpaired rows before promotion, because the utility exposes them and intentionally computes complete-case statistics. - -APPROVE — no correctness blocker remains in the unreleased API. diff --git a/src/statistics.ts b/src/statistics.ts index d27e1ae6..7e8c70f6 100644 --- a/src/statistics.ts +++ b/src/statistics.ts @@ -858,11 +858,11 @@ export function holm( alpha = 0.05, ): { adjusted: number[]; significant: boolean[] } { if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 1) { - throw new Error(`holm: alpha must be in (0,1), got ${alpha}`) + throw new ValidationError(`holm: alpha must be in (0,1), got ${alpha}`) } for (const [index, pValue] of pValues.entries()) { if (!Number.isFinite(pValue) || pValue < 0 || pValue > 1) { - throw new Error(`holm: pValues[${index}] must be in [0,1], got ${pValue}`) + throw new ValidationError(`holm: pValues[${index}] must be in [0,1], got ${pValue}`) } } const count = pValues.length @@ -879,6 +879,7 @@ export function holm( previous = Math.max(previous, stepAdjusted) adjusted[entry.index] = previous } + // Holm's rejection rule is inclusive at the adjusted alpha boundary. return { adjusted, significant: adjusted.map((pValue) => pValue <= alpha) } } diff --git a/tests/clustered-paired-binary.test.ts b/tests/clustered-paired-binary.test.ts index e3187749..44f3b769 100644 --- a/tests/clustered-paired-binary.test.ts +++ b/tests/clustered-paired-binary.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest' -import { type ClusteredPairedBinaryOptions, clusteredPairedBinary, holm } from '../src/index' +import { + type ClusteredPairedBinaryOptions, + clusteredPairedBinary, + holm, + ValidationError, +} from '../src/index' interface BinaryRow { task: string @@ -295,6 +300,9 @@ describe('holm', () => { }) it('rejects invalid p-values and alpha', () => { + for (const call of [() => holm([0.1, Number.NaN]), () => holm([-0.1]), () => holm([0.1], 0)]) { + expect(call).toThrow(ValidationError) + } expect(() => holm([0.1, Number.NaN])).toThrow(/pValues\[1\] must be in \[0,1\]/) expect(() => holm([-0.1])).toThrow(/pValues\[0\] must be in \[0,1\]/) expect(() => holm([0.1], 0)).toThrow(/alpha must be in \(0,1\)/) From abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:34:49 -0600 Subject: [PATCH 4/5] fix(statistics): bound clustered resampling inputs --- src/clustered-paired-binary.ts | 16 +++++++---- tests/clustered-paired-binary.test.ts | 41 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/clustered-paired-binary.ts b/src/clustered-paired-binary.ts index 6cf0a8ff..0afa6a5c 100644 --- a/src/clustered-paired-binary.ts +++ b/src/clustered-paired-binary.ts @@ -32,7 +32,7 @@ export interface ClusteredPairedBinaryOptions { seed: number /** Percentile confidence level. Default 0.95. */ confidence?: number - /** Whole-cluster bootstrap draws. Default 10,000. */ + /** Whole-cluster bootstrap draws. Default 10,000; maximum 1,000,000. */ bootstrapResamples?: number /** Sign-flip alternative. Default 'two-sided'. */ alternative?: ClusterSignFlipAlternative @@ -41,7 +41,7 @@ export interface ClusteredPairedBinaryOptions { * otherwise use Monte Carlo. Default 20; maximum 20. */ exactClusterLimit?: number - /** Monte Carlo sign-flip draws when exact enumeration is not used. Default 100,000. */ + /** Monte Carlo sign-flip draws when exact enumeration is not used. Default 100,000; maximum 1,000,000. */ signFlipResamples?: number } @@ -127,6 +127,7 @@ interface ProjectedRow extends PairedArmRow { const DEFAULT_BOOTSTRAP_RESAMPLES = 10_000 const DEFAULT_SIGN_FLIP_RESAMPLES = 100_000 +const MAX_RESAMPLES = 1_000_000 const DEFAULT_EXACT_CLUSTER_LIMIT = 20 const SIGN_FLIP_SEED_SALT = 0x9e3779b9 @@ -231,7 +232,7 @@ function validateOptions(options: ClusteredPairedBinaryOptions): Val ) } const bootstrapResamples = options.bootstrapResamples ?? DEFAULT_BOOTSTRAP_RESAMPLES - assertPositiveInteger('bootstrapResamples', bootstrapResamples) + assertResampleCount('bootstrapResamples', bootstrapResamples) const rawMinimumBootstrapResamples = 2 / (1 - confidence) const minimumBootstrapResamples = Math.ceil( rawMinimumBootstrapResamples - Number.EPSILON * Math.max(1, rawMinimumBootstrapResamples) * 8, @@ -242,7 +243,7 @@ function validateOptions(options: ClusteredPairedBinaryOptions): Val ) } const signFlipResamples = options.signFlipResamples ?? DEFAULT_SIGN_FLIP_RESAMPLES - assertPositiveInteger('signFlipResamples', signFlipResamples) + assertResampleCount('signFlipResamples', signFlipResamples) const exactClusterLimit = options.exactClusterLimit ?? DEFAULT_EXACT_CLUSTER_LIMIT if ( !Number.isInteger(exactClusterLimit) || @@ -269,12 +270,17 @@ function validateOptions(options: ClusteredPairedBinaryOptions): Val } } -function assertPositiveInteger(name: string, value: number): void { +function assertResampleCount(name: string, value: number): void { if (!Number.isInteger(value) || value <= 0) { throw new ValidationError( `clusteredPairedBinary: ${name} must be a positive integer, got ${value}`, ) } + if (value > MAX_RESAMPLES) { + throw new ValidationError( + `clusteredPairedBinary: ${name} must not exceed ${MAX_RESAMPLES}, got ${value}`, + ) + } } function projectSelectedRows( diff --git a/tests/clustered-paired-binary.test.ts b/tests/clustered-paired-binary.test.ts index 44f3b769..06965819 100644 --- a/tests/clustered-paired-binary.test.ts +++ b/tests/clustered-paired-binary.test.ts @@ -134,6 +134,41 @@ describe('clusteredPairedBinary', () => { }) }) + it('supports the less alternative with the mirrored exact probability', () => { + const rows: BinaryRow[] = [] + for (let index = 0; index < 3; index++) { + rows.push(row(`task-${index}`, `cluster-${index}`, 'baseline', true)) + rows.push(row(`task-${index}`, `cluster-${index}`, 'candidate', false)) + } + + const result = clusteredPairedBinary(rows, options({ alternative: 'less' })) + + expect(result.statistics!.signFlip).toMatchObject({ + statistic: -1, + pValue: 1 / 8, + alternative: 'less', + method: 'exact', + }) + }) + + it('pairs multiple replicates by repKey before aggregating their cluster', () => { + const rows = [ + { ...row('task', 'cluster', 'baseline', true), rep: 'b' }, + { ...row('task', 'cluster', 'candidate', false), rep: 'b' }, + { ...row('task', 'cluster', 'baseline', false), rep: 'a' }, + { ...row('task', 'cluster', 'candidate', true), rep: 'a' }, + ] + + const result = clusteredPairedBinary(rows, options()) + + expect(result.matchedPairs.map((pair) => pair.repIndex)).toEqual([0, 1]) + expect(result.matchedPairs.map((pair) => [pair.baseline.rep, pair.treatment.rep])).toEqual([ + ['a', 'a'], + ['b', 'b'], + ]) + expect(result.statistics).toMatchObject({ nPairs: 2, nClusters: 1, b10: 1, b01: 1 }) + }) + it('uses deterministic Monte Carlo sign flips above the exact limit', () => { const rows: BinaryRow[] = [] for (let index = 0; index < 8; index++) { @@ -265,6 +300,12 @@ describe('clusteredPairedBinary', () => { expect(() => clusteredPairedBinary(rows, options({ bootstrapResamples: 0 }))).toThrow( /bootstrapResamples must be a positive integer/, ) + expect(() => clusteredPairedBinary(rows, options({ bootstrapResamples: 1_000_001 }))).toThrow( + /bootstrapResamples must not exceed 1000000/, + ) + expect(() => clusteredPairedBinary(rows, options({ signFlipResamples: 1_000_001 }))).toThrow( + /signFlipResamples must not exceed 1000000/, + ) expect(() => clusteredPairedBinary(rows, options({ exactClusterLimit: 21 }))).toThrow( /exactClusterLimit must be an integer in \[0,20\]/, ) From 7dad88249343acdf9966a225f07b13cb24cf2dbd Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:36:13 -0600 Subject: [PATCH 5/5] chore(audit): record clustered inference review provenance --- .../2026-07-09T22-35-41Z/findings.jsonl | 1 + .../2026-07-09T22-35-41Z/manifest.json | 29 +++++++++++++++++++ .../2026-07-09T22-35-41Z/summary.md | 20 +++++++++++++ .evolve/skill-runs.jsonl | 1 + 4 files changed, 51 insertions(+) create mode 100644 .evolve/critical-audit/2026-07-09T22-35-41Z/findings.jsonl create mode 100644 .evolve/critical-audit/2026-07-09T22-35-41Z/manifest.json create mode 100644 .evolve/critical-audit/2026-07-09T22-35-41Z/summary.md diff --git a/.evolve/critical-audit/2026-07-09T22-35-41Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T22-35-41Z/findings.jsonl new file mode 100644 index 00000000..e6964c8c --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T22-35-41Z/findings.jsonl @@ -0,0 +1 @@ +{"severity":"LOW","status":"accepted-risk","file":"src/clustered-paired-binary.ts","line":134,"issue":"The implementation is covered by exact synthetic cases and adversarial counterexamples but has no checked-in parity vector from an independent statistics package.","action":"Before a paper claim depends on this function, add one offline golden vector produced by an independent implementation and pin every reported statistic.","verification":"The new test should compare pair counts, both effect estimates, the interval, and the randomization p-value against the frozen external vector."} diff --git a/.evolve/critical-audit/2026-07-09T22-35-41Z/manifest.json b/.evolve/critical-audit/2026-07-09T22-35-41Z/manifest.json new file mode 100644 index 00000000..d09480f0 --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T22-35-41Z/manifest.json @@ -0,0 +1,29 @@ +{ + "scope": [ + "src/clustered-paired-binary.ts", + "src/statistics.ts", + "src/index.ts", + "tests/clustered-paired-binary.test.ts", + ".claude/skills/agent-eval/SKILL.md" + ], + "base": "39e8002748f43aa9b440805e7c73000d8aba1a19", + "head": "abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1", + "project_type": "typescript-node", + "flags": ["--diff-only"], + "score": "8/10", + "verdict": "APPROVE", + "findings_count_by_severity": { + "CRITICAL": 0, + "HIGH": 0, + "MEDIUM": 0, + "LOW": 1 + }, + "verification": [ + "pnpm test: 270 files, 2752 passed, 2 skipped", + "pnpm typecheck", + "pnpm build", + "pnpm verify:package", + "pnpm lint: 0 errors, 4 pre-existing warnings", + "git diff --check" + ] +} diff --git a/.evolve/critical-audit/2026-07-09T22-35-41Z/summary.md b/.evolve/critical-audit/2026-07-09T22-35-41Z/summary.md new file mode 100644 index 00000000..b6234b1a --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T22-35-41Z/summary.md @@ -0,0 +1,20 @@ +# Critical audit: clustered paired binary inference + +The implementation is correct for the reviewed contract, the previous untrustworthy audit records are gone, and the exact source commit audited here is `abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1`. + +## Finding + +- LOW — `src/clustered-paired-binary.ts:134`: no checked-in parity vector from an independent statistics package. + Action: add one frozen external comparison before this function supports a paper claim. + Verification: pin every output dimension against the independent vector. + +## Checks + +- `pnpm test`: 2,752 passed and 2 skipped across 270 files. +- `pnpm typecheck`: passed. +- `pnpm build`: passed. +- `pnpm verify:package`: passed. +- `pnpm lint`: zero errors and four pre-existing warnings. +- `git diff --check`: passed. + +APPROVE — no critical, high, or incident-causing medium finding remains. diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index ba0524d7..e2586c67 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -16,3 +16,4 @@ {"skill":"/research","ts":"2026-06-05T13:21:01Z","project":"agent-eval-belief-roadmap","target":"belief-state-code-agent-corpus","operatorPrompt":"","durationMin":null,"verdict":"SHIP","dispatchedTo":"stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"critical-audit","ts":"2026-06-21T12:57:05Z","project":"agent-eval","target":"PR #264 review comment 4762012574","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"1 medium, 9 low; 8 fixed, 1 not_actionable, 1 deferred","files":[".gitignore","CHANGELOG.md","src/agent-profile.ts","src/agent-profile.test.ts","src/contract/define-agent-eval.ts","src/contract/index.ts","tests/contract-define-agent-eval.test.ts"]} {"skill":"critical-audit","ts":"2026-07-09T21:54:37Z","project":"agent-eval","target":"clustered paired binary inference for repository-disjoint coding benchmarks","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"3 medium found and resolved: estimand mismatch, one-cluster interval, insufficient bootstrap draws","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} +{"skill":"critical-audit","ts":"2026-07-09T22:35:41Z","project":"agent-eval","target":"clustered paired binary inference at abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1","verdict":"APPROVE","score":"8/10","findings":"1 low accepted risk: independent-package parity vector remains before paper use","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]}