From a5a0f3545e87d884b03f0c63bc17d22c112c827b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 14:29:11 -0600 Subject: [PATCH] fix(improvement): analyze baseline traces before generation 0 so single-generation runs propose with trace context --- src/campaign/presets/run-optimization.ts | 30 ++++++-- src/contract/self-improve.ts | 11 +-- tests/campaign/presets.test.ts | 89 ++++++++++++++++++++++-- tests/contract-self-improve.test.ts | 19 ++++- 4 files changed, 134 insertions(+), 15 deletions(-) diff --git a/src/campaign/presets/run-optimization.ts b/src/campaign/presets/run-optimization.ts index 1d6e314d..96562b8c 100644 --- a/src/campaign/presets/run-optimization.ts +++ b/src/campaign/presets/run-optimization.ts @@ -59,10 +59,12 @@ export interface RunOptimizationBaseOptions( toParetoParent(opts.baselineSurface, winnerSurfaceHash, baselineCampaign, -1), ] + // Diagnose the BASELINE traces before generation 0 proposes. The + // between-generation producer call below only fires after gen g to feed gen + // g+1, so without this a single-generation run (maxGenerations = 1) + // proposes blind even though baseline traces exist. Baseline is + // `generation: -1` — the same convention the Pareto accumulator uses above. + // Skipped when the baseline produced no cells (dry/offline modes have no + // traces to analyze) or there is no generation 0 to feed; `propose()` then + // sees the static seed findings exactly as before. + if (opts.analyzeGeneration && opts.maxGenerations > 0 && baselineCampaign.cells.length > 0) { + const fresh = await opts.analyzeGeneration({ + generation: -1, + runDir: `${opts.runDir}/baseline`, + candidates: [ + { surfaceHash: winnerSurfaceHash, campaign: baselineCampaign, composite: winnerComposite }, + ], + history, + }) + if (Array.isArray(fresh)) currentFindings = fresh + } + for (let gen = 0; gen < opts.maxGenerations; gen++) { // Decide: the proposer may stop early based on accumulated history. if (proposer.decide?.({ history }).stop) break diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 2e5137c7..3be248f2 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -239,11 +239,12 @@ export interface SelfImproveOptions { expectUsage?: 'assert' | 'warn' | 'off' /** - * Per-generation findings producer. After each - * generation is scored, this is called; whatever it returns REPLACES the - * proposer's `findings` for the next generation's `propose()`. Plug a - * trace-analyst registry / HALO here. When absent, findings stay - * `opts.findings`. + * Per-generation findings producer. Runs once on the baseline campaign (as + * `generation: -1`) before generation 0 proposes — so single-generation runs + * propose with trace context — and again after each generation is scored; + * whatever it returns REPLACES the proposer's `findings` for the next + * `propose()`. Plug a trace-analyst registry / HALO here. When absent, + * findings stay `opts.findings`. */ analyzeGeneration?: RunOptimizationOptions['analyzeGeneration'] diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index deaf97b1..fd080065 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -1414,12 +1414,93 @@ describe('runOptimization — analyzeGeneration feeds findings forward', () => { }) expect(seen).toHaveLength(3) - // Gen 0 sees the static seed; gen 1 sees gen-0's produced finding; gen 2 gen-1's. - expect(seen[0]).toEqual([{ claim: 'seed' }]) + // Gen 0 sees the baseline (gen -1) analysis — not the static seed; gen 1 + // sees gen-0's produced finding; gen 2 gen-1's. + expect(seen[0]).toEqual([{ claim: 'gen--1 finding' }]) expect(seen[1]).toEqual([{ claim: 'gen-0 finding' }]) expect(seen[2]).toEqual([{ claim: 'gen-1 finding' }]) - // Producer runs after gens 0 and 1, NOT the last (gen 2 has no next propose()). - expect(analyzed).toEqual([0, 1]) + // Producer runs on the baseline (-1) and after gens 0 and 1, NOT the last + // (gen 2 has no next propose()). + expect(analyzed).toEqual([-1, 0, 1]) + }) + + it("with generations=1, gen 0's propose() receives the baseline (gen -1) trace analysis", async () => { + const seen: unknown[][] = [] + const proposer = { + kind: 'recorder', + async propose(ctx: ProposeContext) { + seen.push(ctx.findings) + return [{ surface: 'S-gen0', label: 'g0', rationale: 'r' }] + }, + } + const result = await runOptimization({ + scenarios: SCENARIOS, + baselineSurface: 'BASE', + dispatchWithSurface: async (surface) => ({ text: String(surface) }), + judges: [passJudge], + proposer, + populationSize: 1, + maxGenerations: 1, + promoteTopK: 1, + runDir, + seed: 1, + findings: [{ claim: 'seed' }], + analyzeGeneration: async ({ generation, runDir: analyzedDir, candidates, history }) => { + // The single propose() of the run is fed by a BASELINE analysis: + // generation -1 (the baseline convention), the baseline runDir, no + // history yet, and the baseline campaign as the sole candidate. + expect(generation).toBe(-1) + expect(analyzedDir).toBe(`${runDir}/baseline`) + expect(history).toEqual([]) + expect(candidates).toHaveLength(1) + expect(candidates[0]!.surfaceHash).toBe(surfaceHash('BASE')) + // Marker derived from the baseline traces: the analyzed cells carry + // the artifact the BASELINE surface dispatched. + const artifacts = candidates[0]!.campaign.cells.map( + (c) => (c.artifact as FakeArtifact).text, + ) + expect(artifacts).toEqual(['BASE', 'BASE']) + return [{ claim: 'baseline finding', from: artifacts.join(',') }] + }, + }) + + // Gen 0 proposed WITH the baseline analysis (not the static seed), and the + // report is traceably derived from the baseline artifacts. + expect(seen).toEqual([[{ claim: 'baseline finding', from: 'BASE,BASE' }]]) + expect(result.generations).toHaveLength(1) + }) + + it('skips the baseline analysis when the baseline produced no cells (nothing to analyze)', async () => { + const seen: unknown[][] = [] + const analyzed: number[] = [] + const proposer = { + kind: 'recorder', + async propose(ctx: ProposeContext) { + seen.push(ctx.findings) + return [{ surface: 'S-gen0', label: 'g0', rationale: 'r' }] + }, + } + await runOptimization({ + // No scenarios → the baseline campaign has zero cells (the dry shape) — + // the producer must NOT run on it, and propose() sees the static seed. + scenarios: [], + baselineSurface: 'BASE', + dispatchWithSurface: async (surface) => ({ text: String(surface) }), + judges: [passJudge], + proposer, + populationSize: 1, + maxGenerations: 1, + promoteTopK: 1, + runDir, + seed: 1, + findings: [{ claim: 'seed' }], + analyzeGeneration: async ({ generation }) => { + analyzed.push(generation) + return [{ claim: 'should not reach gen 0' }] + }, + }) + expect(analyzed).toEqual([]) + expect(seen).toEqual([[{ claim: 'seed' }]]) }) it('without analyzeGeneration, findings stay the static seed every generation', async () => { diff --git a/tests/contract-self-improve.test.ts b/tests/contract-self-improve.test.ts index b2518b96..9089c3ec 100644 --- a/tests/contract-self-improve.test.ts +++ b/tests/contract-self-improve.test.ts @@ -210,8 +210,6 @@ describe('selfImprove — forwarded loop knobs', () => { let calls = 0 await selfImprove({ ...base, - // analyzeGeneration runs BETWEEN generations (its findings feed the next), - // so it needs ≥2 generations to fire at least once. budget: { generations: 2, populationSize: 1 }, expectUsage: 'off', analyzeGeneration: async () => { @@ -221,4 +219,21 @@ describe('selfImprove — forwarded loop knobs', () => { }) expect(calls).toBeGreaterThanOrEqual(1) }) + + it('with generations=1, analyzeGeneration fires on the BASELINE (gen -1) so propose() is not blind', async () => { + const analyzed: number[] = [] + await selfImprove({ + ...base, + // Single generation — the common case. The producer analyzes the + // baseline traces first (gen -1) so gen 0 proposes with that report; + // the between-generation call is skipped (gen 0 is the last). + budget: { generations: 1, populationSize: 1 }, + expectUsage: 'off', + analyzeGeneration: async ({ generation }) => { + analyzed.push(generation) + return [{ claim: 'baseline finding' }] + }, + }) + expect(analyzed).toEqual([-1]) + }) })