Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 26 additions & 4 deletions src/campaign/presets/run-optimization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@ export interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifac
* generation's traces; findings-grounded proposers consume them. Opaque here;
* the proposer types its `TFindings`. Empty when no producer is wired. */
findings?: unknown[]
/** Per-generation findings producer. After each
* generation's candidates are scored, this is called with that generation's
* results; whatever it returns REPLACES `ctx.findings` for the NEXT
* generation's `propose()`, so the diagnosis is refreshed each round instead
/** Per-generation findings producer. Runs once on the BASELINE campaign
* (as `generation: -1`, the baseline convention) before generation 0
* proposes — so even a single-generation run proposes with trace context —
* and then after each generation's candidates are scored with that
* generation's results; whatever it returns REPLACES `ctx.findings` for the
* NEXT `propose()`, so the diagnosis is refreshed each round instead
* of being a static one-shot. Generic by design: the substrate does not
* import an analyst — the consumer plugs its trace-analyst registry / HALO
* here (reading the per-candidate `runDir` traces). When absent, findings
Expand Down Expand Up @@ -151,6 +153,26 @@ export async function runOptimization<TScenario extends Scenario, TArtifact>(
toParetoParent(opts.baselineSurface, winnerSurfaceHash, baselineCampaign, -1),
]

// Diagnose the BASELINE traces before generation 0 proposes. The
// between-generation producer call below only fires after gen g to feed gen
// g+1, so without this a single-generation run (maxGenerations = 1)
// proposes blind even though baseline traces exist. Baseline is
// `generation: -1` — the same convention the Pareto accumulator uses above.
// Skipped when the baseline produced no cells (dry/offline modes have no
// traces to analyze) or there is no generation 0 to feed; `propose()` then
// sees the static seed findings exactly as before.
if (opts.analyzeGeneration && opts.maxGenerations > 0 && baselineCampaign.cells.length > 0) {
const fresh = await opts.analyzeGeneration({
generation: -1,
runDir: `${opts.runDir}/baseline`,
candidates: [
{ surfaceHash: winnerSurfaceHash, campaign: baselineCampaign, composite: winnerComposite },
],
history,
})
if (Array.isArray(fresh)) currentFindings = fresh
}

for (let gen = 0; gen < opts.maxGenerations; gen++) {
// Decide: the proposer may stop early based on accumulated history.
if (proposer.decide?.({ history }).stop) break
Expand Down
11 changes: 6 additions & 5 deletions src/contract/self-improve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,11 +239,12 @@ export interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
expectUsage?: 'assert' | 'warn' | 'off'

/**
* Per-generation findings producer. After each
* generation is scored, this is called; whatever it returns REPLACES the
* proposer's `findings` for the next generation's `propose()`. Plug a
* trace-analyst registry / HALO here. When absent, findings stay
* `opts.findings`.
* Per-generation findings producer. Runs once on the baseline campaign (as
* `generation: -1`) before generation 0 proposes — so single-generation runs
* propose with trace context — and again after each generation is scored;
* whatever it returns REPLACES the proposer's `findings` for the next
* `propose()`. Plug a trace-analyst registry / HALO here. When absent,
* findings stay `opts.findings`.
*/
analyzeGeneration?: RunOptimizationOptions<TScenario, TArtifact>['analyzeGeneration']

Expand Down
89 changes: 85 additions & 4 deletions tests/campaign/presets.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1414,12 +1414,93 @@ describe('runOptimization — analyzeGeneration feeds findings forward', () => {
})

expect(seen).toHaveLength(3)
// Gen 0 sees the static seed; gen 1 sees gen-0's produced finding; gen 2 gen-1's.
expect(seen[0]).toEqual([{ claim: 'seed' }])
// Gen 0 sees the baseline (gen -1) analysis — not the static seed; gen 1
// sees gen-0's produced finding; gen 2 gen-1's.
expect(seen[0]).toEqual([{ claim: 'gen--1 finding' }])
expect(seen[1]).toEqual([{ claim: 'gen-0 finding' }])
expect(seen[2]).toEqual([{ claim: 'gen-1 finding' }])
// Producer runs after gens 0 and 1, NOT the last (gen 2 has no next propose()).
expect(analyzed).toEqual([0, 1])
// Producer runs on the baseline (-1) and after gens 0 and 1, NOT the last
// (gen 2 has no next propose()).
expect(analyzed).toEqual([-1, 0, 1])
})

it("with generations=1, gen 0's propose() receives the baseline (gen -1) trace analysis", async () => {
const seen: unknown[][] = []
const proposer = {
kind: 'recorder',
async propose(ctx: ProposeContext) {
seen.push(ctx.findings)
return [{ surface: 'S-gen0', label: 'g0', rationale: 'r' }]
},
}
const result = await runOptimization<FakeScenario, FakeArtifact>({
scenarios: SCENARIOS,
baselineSurface: 'BASE',
dispatchWithSurface: async (surface) => ({ text: String(surface) }),
judges: [passJudge],
proposer,
populationSize: 1,
maxGenerations: 1,
promoteTopK: 1,
runDir,
seed: 1,
findings: [{ claim: 'seed' }],
analyzeGeneration: async ({ generation, runDir: analyzedDir, candidates, history }) => {
// The single propose() of the run is fed by a BASELINE analysis:
// generation -1 (the baseline convention), the baseline runDir, no
// history yet, and the baseline campaign as the sole candidate.
expect(generation).toBe(-1)
expect(analyzedDir).toBe(`${runDir}/baseline`)
expect(history).toEqual([])
expect(candidates).toHaveLength(1)
expect(candidates[0]!.surfaceHash).toBe(surfaceHash('BASE'))
// Marker derived from the baseline traces: the analyzed cells carry
// the artifact the BASELINE surface dispatched.
const artifacts = candidates[0]!.campaign.cells.map(
(c) => (c.artifact as FakeArtifact).text,
)
expect(artifacts).toEqual(['BASE', 'BASE'])
return [{ claim: 'baseline finding', from: artifacts.join(',') }]
},
})

// Gen 0 proposed WITH the baseline analysis (not the static seed), and the
// report is traceably derived from the baseline artifacts.
expect(seen).toEqual([[{ claim: 'baseline finding', from: 'BASE,BASE' }]])
expect(result.generations).toHaveLength(1)
})

it('skips the baseline analysis when the baseline produced no cells (nothing to analyze)', async () => {
const seen: unknown[][] = []
const analyzed: number[] = []
const proposer = {
kind: 'recorder',
async propose(ctx: ProposeContext) {
seen.push(ctx.findings)
return [{ surface: 'S-gen0', label: 'g0', rationale: 'r' }]
},
}
await runOptimization<FakeScenario, FakeArtifact>({
// No scenarios → the baseline campaign has zero cells (the dry shape) —
// the producer must NOT run on it, and propose() sees the static seed.
scenarios: [],
baselineSurface: 'BASE',
dispatchWithSurface: async (surface) => ({ text: String(surface) }),
judges: [passJudge],
proposer,
populationSize: 1,
maxGenerations: 1,
promoteTopK: 1,
runDir,
seed: 1,
findings: [{ claim: 'seed' }],
analyzeGeneration: async ({ generation }) => {
analyzed.push(generation)
return [{ claim: 'should not reach gen 0' }]
},
})
expect(analyzed).toEqual([])
expect(seen).toEqual([[{ claim: 'seed' }]])
})

it('without analyzeGeneration, findings stay the static seed every generation', async () => {
Expand Down
19 changes: 17 additions & 2 deletions tests/contract-self-improve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,6 @@ describe('selfImprove — forwarded loop knobs', () => {
let calls = 0
await selfImprove<S, A>({
...base,
// analyzeGeneration runs BETWEEN generations (its findings feed the next),
// so it needs ≥2 generations to fire at least once.
budget: { generations: 2, populationSize: 1 },
expectUsage: 'off',
analyzeGeneration: async () => {
Expand All @@ -221,4 +219,21 @@ describe('selfImprove — forwarded loop knobs', () => {
})
expect(calls).toBeGreaterThanOrEqual(1)
})

it('with generations=1, analyzeGeneration fires on the BASELINE (gen -1) so propose() is not blind', async () => {
const analyzed: number[] = []
await selfImprove<S, A>({
...base,
// Single generation — the common case. The producer analyzes the
// baseline traces first (gen -1) so gen 0 proposes with that report;
// the between-generation call is skipped (gen 0 is the last).
budget: { generations: 1, populationSize: 1 },
expectUsage: 'off',
analyzeGeneration: async ({ generation }) => {
analyzed.push(generation)
return [{ claim: 'baseline finding' }]
},
})
expect(analyzed).toEqual([-1])
})
})
Loading