feat(runtime): structuralRollout strategy + rollout-policy improve surface (measured +8..+21pp)

bench/src/benchmarks/humaneval.ts

@@ -17,7 +17,7 @@
  */
 
 import { execFile } from 'node:child_process'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { gunzipSync } from 'node:zlib'
@@ -41,9 +41,17 @@ export interface HumanEvalTask {
  *  selects a deeper slice (the later tasks are harder) so the worker has a
  *  correctable middle band rather than a saturated easy prefix. */
 export async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {
-  const res = await fetch(humanevalUrl)
-  if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
-  const gz = Buffer.from(await res.arrayBuffer())
+  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits
+  // (429) under repeated runs. Fall back to the network fetch when unset.
+  const localGz = process.env.HUMANEVAL_GZ
+  let gz: Buffer
+  if (localGz) {
+    gz = readFileSync(localGz)
+  } else {
+    const res = await fetch(humanevalUrl)
+    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)
+    gz = Buffer.from(await res.arrayBuffer())
+  }
   const text = gunzipSync(gz).toString('utf8')
   const tasks: HumanEvalTask[] = []
   for (const line of text.split('\n')) {

bench/src/hev-eval.mts

@@ -0,0 +1,69 @@
+/**
+ * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the
+ * worker model on each task and print the pass rate + the per-task result. Used to
+ * measure a baseline instruction vs a proposer-supplied instruction on the SAME
+ * held-out set (the proposer proposes; this grades — kept separate for honesty).
+ *
+ *   INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \
+ *     HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts
+ */
+import { readFileSync } from 'node:fs'
+import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+
+const SEED_INSTRUCTION =
+  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
+
+async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<string> {
+  const res = await fetch(`${base}/chat/completions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
+  })
+  if (!res.ok) return ''
+  const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  return d.choices?.[0]?.message?.content ?? ''
+}
+
+async function main(): Promise<void> {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY required')
+  const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
+  const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
+  const instruction = process.env.INSTRUCTION_FILE
+    ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8')
+    : (process.env.INSTRUCTION ?? SEED_INSTRUCTION)
+  const maxTokens = Number(process.env.MAX_TOKENS ?? 2500)
+  const conc = Number(process.env.CONC ?? 6)
+  const offset = Number(process.env.OFFSET ?? 55)
+  const n = Number(process.env.N ?? 40)
+  const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+
+  const all = await loadHumanEval(164, 0)
+  const byId = new Map(all.map((t) => [t.taskId, t]))
+  const tasks: HumanEvalTask[] = idsEnv.length
+    ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t)
+    : all.slice(offset, offset + n)
+
+  console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`)
+  let pass = 0
+  const fails: string[] = []
+  // simple concurrency pool
+  let i = 0
+  async function worker(): Promise<void> {
+    while (i < tasks.length) {
+      const t = tasks[i++]
+      const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const { pass: p } = await runChecker(t, extractCode(reply))
+      if (p === 1) pass += 1
+      else fails.push(t.taskId)
+    }
+  }
+  await Promise.all(Array.from({ length: conc }, () => worker()))
+  console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`)
+  console.log(`FAILED: ${fails.sort().join(', ')}`)
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

bench/src/hev-improve.mts

@@ -0,0 +1,168 @@
+/**
+ * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart
+ * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer +
+ * held-out gate), but the worker is a single chat completion and the judge is the
+ * deterministic Docker checker (run the function against its own hidden unit tests).
+ *
+ * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test
+ * is withheld — the worker cannot verify, so prompt wording cannot move resolve.
+ * HumanEval hands the worker a well-specified function to complete and grades by
+ * running tests, so the instruction prompt DOES move pass-rate. This run measures
+ * whether self-improvement lifts a CHEAP model when the task is prompt-sensitive.
+ *
+ * Worker + reflect models call the zai coding endpoint directly (no tangle router,
+ * no WAF, no 503):  TANGLE_API_KEY=$ZAI_API_KEY  ROUTER_BASE=https://api.z.ai/api/coding/paas/v4
+ */
+import { improve } from '@tangle-network/agent-runtime'
+import type { AgentProfile } from '@tangle-network/agent-interface'
+import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
+import { gepaProposer } from '@tangle-network/agent-eval/campaign'
+import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval'
+
+// The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's
+// solveInstruction so the baseline arm reproduces the plain-prompt denominator.
+const SEED_INSTRUCTION =
+  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'
+
+interface Completion {
+  text: string
+  usd: number
+  tokIn: number
+  tokOut: number
+}
+
+async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise<Completion> {
+  const res = await fetch(`${base}/chat/completions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }),
+  })
+  if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`)
+  const d = (await res.json()) as {
+    choices?: Array<{ message?: { content?: string } }>
+    usage?: { prompt_tokens?: number; completion_tokens?: number }
+  }
+  const text = d.choices?.[0]?.message?.content ?? ''
+  const tokIn = d.usage?.prompt_tokens ?? 0
+  const tokOut = d.usage?.completion_tokens ?? 0
+  // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the
+  // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is).
+  const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000
+  return { text, usd, tokIn, tokOut }
+}
+
+async function main(): Promise<void> {
+  const key = process.env.TANGLE_API_KEY
+  if (!key) throw new Error('TANGLE_API_KEY required (worker + reflect completions)')
+  const base = process.env.ROUTER_BASE ?? 'https://api.z.ai/api/coding/paas/v4'
+  const workerModel = process.env.WORKER_MODEL ?? 'glm-4.5-air'
+  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
+  // The GEPA reflector may live on a DIFFERENT endpoint than the (cheap) worker —
+  // e.g. a small worker on Together + a strong optimizer on zai. Defaults to the
+  // worker endpoint when unset.
+  const reflectBase = process.env.REFLECT_BASE ?? base
+  const reflectKey = process.env.REFLECT_KEY ?? key
+  const trainN = Number(process.env.TRAIN_N ?? 12)
+  const holdoutN = Number(process.env.HOLDOUT_N ?? 12)
+  const offset = Number(process.env.OFFSET ?? 80)
+  const generations = Number(process.env.GENERATIONS ?? 1)
+  const population = Number(process.env.POPULATION ?? 2)
+  const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000)
+  const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000)
+  const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4)
+
+  // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset).
+  const train = await loadHumanEval(trainN, offset)
+  const holdout = await loadHumanEval(holdoutN, offset + trainN)
+  const byId = new Map<string, HumanEvalTask>([...train, ...holdout].map((t) => [t.taskId, t]))
+  const allIds = [...byId.keys()]
+
+  console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══')
+  console.log(`worker=${workerModel}  reflect=${reflectModel}  base=${base}`)
+  console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`)
+  console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`)
+  console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`)
+  console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`)
+
+  const stats = { n: 0 }
+  const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise<string | null> => {
+    const instr = String(surface)
+    const t = byId.get(scenario.id)
+    if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`)
+    const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\``
+    const t0 = Date.now()
+    const r = await complete(base, key, workerModel, prompt, workerMaxTokens)
+    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
+    const hasText = r.text.trim().length > 0
+    ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
+    ctx.cost.observeTokens(
+      zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
+    )
+    stats.n += 1
+    const codeLen = extractCode(r.text).length
+    console.log(`  [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`)
+    return hasText ? r.text : null
+  }
+
+  const judge: JudgeConfig<string, Scenario> = {
+    name: 'humaneval-docker',
+    dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }],
+    async score({ artifact, scenario }) {
+      const t = byId.get(scenario.id)
+      if (!t) throw new Error(`judge: unknown scenario ${scenario.id}`)
+      const code = extractCode(String(artifact ?? ''))
+      if (!code.trim()) {
+        console.log(`  [judge] ${scenario.id} pass=0 (empty)`)
+        return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' }
+      }
+      const { pass } = await runChecker(t, code)
+      console.log(`  [judge] ${scenario.id} pass=${pass}`)
+      return { dimensions: { pass }, composite: pass, notes: pass === 1 ? 'passed' : 'failed' }
+    },
+  }
+
+  const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } }
+  const proposer = gepaProposer({
+    llm: { baseUrl: reflectBase, apiKey: reflectKey },
+    model: reflectModel,
+    target:
+      'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' +
+      'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' +
+      'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' +
+      'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' +
+      'Bold rewrites that change model BEHAVIOR beat cosmetic edits.',
+    maxTokens: reflectMaxTokens,
+    temperature: 0.7,
+  })
+
+  const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' }))
+  const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' }))
+
+  const out = await improve(profile, [], {
+    surface: 'prompt',
+    gate: 'holdout',
+    generator: proposer,
+    scenarios,
+    judge,
+    agent,
+    expectUsage: 'warn',
+    budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 },
+    llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel },
+  })
+
+  console.log('\n═══ RESULT ═══')
+  console.log(`gateDecision=${out.gateDecision}  shipped=${out.shipped}  lift=${out.lift}`)
+  console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`)
+  console.log(`winner   holdout pass-rate = ${out.raw.winner.compositeMean}`)
+  console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
+  console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
+  if (out.raw.winner.label) console.log(`winner label   : ${out.raw.winner.label}`)
+  if ((out.raw.winner as { surface?: unknown }).surface) {
+    console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`)
+  }
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})