diff --git a/bench/src/benchmarks/humaneval.ts b/bench/src/benchmarks/humaneval.ts index 619d11f7..7a03bad7 100644 --- a/bench/src/benchmarks/humaneval.ts +++ b/bench/src/benchmarks/humaneval.ts @@ -17,7 +17,7 @@ */ import { execFile } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { gunzipSync } from 'node:zlib' @@ -41,9 +41,17 @@ export interface HumanEvalTask { * selects a deeper slice (the later tasks are harder) so the worker has a * correctable middle band rather than a saturated easy prefix. */ export async function loadHumanEval(limit: number, offset = 0): Promise { - const res = await fetch(humanevalUrl) - if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`) - const gz = Buffer.from(await res.arrayBuffer()) + // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits + // (429) under repeated runs. Fall back to the network fetch when unset. + const localGz = process.env.HUMANEVAL_GZ + let gz: Buffer + if (localGz) { + gz = readFileSync(localGz) + } else { + const res = await fetch(humanevalUrl) + if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`) + gz = Buffer.from(await res.arrayBuffer()) + } const text = gunzipSync(gz).toString('utf8') const tasks: HumanEvalTask[] = [] for (const line of text.split('\n')) { diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts new file mode 100644 index 00000000..dfba923a --- /dev/null +++ b/bench/src/hev-eval.mts @@ -0,0 +1,69 @@ +/** + * Minimal HumanEval evaluator: given an INSTRUCTION (env) + a fixed task set, run the + * worker model on each task and print the pass rate + the per-task result. Used to + * measure a baseline instruction vs a proposer-supplied instruction on the SAME + * held-out set (the proposer proposes; this grades — kept separate for honesty). + * + * INSTRUCTION="..." IDS=HumanEval/55,... WORKER_MODEL=... ROUTER_BASE=... TANGLE_API_KEY=... \ + * HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz tsx src/hev-eval.mts + */ +import { readFileSync } from 'node:fs' +import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' + +const SEED_INSTRUCTION = + 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' + +async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { + const res = await fetch(`${base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), + }) + if (!res.ok) return '' + const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } + return d.choices?.[0]?.message?.content ?? '' +} + +async function main(): Promise { + const key = process.env.TANGLE_API_KEY + if (!key) throw new Error('TANGLE_API_KEY required') + const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' + const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + const instruction = process.env.INSTRUCTION_FILE + ? readFileSync(process.env.INSTRUCTION_FILE, 'utf8') + : (process.env.INSTRUCTION ?? SEED_INSTRUCTION) + const maxTokens = Number(process.env.MAX_TOKENS ?? 2500) + const conc = Number(process.env.CONC ?? 6) + const offset = Number(process.env.OFFSET ?? 55) + const n = Number(process.env.N ?? 40) + const idsEnv = (process.env.IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean) + + const all = await loadHumanEval(164, 0) + const byId = new Map(all.map((t) => [t.taskId, t])) + const tasks: HumanEvalTask[] = idsEnv.length + ? idsEnv.map((id) => byId.get(id)).filter((t): t is HumanEvalTask => !!t) + : all.slice(offset, offset + n) + + console.log(`eval model=${model} n=${tasks.length} instr_len=${instruction.length}`) + let pass = 0 + const fails: string[] = [] + // simple concurrency pool + let i = 0 + async function worker(): Promise { + while (i < tasks.length) { + const t = tasks[i++] + const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens) + const { pass: p } = await runChecker(t, extractCode(reply)) + if (p === 1) pass += 1 + else fails.push(t.taskId) + } + } + await Promise.all(Array.from({ length: conc }, () => worker())) + console.log(`PASS ${pass}/${tasks.length} = ${((100 * pass) / tasks.length).toFixed(1)}%`) + console.log(`FAILED: ${fails.sort().join(', ')}`) +} + +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts new file mode 100644 index 00000000..fd86b8f3 --- /dev/null +++ b/bench/src/hev-improve.mts @@ -0,0 +1,168 @@ +/** + * SELF-IMPROVEMENT on HumanEval — the prompt-sensitive, VISIBLE-ORACLE counterpart + * to the SWE-bench run. Same machinery (improve(surface:'prompt') + gepaProposer + + * held-out gate), but the worker is a single chat completion and the judge is the + * deterministic Docker checker (run the function against its own hidden unit tests). + * + * WHY this exists: on SWE-bench the same GEPA loop was NULL because the grading test + * is withheld — the worker cannot verify, so prompt wording cannot move resolve. + * HumanEval hands the worker a well-specified function to complete and grades by + * running tests, so the instruction prompt DOES move pass-rate. This run measures + * whether self-improvement lifts a CHEAP model when the task is prompt-sensitive. + * + * Worker + reflect models call the zai coding endpoint directly (no tangle router, + * no WAF, no 503): TANGLE_API_KEY=$ZAI_API_KEY ROUTER_BASE=https://api.z.ai/api/coding/paas/v4 + */ +import { improve } from '@tangle-network/agent-runtime' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import { gepaProposer } from '@tangle-network/agent-eval/campaign' +import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' + +// The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's +// solveInstruction so the baseline arm reproduces the plain-prompt denominator. +const SEED_INSTRUCTION = + 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' + +interface Completion { + text: string + usd: number + tokIn: number + tokOut: number +} + +async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { + const res = await fetch(`${base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), + }) + if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) + const d = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const text = d.choices?.[0]?.message?.content ?? '' + const tokIn = d.usage?.prompt_tokens ?? 0 + const tokOut = d.usage?.completion_tokens ?? 0 + // zai glm pricing is ~ $0.6/M in, $2.2/M out (coding plan); a rough cost tag so the + // stub-guard sees a real backend. Exact cost is not the metric (pass-rate is). + const usd = (tokIn * 0.6 + tokOut * 2.2) / 1_000_000 + return { text, usd, tokIn, tokOut } +} + +async function main(): Promise { + const key = process.env.TANGLE_API_KEY + if (!key) throw new Error('TANGLE_API_KEY required (worker + reflect completions)') + const base = process.env.ROUTER_BASE ?? 'https://api.z.ai/api/coding/paas/v4' + const workerModel = process.env.WORKER_MODEL ?? 'glm-4.5-air' + const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6' + // The GEPA reflector may live on a DIFFERENT endpoint than the (cheap) worker — + // e.g. a small worker on Together + a strong optimizer on zai. Defaults to the + // worker endpoint when unset. + const reflectBase = process.env.REFLECT_BASE ?? base + const reflectKey = process.env.REFLECT_KEY ?? key + const trainN = Number(process.env.TRAIN_N ?? 12) + const holdoutN = Number(process.env.HOLDOUT_N ?? 12) + const offset = Number(process.env.OFFSET ?? 80) + const generations = Number(process.env.GENERATIONS ?? 1) + const population = Number(process.env.POPULATION ?? 2) + const workerMaxTokens = Number(process.env.MAX_TOKENS ?? 6000) + const reflectMaxTokens = Number(process.env.REFLECT_MAX_TOKENS ?? 8000) + const maxConcurrency = Number(process.env.MAX_CONCURRENCY ?? 4) + + // TRAIN and HOLDOUT are DISJOINT slices of the harder middle band (offset). + const train = await loadHumanEval(trainN, offset) + const holdout = await loadHumanEval(holdoutN, offset + trainN) + const byId = new Map([...train, ...holdout].map((t) => [t.taskId, t])) + const allIds = [...byId.keys()] + + console.log('═══ HumanEval self-improvement — VISIBLE oracle (deterministic Docker checker) ═══') + console.log(`worker=${workerModel} reflect=${reflectModel} base=${base}`) + console.log(`train=[${train.map((t) => t.taskId).join(', ')}]`) + console.log(`holdout=[${holdout.map((t) => t.taskId).join(', ')}]`) + console.log(`generations=${generations} population=${population} offset=${offset} maxTokens=${workerMaxTokens}`) + console.log(`≈ ${trainN * (1 + generations * population) + 2 * holdoutN} cells (each = 1 completion + 1 Docker check)\n`) + + const stats = { n: 0 } + const agent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext): Promise => { + const instr = String(surface) + const t = byId.get(scenario.id) + if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) + const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\`` + const t0 = Date.now() + const r = await complete(base, key, workerModel, prompt, workerMaxTokens) + const zeroUsage = r.tokIn === 0 && r.tokOut === 0 + const hasText = r.text.trim().length > 0 + ctx.cost.observe(zeroUsage && hasText ? Math.max(r.usd, 0.0001) : r.usd, workerModel) + ctx.cost.observeTokens( + zeroUsage && hasText ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut }, + ) + stats.n += 1 + const codeLen = extractCode(r.text).length + console.log(` [agent] ${scenario.id} instr=${instr.length}c code=${codeLen}b tok=in:${r.tokIn}/out:${r.tokOut} ${Math.round((Date.now() - t0) / 1000)}s`) + return hasText ? r.text : null + } + + const judge: JudgeConfig = { + name: 'humaneval-docker', + dimensions: [{ key: 'pass', description: 'the completed function passes its hidden unit tests (deterministic Docker checker)' }], + async score({ artifact, scenario }) { + const t = byId.get(scenario.id) + if (!t) throw new Error(`judge: unknown scenario ${scenario.id}`) + const code = extractCode(String(artifact ?? '')) + if (!code.trim()) { + console.log(` [judge] ${scenario.id} pass=0 (empty)`) + return { dimensions: { pass: 0 }, composite: 0, notes: 'empty' } + } + const { pass } = await runChecker(t, code) + console.log(` [judge] ${scenario.id} pass=${pass}`) + return { dimensions: { pass }, composite: pass, notes: pass === 1 ? 'passed' : 'failed' } + }, + } + + const profile: AgentProfile = { name: 'hev-solver', prompt: { systemPrompt: SEED_INSTRUCTION } } + const proposer = gepaProposer({ + llm: { baseUrl: reflectBase, apiKey: reflectKey }, + model: reflectModel, + target: + 'the instruction/system prompt strategy for a SMALL model completing Python functions to pass hidden unit tests. ' + + 'Propose SUBSTANTIALLY different strategies, not wording tweaks: e.g. require the model to first reason step-by-step ' + + 'about the algorithm and edge cases (empty inputs, off-by-one, boundary values, types) in a brief plan or comments ' + + 'BEFORE writing the code; provide a short worked example; or add an explicit self-check against the docstring. ' + + 'Bold rewrites that change model BEHAVIOR beat cosmetic edits.', + maxTokens: reflectMaxTokens, + temperature: 0.7, + }) + + const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'humaneval' })) + const holdoutScenarios: Scenario[] = holdout.map((t) => ({ id: t.taskId, kind: 'humaneval' })) + + const out = await improve(profile, [], { + surface: 'prompt', + gate: 'holdout', + generator: proposer, + scenarios, + judge, + agent, + expectUsage: 'warn', + budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency, reps: 1 }, + llm: { baseUrl: reflectBase, apiKey: reflectKey, model: reflectModel }, + }) + + console.log('\n═══ RESULT ═══') + console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`) + console.log(`baseline holdout pass-rate = ${out.raw.baseline.compositeMean}`) + console.log(`winner holdout pass-rate = ${out.raw.winner.compositeMean}`) + console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`) + console.log(`winner per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`) + if (out.raw.winner.label) console.log(`winner label : ${out.raw.winner.label}`) + if ((out.raw.winner as { surface?: unknown }).surface) { + console.log(`winner instruction:\n${String((out.raw.winner as { surface?: unknown }).surface).slice(0, 1200)}`) + } +} + +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/src/hev-structural.mts b/bench/src/hev-structural.mts new file mode 100644 index 00000000..ed2d1585 --- /dev/null +++ b/bench/src/hev-structural.mts @@ -0,0 +1,688 @@ +/** + * HumanEval STRUCTURAL lever — best-of-k selection + self-repair grounded ONLY on the + * VISIBLE docstring `>>>` examples (the honest oracle), graded on the HIDDEN check() + * suite. This is the experiment the self-improvement push identified but never ran: + * the existing gates (`humaneval-gate.mts`, `humaneval-repair-gate.mts`) select/steer + * on the task's own grading test — defensible in a deployable-verifier framing, but + * NOT a benchmark-lift claim. Here the harness sees nothing the model can't already + * read in its prompt. + * + * Honesty by construction — two physically separated phases: + * Phase A (harness): k samples/task at one temperature → honest doctest score per + * sample (docker, --network=none) → argmax select → ≤R repair rounds steered by + * the doctest FAILURE OUTPUT → final artifact locked. No access to task.test. + * Phase B (grading): the hidden check() suite grades every sample and every locked + * final. Nothing from this phase flows back. + * + * Judge integrity (adversarially reviewed; both spoof channels closed): + * - both judges print a per-call random NONCE sentinel and the verdict is parsed + * from that exact nonce — a candidate printing a forged summary line cannot win; + * - the hidden judge requires the sentinel, not exit-0 — `sys.exit(0)` before + * check() is a FAIL, not a pass; + * - containers run under an in-container `timeout -s KILL` so a hung candidate + * cannot outlive a crashed harness; a process-exit reaper force-removes strays. + * + * Estimators (paired across the same tasks, same sample batch): + * blind1_mean — mean hidden-pass over ALL k samples = expected pass@1 at this + * temperature (a built-in k-rep baseline; the primary control) + * blind1_first — hidden-pass of sample 0 (single-rep reference only) + * selected@1 — hidden-pass of the honest-oracle argmax sample (selection value) + * repaired@1 — hidden-pass of the final after honest-grounded repair (full harness) + * oracle@k — any sample passes hidden (the pass@k ceiling) + * Every lift carries a 95% paired-bootstrap CI (B=10000, seeded) AND an exact + * two-sided sign test — the bootstrap alone is anticonservative when few tasks move. + * + * CALIBRATE=1 skips the model entirely: canonical solutions vs both judges → + * hidden-judge self-check (must be ~100%) + honest-oracle coverage & false-fail rate. + * + * TANGLE_API_KEY=… WORKER_MODEL=meta-llama/Meta-Llama-3-8B-Instruct-Lite \ + * ROUTER_BASE=https://api.together.xyz/v1 HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz \ + * N=164 K=5 REPAIRS=2 TEMPERATURE=0.8 OUT=/abs/rows.jsonl tsx src/hev-structural.mts + */ +import { execFile, execFileSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { appendFileSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { type HumanEvalTask, extractCode, loadHumanEval } from './benchmarks/humaneval' +import { composeStrategies } from './directives' +import { type PairedLift, pairedLift, pool } from './stats.mts' + +const dockerImage = 'python:3.12-slim' +const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000) + +const solveInstruction = + 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' + +function must(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`env ${name} is required`) + return v +} + +// ---------- docker semaphore + jailed runner (shared by BOTH judges) ---------- +// Phase A workers each run docker calls too, so the container count must be bounded +// by ONE global semaphore, not by whichever pool happens to wrap the caller. + +let dockerSlots = 6 +let dockerInFlight = 0 +const dockerWaiters: Array<() => void> = [] +async function withDockerSlot(fn: () => Promise): Promise { + if (dockerInFlight >= dockerSlots) await new Promise((r) => dockerWaiters.push(r)) + dockerInFlight += 1 + try { + return await fn() + } finally { + dockerInFlight -= 1 + dockerWaiters.shift()?.() + } +} + +const containerPrefix = `hevs-${process.pid}` +let containerSeq = 0 + +// Best-effort stray-container reap on any exit path (crash, SIGINT, clean end). +function reapContainers(): void { + try { + const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], { timeout: 10000 }).toString().trim() + if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 }) + } catch { + /* reaper is best-effort by design */ + } +} +process.on('SIGINT', () => { + reapContainers() + process.exit(130) +}) +process.on('SIGTERM', () => { + reapContainers() + process.exit(143) +}) + +interface JailResult { + exitCode: number + stdout: string + stderr: string +} + +/** Run one python program in the jail: --network=none, cpu/mem caps, an IN-CONTAINER + * `timeout -s KILL` (so a hung candidate's container self-terminates even if this + * process dies), a client timeout, and a backstop. Docker INFRA faults (daemon, + * image, permission) throw — a broken checker must fail loud, not score zeros. */ +function runJailed(program: string): Promise { + return withDockerSlot( + () => + new Promise((resolvePromise, reject) => { + const dir = mkdtempSync(join(tmpdir(), 'hevs-')) + writeFileSync(join(dir, 'p.py'), program) + const name = `${containerPrefix}-${containerSeq++}` + let settled = false + const cleanup = () => { + rmSync(dir, { recursive: true, force: true }) + execFile('docker', ['rm', '-f', name], () => {}) + } + const finish = (res: JailResult) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + resolvePromise(res) + } + const fail = (e: Error) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + reject(e) + } + const backstop = setTimeout(() => finish({ exitCode: 124, stdout: '', stderr: 'backstop timeout (no output)' }), dockerTimeoutMs + 5000) + const inContainerSecs = Math.ceil(dockerTimeoutMs / 1000) + 2 + execFile( + 'docker', + [ + 'run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', + '-v', `${dir}:/w:ro`, '-w', '/w', dockerImage, + 'timeout', '-s', 'KILL', String(inContainerSecs), 'python', '/w/p.py', + ], + { timeout: dockerTimeoutMs + 3000, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, + (err, stdout, stderr) => { + if (err) { + const e = err as NodeJS.ErrnoException & { code?: number | string } + if (e.code === 'ENOENT') return fail(new Error('docker binary not found on PATH')) + const se = stderr ?? '' + if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(se)) { + return fail(new Error(`docker daemon unreachable: ${se.slice(0, 200)}`)) + } + if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(se)) { + return fail(new Error(`docker image ${dockerImage} unavailable: ${se.slice(0, 200)}`)) + } + const code = typeof e.code === 'number' ? e.code : 1 + return finish({ exitCode: code, stdout: stdout ?? '', stderr: se }) + } + finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) + }, + ) + }), + ) +} + +// ---------- the honest oracle: doctest over the VISIBLE docstring examples ---------- + +export interface HonestResult { + /** total visible checks: doctest examples + model-generated asserts (0 = no + * signal, -1 = the candidate crashed before the oracle could run) */ + attempted: number + failed: number + /** doctest's failure report — the ONLY feedback the repair loop may see */ + failureOutput: string + /** attempted > 0 && failed === 0 */ + pass: boolean + /** split for post-hoc audit: doctest vs generated-assert counts */ + dAttempted?: number + dFailed?: number + gAttempted?: number + gFailed?: number +} + +/** The honest program: candidate executes (prompt header first, for its imports; + * candidate def shadows the stub), then doctest runs the examples taken from the + * STUB's `__doc__` (parsing the raw prompt text instead swallows the closing `"""` + * into the last example — caught by CALIBRATE=1). Verdict line carries a per-call + * NONCE so candidate-printed forgeries can't be parsed as the summary. task.test + * never appears here. */ +function buildHonestProgram(task: HumanEvalTask, candidate: string, nonce: string, genTests: string[] = []): string { + const promptB64 = Buffer.from(task.prompt, 'utf8').toString('base64') + const entryB64 = Buffer.from(task.entryPoint, 'utf8').toString('base64') + const genB64 = Buffer.from(JSON.stringify(genTests), 'utf8').toString('base64') + return `${task.prompt}\n${candidate}\n +import ast as _ast, base64 as _b64, doctest as _doctest, io as _io, json as _json, sys as _sys +_prompt_text = _b64.b64decode("${promptB64}").decode("utf8") +_entry = _b64.b64decode("${entryB64}").decode("utf8") +_gen_tests = _json.loads(_b64.b64decode("${genB64}").decode("utf8")) +_stub_ns = {} +exec(_prompt_text, _stub_ns) +_doc = getattr(_stub_ns.get(_entry), "__doc__", None) or "" +try: + _examples = _doctest.DocTestParser().get_examples(_doc) +except ValueError: + _examples = [] # malformed docstring indentation -> no usable signal, not a crash + +# Dataset-quirk normalizations, all decidable from VISIBLE output alone: +# assertion-style examples ("f(x) == 0" with no expected output) pass iff they print True; +# quote-style repr mismatches ("21" vs '21') compare by literal value. +class _Checker(_doctest.OutputChecker): + def check_output(self, want, got, optionflags): + if super().check_output(want, got, optionflags): + return True + if want.strip() == "" and got.strip() == "True": + return True + try: + return _ast.literal_eval(want.strip()) == _ast.literal_eval(got.strip()) + except Exception: + return False + +_test = _doctest.DocTest(_examples, globs=dict(globals()), name="visible", filename="p", lineno=0, docstring=_doc) +_runner = _doctest.DocTestRunner(checker=_Checker(), verbose=False, optionflags=_doctest.NORMALIZE_WHITESPACE | _doctest.IGNORE_EXCEPTION_DETAIL) +_buf = _io.StringIO() +_res = _runner.run(_test, out=_buf.write) + +# Model-generated asserts (CodeT-style; written from the prompt BEFORE any candidate +# existed). Each runs individually so one malformed assert doesn't zero the rest. +_g_att, _g_fail = 0, 0 +for _t in _gen_tests: + _g_att += 1 + try: + exec(_t, dict(globals())) + except Exception as _e: + _g_fail += 1 + _buf.write("GENTEST FAILED: %s -> %s: %s\\n" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200])) + +_att = _res.attempted + _g_att +_fail = _res.failed + _g_fail +print("HONEST-${nonce} attempted=%d failed=%d datt=%d dfail=%d gatt=%d gfail=%d" % (_att, _fail, _res.attempted, _res.failed, _g_att, _g_fail)) +_sys.stdout.write(_buf.getvalue()[-1500:]) +_sys.exit(0 if _att > 0 and _fail == 0 else 1) +` +} + +export async function runHonestOracle(task: HumanEvalTask, candidate: string, genTests: string[] = []): Promise { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildHonestProgram(task, candidate, nonce, genTests)) + const summary = new RegExp(`HONEST-${nonce} attempted=(\\d+) failed=(\\d+) datt=(\\d+) dfail=(\\d+) gatt=(\\d+) gfail=(\\d+)`).exec(r.stdout) + if (!summary) { + // candidate crashed / hung before the oracle scaffold could report + const detail = (r.stderr || r.stdout).slice(-1500) || 'timed out (no output)' + return { attempted: -1, failed: -1, failureOutput: detail, pass: false } + } + const attempted = Number(summary[1]) + const failed = Number(summary[2]) + // Strip the sentinel line from the feedback shown to the repair loop — the model + // must never learn the summary format it could try to forge. + const failureOutput = r.stdout.replace(summary[0], '').slice(-1500) + return { + attempted, + failed, + failureOutput, + pass: attempted > 0 && failed === 0, + dAttempted: Number(summary[3]), + dFailed: Number(summary[4]), + gAttempted: Number(summary[5]), + gFailed: Number(summary[6]), + } +} + +// ---------- CodeT-style test generation (visible info only, BEFORE any candidate) ---------- + +const testGenInstruction = (count: number, entry: string) => + `Read the following Python function signature and docstring. Write exactly ${count} single-line assert statements that test the function \`${entry}\`, based ONLY on the behavior the docstring describes. Each assert must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False check). Do NOT implement the function. Do NOT copy examples verbatim if you can test other cases too. Output ONLY the assert lines inside a single \`\`\`python code block.` + +/** One LLM call per task, before sampling. Keeps only single-line, paren-balanced + * asserts that reference the entry point — malformed lines are dropped here rather + * than poisoning every candidate's score identically. */ +async function generateTests(cfg: ClientCfg, task: HumanEvalTask, count: number): Promise<{ tests: string[]; completion: Completion }> { + const c = await complete(cfg, [ + { role: 'user', content: `${testGenInstruction(count, task.entryPoint)}\n\n\`\`\`python\n${task.prompt}\`\`\`` }, + ]) + const block = extractCode(c.content) + const balanced = (s: string) => { + let d = 0 + for (const ch of s) { + if (ch === '(' || ch === '[' || ch === '{') d += 1 + else if (ch === ')' || ch === ']' || ch === '}') d -= 1 + if (d < 0) return false + } + return d === 0 + } + const tests = block + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('assert ') && l.includes(task.entryPoint) && balanced(l)) + .slice(0, count) + return { tests, completion: c } +} + +/** Honest score for ranking: fraction of visible examples passed; a candidate that + * crashed before doctest ran ranks below one that ran and failed everything. */ +function honestScore(h: HonestResult): number { + if (h.attempted <= 0) return h.attempted === 0 ? 0 : -1 + return (h.attempted - h.failed) / h.attempted +} + +// ---------- the hidden judge (Phase B / calibration ONLY) ---------- +// Rig-local rather than the shared runChecker: pass requires the nonce sentinel that +// check() prints AFTER succeeding — exit-0-before-check (sys.exit(0) in a candidate) +// is a fail here, where trusting the exit code alone would score it a pass. + +function buildHiddenProgram(task: HumanEvalTask, candidate: string, nonce: string): string { + return `${task.prompt}\n${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\nprint("HIDDEN-${nonce} PASS")\n` +} + +async function runHiddenJudge(task: HumanEvalTask, candidate: string): Promise<{ pass: number; detail?: string }> { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildHiddenProgram(task, candidate, nonce)) + if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 } + return { pass: 0, detail: (r.stderr || r.stdout).slice(-600) || 'timed out (no output)' } +} + +// ---------- model client (plain fetch; retries on transient HTTP + empty content) ---------- + +interface ClientCfg { + base: string + key: string + model: string + maxTokens: number + temperature: number +} + +interface Completion { + content: string + attempts: number + tokensIn: number + tokensOut: number +} + +async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: string }>): Promise { + let lastErr = '' + for (let attempt = 1; attempt <= 4; attempt += 1) { + if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) + const ctl = new AbortController() + const timer = setTimeout(() => ctl.abort(), 240_000) + try { + const res = await fetch(`${cfg.base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), + signal: ctl.signal, + }) + if (!res.ok) { + lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` + continue + } + const d = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const content = d.choices?.[0]?.message?.content ?? '' + // Reasoning models starve `content` when reasoning exhausts max_tokens — an + // empty reply is a transient fault to retry, not a candidate to score. + if (content.trim() === '') { + lastErr = 'empty content' + continue + } + return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + } catch (e) { + lastErr = e instanceof Error ? e.message : String(e) + } finally { + clearTimeout(timer) + } + } + throw new Error(`completion failed after retries: ${lastErr}`) +} + +/** Repair replies often echo the failure report in a bare ``` block before the fixed + * code — first-fence extraction would grab the echo. Prefer the LAST fenced block + * that contains a `def`, else fall back to the shared extractor. Purely mechanical + * parsing of the model's own reply; no task information involved. */ +function extractRepairCode(reply: string): string { + const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? '').trim()) + for (let i = fences.length - 1; i >= 0; i -= 1) { + if (/(^|\n)\s*def\s+\w+/.test(fences[i] as string)) return fences[i] as string + } + return extractCode(reply) +} + +// ---------- Phase A: the harness (sees ONLY visible information) ---------- + +interface HarnessOutcome { + taskId: string + samples: string[] + honest: HonestResult[] + selectedIdx: number + repairs: Array<{ code: string; honest: HonestResult }> + finalCode: string + /** 'already-passing' | 'no-signal' | 'repaired-pass' | 'rounds-exhausted' */ + repairStop: string + /** model-generated asserts used as extra oracle signal ([] when TESTGEN off) */ + genTests: string[] + llmCalls: number + llmAttempts: number + tokensIn: number + tokensOut: number +} + +async function runHarnessForTask(cfg: ClientCfg, task: HumanEvalTask, k: number, maxRepairs: number, testGen: number, diverse: boolean): Promise { + const basePrompt = `${solveInstruction}\n\n\`\`\`python\n${task.prompt}\`\`\`` + // DIVERSE mode: each sample slot gets a distinct strategy prefix — targets the + // all-k-samples-fail bucket, where iid resampling keeps drawing the same bug. + const slotPrompts = diverse ? composeStrategies(basePrompt, k) : Array.from({ length: k }, () => basePrompt) + let llmCalls = 0 + let llmAttempts = 0 + let tokensIn = 0 + let tokensOut = 0 + const track = (c: Completion) => { + llmCalls += 1 + llmAttempts += c.attempts + tokensIn += c.tokensIn + tokensOut += c.tokensOut + } + + // Generated tests come from the prompt alone, BEFORE any candidate exists, and + // are frozen for every sample and repair round of this task. + let genTests: string[] = [] + if (testGen > 0) { + const g = await generateTests(cfg, task, testGen) + track(g.completion) + genTests = g.tests + } + + const samples: string[] = [] + for (let i = 0; i < k; i += 1) { + const c = await complete(cfg, [{ role: 'user', content: slotPrompts[i] as string }]) + track(c) + samples.push(extractCode(c.content)) + } + const honest: HonestResult[] = [] + for (const s of samples) honest.push(await runHonestOracle(task, s, genTests)) + + // argmax by honest score, first index wins ties (deterministic; with zero + // coverage every sample ties at 0 → sample 0 = the blind pick) + let selectedIdx = 0 + for (let i = 1; i < k; i += 1) { + if (honestScore(honest[i] as HonestResult) > honestScore(honest[selectedIdx] as HonestResult)) selectedIdx = i + } + + const selHonest = honest[selectedIdx] as HonestResult + let best = { code: samples[selectedIdx] as string, honest: selHonest } + const repairs: HarnessOutcome['repairs'] = [] + let repairStop = 'already-passing' + if (!selHonest.pass) { + if (selHonest.attempted === 0) { + repairStop = 'no-signal' // no visible examples → nothing honest to steer on + } else { + repairStop = 'rounds-exhausted' + let current = best + for (let r = 0; r < maxRepairs; r += 1) { + const repairPrompt = [ + 'Your Python function failed some of the example checks shown in its own docstring.', + 'Here is the task again:', + '```python', + task.prompt.trimEnd(), + '```', + 'Your current attempt:', + '```python', + current.code, + '```', + 'Result of running the docstring examples against your attempt:', + '```', + current.honest.failureOutput.trim() || '(the code crashed before the examples could run)', + '```', + 'Fix the function so the docstring examples pass. Output the COMPLETE corrected function definition inside a single ```python code block. Do not write tests or example calls.', + ].join('\n') + const c = await complete(cfg, [{ role: 'user', content: repairPrompt }]) + track(c) + const code = extractRepairCode(c.content) + const h = await runHonestOracle(task, code, genTests) + repairs.push({ code, honest: h }) + if (honestScore(h) > honestScore(current.honest)) current = { code, honest: h } + if (honestScore(current.honest) > honestScore(best.honest)) best = current + if (h.pass) { + repairStop = 'repaired-pass' + break + } + } + } + } + + return { taskId: task.taskId, samples, honest, selectedIdx, repairs, finalCode: best.code, repairStop, genTests, llmCalls, llmAttempts, tokensIn, tokensOut } +} + +// ---------- statistics: exact sign test to pair with the bootstrap CI ---------- +// The percentile bootstrap is anticonservative when few tasks move (4 improved / 0 +// regressed at n=164 prints CI [+0.6, +4.9]pp while the exact test says p=0.125). +// The verdict requires BOTH. + +function signTestP(deltas: number[]): { pos: number; neg: number; p: number } { + const pos = deltas.filter((d) => d > 1e-9).length + const neg = deltas.filter((d) => d < -1e-9).length + const m = pos + neg + if (m === 0) return { pos, neg, p: 1 } + // two-sided exact binomial(m, 0.5) tail from the observed extreme + const logC: number[] = [0] + for (let i = 1; i <= m; i += 1) logC.push((logC[i - 1] as number) + Math.log(m - i + 1) - Math.log(i)) + const pmf = (x: number) => Math.exp((logC[x] as number) - m * Math.LN2) + const extreme = Math.max(pos, neg) + let p = 0 + for (let x = extreme; x <= m; x += 1) p += pmf(x) + p *= 2 + if (pos === neg) p = 1 + return { pos, neg, p: Math.min(1, p) } +} + +// ---------- calibration mode ---------- + +async function calibrate(tasks: HumanEvalTask[]): Promise { + console.log(`=== CALIBRATION · canonical solutions vs both judges · n=${tasks.length} ===`) + const usable = tasks.filter((t) => t.canonicalSolution) + if (usable.length !== tasks.length) console.log(` WARNING: ${tasks.length - usable.length} task(s) missing canonical_solution`) + const rows = await pool(usable, 16, async (t) => { + const full = `${t.prompt}${t.canonicalSolution}` + const hidden = await runHiddenJudge(t, full) + const honest = await runHonestOracle(t, full) + return { id: t.taskId, hidden: hidden.pass, attempted: honest.attempted, failed: honest.failed, honestPass: honest.pass } + }) + const hiddenPass = rows.filter((r) => r.hidden === 1) + const covered = rows.filter((r) => r.attempted > 0) + const falseFail = covered.filter((r) => !r.honestPass) + console.log(` hidden judge self-check: ${hiddenPass.length}/${rows.length} canonical solutions pass (must be ~100%)`) + if (hiddenPass.length < rows.length) console.log(` hidden FAILS: ${rows.filter((r) => r.hidden !== 1).map((r) => r.id).join(', ')}`) + console.log(` honest-oracle coverage: ${covered.length}/${rows.length} tasks have >=1 parseable docstring example`) + console.log(` zero-coverage tasks: ${rows.filter((r) => r.attempted === 0).map((r) => r.id).join(', ') || '(none)'}`) + console.log(` crashed-oracle tasks (attempted=-1): ${rows.filter((r) => r.attempted < 0).map((r) => r.id).join(', ') || '(none)'}`) + console.log(` honest-oracle false-fail on canonical: ${falseFail.length}/${covered.length} covered tasks`) + if (falseFail.length > 0) console.log(` false-fail ids: ${falseFail.map((r) => `${r.id}(${r.failed}/${r.attempted})`).join(', ')}`) + const examplesTotal = covered.reduce((s, r) => s + r.attempted, 0) + console.log(` examples per covered task: mean ${(examplesTotal / Math.max(1, covered.length)).toFixed(1)}`) +} + +// ---------- main ---------- + +const pct = (x: number) => `${(x * 100).toFixed(1)}%` +const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp` + +interface GradedRow extends HarnessOutcome { + hiddenSamples: number[] + hiddenFinal: number +} + +async function main(): Promise { + const n = Number(process.env.N ?? 164) + const k = Number(process.env.K ?? 5) + const maxRepairs = Number(process.env.REPAIRS ?? 2) + const offset = Number(process.env.OFFSET ?? 0) + const temperature = Number(process.env.TEMPERATURE ?? 0.8) + const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' + const solveConc = Number(process.env.CONCURRENCY ?? 6) + dockerSlots = Number(process.env.DOCKER_CONCURRENCY ?? 6) + const testGen = Number(process.env.TESTGEN ?? 0) + const diverse = process.env.DIVERSE === '1' + const out = process.env.OUT + + const tasks = await loadHumanEval(n, offset) + + if (process.env.CALIBRATE === '1') { + await calibrate(tasks) + return + } + + const cfg: ClientCfg = { base, key: must('TANGLE_API_KEY'), model, maxTokens: Number(process.env.MAX_TOKENS ?? 2500), temperature } + + console.log(`=== HumanEval STRUCTURAL lever · honest docstring oracle · n=${tasks.length} k=${k} repairs<=${maxRepairs} temp=${temperature} testgen=${testGen} diverse=${diverse ? 1 : 0} ===`) + console.log(` model=${model} base=${base} llm-conc=${solveConc} docker-conc=${dockerSlots} (global semaphore)`) + console.log(` Phase A (harness: sample->honest-select->honest-repair) then Phase B (hidden grading)`) + + // Phase A — all harness decisions locked before any hidden grading. A per-task + // fault becomes an error row (persisted, excluded from stats), not a lost run; + // >15% error rate aborts loud since that means the harness itself is broken. + let done = 0 + let errCount = 0 + const outcomes = await pool(tasks, solveConc, async (task): Promise => { + try { + const o = await runHarnessForTask(cfg, task, k, maxRepairs, testGen, diverse) + done += 1 + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, temperature, k, maxRepairs, ...o })}\n`) + process.stderr.write( + ` [A ${done}/${tasks.length}] ${o.taskId}: sel=${o.selectedIdx} honest=${o.honest.map((h) => honestScore(h).toFixed(2)).join('/')} repairs=${o.repairs.length} stop=${o.repairStop}\n`, + ) + return o + } catch (e) { + errCount += 1 + const error = e instanceof Error ? e.message : String(e) + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, taskId: task.taskId, error })}\n`) + process.stderr.write(` [A ERROR] ${task.taskId}: ${error.slice(0, 160)}\n`) + if (errCount > Math.max(3, 0.15 * tasks.length)) throw new Error(`aborting: ${errCount} task errors — harness-level fault, not task noise (last: ${error})`) + return { taskId: task.taskId, error } + } + }) + + const okOutcomes = outcomes.filter((o): o is HarnessOutcome => !('error' in o)) + const okTasks = okOutcomes.map((o) => tasks.find((t) => t.taskId === o.taskId) as HumanEvalTask) + if (errCount > 0) console.log(` WARNING: ${errCount}/${tasks.length} task(s) errored in Phase A — excluded from stats, recorded in ${out ?? '(no OUT set)'}.phaseA`) + + // Phase B — hidden grading of the locked artifacts. + console.log(`\n▶ Phase B: hidden grading (${okOutcomes.length} tasks × ${k} samples + finals)`) + const graded: GradedRow[] = await pool(okOutcomes, 16, async (o, ti) => { + const task = okTasks[ti] as HumanEvalTask + const hiddenSamples: number[] = [] + for (const s of o.samples) hiddenSamples.push((await runHiddenJudge(task, s)).pass) + const finalIsSelected = o.finalCode === o.samples[o.selectedIdx] + const hiddenFinal = finalIsSelected ? (hiddenSamples[o.selectedIdx] as number) : (await runHiddenJudge(task, o.finalCode)).pass + const g: GradedRow = { ...o, hiddenSamples, hiddenFinal } + if (out) appendFileSync(out, `${JSON.stringify({ model, temperature, k, maxRepairs, ...g })}\n`) + return g + }) + if (out) console.log(` raw rows appended to ${out} (phase-A rows incl. errors: ${out}.phaseA)`) + + // Estimators (all paired over the same graded tasks). + const blind1First = graded.map((g) => g.hiddenSamples[0] as number) + const blind1Mean = graded.map((g) => g.hiddenSamples.reduce((s, x) => s + x, 0) / g.hiddenSamples.length) + const selected = graded.map((g) => g.hiddenSamples[g.selectedIdx] as number) + const repaired = graded.map((g) => g.hiddenFinal) + const oracleK = graded.map((g) => (g.hiddenSamples.some((x) => x === 1) ? 1 : 0)) + // coverage is a task-level property; any non-crashed sample's oracle run proves it + const covered = graded.map((g) => (g.honest.some((h) => h.attempted > 0) ? 1 : 0)) + const rate = (xs: number[]) => xs.reduce((s, x) => s + x, 0) / xs.length + const llmCallsTotal = graded.reduce((s, g) => s + g.llmCalls, 0) + const llmAttemptsTotal = graded.reduce((s, g) => s + g.llmAttempts, 0) + const tokensInTotal = graded.reduce((s, g) => s + g.tokensIn, 0) + const tokensOutTotal = graded.reduce((s, g) => s + g.tokensOut, 0) + const repairFired = graded.filter((g) => g.repairs.length > 0) + + console.log(`\n${'='.repeat(78)}`) + console.log(`RESULTS · HumanEval structural lever · n=${graded.length} · k=${k} · repairs<=${maxRepairs} · temp=${temperature} · ${model}`) + console.log('='.repeat(78)) + console.log(` honest-oracle coverage ${pct(rate(covered))} of tasks (>=1 docstring example)`) + console.log(` blind pass@1 (mean of k) ${pct(rate(blind1Mean))} [PRIMARY baseline — ${k}-rep estimator]`) + console.log(` blind pass@1 (first sample) ${pct(rate(blind1First))} [single-rep reference]`) + console.log(` selected@1 (honest argmax) ${pct(rate(selected))}`) + console.log(` repaired@1 (full harness) ${pct(rate(repaired))}`) + console.log(` oracle pass@${k} (ceiling) ${pct(rate(oracleK))}`) + console.log( + ` compute: ${llmCallsTotal} llm calls (${llmAttemptsTotal} incl. retries) = ${(llmCallsTotal / graded.length).toFixed(2)}/task; tokens in/out ${tokensInTotal}/${tokensOutTotal} (blind@1 spends 1 call/task)`, + ) + console.log(` repair fired on ${repairFired.length}/${graded.length} tasks (stop: ${['already-passing', 'no-signal', 'repaired-pass', 'rounds-exhausted'].map((s) => `${s}=${graded.filter((g) => g.repairStop === s).length}`).join(', ')})`) + + const row = (label: string, baseline: number[], treatment: number[]) => { + const l = pairedLift(baseline, treatment) + const st = signTestP(baseline.map((b, i) => (treatment[i] as number) - b)) + console.log( + ` ${label.padEnd(36)} ${pp(l.point).padStart(7)} CI [${pp(l.low)}, ${pp(l.high)}] sign-test p=${st.p < 0.001 ? st.p.toExponential(1) : st.p.toFixed(3)} (+${st.pos}/−${st.neg}) (pairs ${l.pairs})`, + ) + return { l, st } + } + + console.log(`\n PAIRED LIFTS vs blind pass@1 (mean-of-${k}) · 95% bootstrap CI (B=10000) + exact sign test:`) + const sel = row('selected@1 − blind@1 (selection)', blind1Mean, selected) + const rep = row('repaired@1 − blind@1 (full harness)', blind1Mean, repaired) + row('repaired@1 − selected@1 (repair)', selected, repaired) + row(`oracle@${k} − repaired@1 (unrealized)`, repaired, oracleK) + + // Subgroup views (report-only; the primary claim is the unconditional lift): + const coveredIdx = graded.map((_, i) => i).filter((i) => covered[i] === 1) + if (coveredIdx.length > 0 && coveredIdx.length < graded.length) { + const pick = (xs: number[]) => coveredIdx.map((i) => xs[i] as number) + console.log(`\n COVERED-ONLY subgroup (n=${coveredIdx.length} tasks with visible examples; oracle can only act here):`) + row(' selected@1 − blind@1', pick(blind1Mean), pick(selected)) + row(' repaired@1 − blind@1', pick(blind1Mean), pick(repaired)) + } + + const verdict = (name: string, r: { l: PairedLift; st: { p: number } }) => + `${name}: ${pp(r.l.point)} — ${r.l.low > 0 && r.st.p < 0.05 ? 'POSITIVE (CI excludes 0 AND sign-test p<0.05)' : r.l.high < 0 && r.st.p < 0.05 ? 'NEGATIVE' : 'n.s.'}` + console.log(`\n VERDICT: ${verdict('full harness', rep)}; ${verdict('selection alone', sel)}`) +} + +main().catch((e) => { + reapContainers() + console.error(`hev-structural: ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`) + process.exit(1) +}) diff --git a/bench/src/mbpp-structural.mts b/bench/src/mbpp-structural.mts new file mode 100644 index 00000000..abf5d33b --- /dev/null +++ b/bench/src/mbpp-structural.mts @@ -0,0 +1,662 @@ +/** + * MBPP STRUCTURAL lever — the transfer test for the HumanEval result in + * `hev-structural.mts`: best-of-k selection + self-repair grounded ONLY on + * VISIBLE checks, graded on HIDDEN tests the harness never shows the model. + * + * MBPP (sanitized, 427 tasks) has no docstring examples; the standard protocol + * shows the model test_list[0] (it pins the function name/signature). So: + * VISIBLE = test_list[0] (+ TESTGEN model-written asserts, generated from the + * description BEFORE any candidate exists) + * HIDDEN = test_list[1:] (with test_imports) — the grading suite + * A task with <2 asserts cannot split visible/hidden and is dropped at load. + * + * Architecture is hev-structural.mts verbatim (kept self-contained on purpose — + * the HumanEval rig is frozen post-verification): Phase A (harness: sample → + * visible-check select → visible-grounded repair, all decisions locked) then + * Phase B (hidden grading); per-call NONCE sentinels on both judges; global + * docker semaphore; in-container timeout + exit reaper; incremental OUT jsonl; + * per-task error rows with >15% abort; paired bootstrap + exact sign test. + * + * CALIBRATE=1: reference solutions through both judges (hidden self-check must + * be ~100%; failures listed — some MBPP references are known-defective). + * + * TANGLE_API_KEY=… WORKER_MODEL=meta-llama/Meta-Llama-3-8B-Instruct-Lite \ + * ROUTER_BASE=https://api.together.xyz/v1 MBPP_JSON=/abs/sanitized-mbpp.json \ + * N=427 K=5 REPAIRS=2 TESTGEN=6 TEMPERATURE=0.8 OUT=/abs/rows.jsonl \ + * tsx src/mbpp-structural.mts + */ +import { execFile, execFileSync } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { extractCode } from './benchmarks/humaneval' +import { type PairedLift, pairedLift, pool } from './stats.mts' + +const dockerImage = 'python:3.12-slim' +const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000) + +function must(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`env ${name} is required`) + return v +} + +// ---------- dataset ---------- + +export interface MbppTask { + taskId: number + description: string + /** test_list[0] — shown to the model; pins the function name/signature */ + shownAssert: string + /** test_list[1:] — the hidden grading suite */ + hiddenAsserts: string[] + testImports: string[] + entryPoint: string + /** reference solution — judge self-check only, never shown to the model */ + referenceCode: string +} + +interface RawMbpp { + task_id: number + prompt: string + code: string + test_imports?: string[] + test_list: string[] +} + +/** Load sanitized MBPP, sorted by task_id. Drops (and counts) tasks that cannot + * split visible/hidden (<2 asserts) or whose entry point cannot be resolved. + * Entry point = the FIRST called name in test_list[0] that also has a + * `def ` in the reference code — `assert set(f(...)) == …` must resolve + * to f, not set. */ +export function loadMbpp(limit: number, offset = 0): { tasks: MbppTask[]; droppedShort: number[]; droppedEntry: number[] } { + const path = process.env.MBPP_JSON + if (!path) throw new Error('env MBPP_JSON is required (path to sanitized-mbpp.json)') + const raw = JSON.parse(readFileSync(path, 'utf8')) as RawMbpp[] + raw.sort((a, b) => a.task_id - b.task_id) + const tasks: MbppTask[] = [] + const droppedShort: number[] = [] + const droppedEntry: number[] = [] + for (const d of raw) { + if (!d.test_list || d.test_list.length < 2) { + droppedShort.push(d.task_id) + continue + } + const shown = d.test_list[0] as string + const calls = [...shown.matchAll(/(\w+)\s*\(/g)].map((m) => m[1] as string) + const entry = calls.find((c) => new RegExp(`def\\s+${c}\\s*\\(`).test(d.code)) + if (!entry) { + droppedEntry.push(d.task_id) + continue + } + tasks.push({ + taskId: d.task_id, + description: d.prompt, + shownAssert: shown, + hiddenAsserts: d.test_list.slice(1), + testImports: d.test_imports ?? [], + entryPoint: entry, + referenceCode: d.code, + }) + } + if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= usable dataset size ${tasks.length}`) + return { tasks: tasks.slice(offset, offset + limit), droppedShort, droppedEntry } +} + +const solveInstruction = + 'Write a Python function for the following task. Output the COMPLETE function definition (plus any imports it needs) inside a single ```python code block. Do not write tests or example calls.' + +function basePrompt(task: MbppTask): string { + return `${solveInstruction}\n\nTask: ${task.description}\nYour function must satisfy this example test:\n\`\`\`python\n${task.shownAssert}\n\`\`\`` +} + +// ---------- docker semaphore + jailed runner (mirrors hev-structural) ---------- + +let dockerSlots = 6 +let dockerInFlight = 0 +const dockerWaiters: Array<() => void> = [] +async function withDockerSlot(fn: () => Promise): Promise { + if (dockerInFlight >= dockerSlots) await new Promise((r) => dockerWaiters.push(r)) + dockerInFlight += 1 + try { + return await fn() + } finally { + dockerInFlight -= 1 + dockerWaiters.shift()?.() + } +} + +const containerPrefix = `mbpps-${process.pid}` +let containerSeq = 0 + +function reapContainers(): void { + try { + const ids = execFileSync('docker', ['ps', '-aq', '--filter', `name=${containerPrefix}`], { timeout: 10000 }).toString().trim() + if (ids) execFileSync('docker', ['rm', '-f', ...ids.split('\n')], { timeout: 15000 }) + } catch { + /* reaper is best-effort by design */ + } +} +process.on('SIGINT', () => { + reapContainers() + process.exit(130) +}) +process.on('SIGTERM', () => { + reapContainers() + process.exit(143) +}) + +interface JailResult { + exitCode: number + stdout: string + stderr: string +} + +function runJailed(program: string): Promise { + return withDockerSlot( + () => + new Promise((resolvePromise, reject) => { + const dir = mkdtempSync(join(tmpdir(), 'mbpps-')) + writeFileSync(join(dir, 'p.py'), program) + const name = `${containerPrefix}-${containerSeq++}` + let settled = false + const cleanup = () => { + rmSync(dir, { recursive: true, force: true }) + execFile('docker', ['rm', '-f', name], () => {}) + } + const finish = (res: JailResult) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + resolvePromise(res) + } + const fail = (e: Error) => { + if (settled) return + settled = true + clearTimeout(backstop) + cleanup() + reject(e) + } + const backstop = setTimeout(() => finish({ exitCode: 124, stdout: '', stderr: 'backstop timeout (no output)' }), dockerTimeoutMs + 5000) + const inContainerSecs = Math.ceil(dockerTimeoutMs / 1000) + 2 + execFile( + 'docker', + [ + 'run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', + '-v', `${dir}:/w:ro`, '-w', '/w', dockerImage, + 'timeout', '-s', 'KILL', String(inContainerSecs), 'python', '/w/p.py', + ], + { timeout: dockerTimeoutMs + 3000, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, + (err, stdout, stderr) => { + if (err) { + const e = err as NodeJS.ErrnoException & { code?: number | string } + if (e.code === 'ENOENT') return fail(new Error('docker binary not found on PATH')) + const se = stderr ?? '' + if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(se)) { + return fail(new Error(`docker daemon unreachable: ${se.slice(0, 200)}`)) + } + if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(se)) { + return fail(new Error(`docker image ${dockerImage} unavailable: ${se.slice(0, 200)}`)) + } + const code = typeof e.code === 'number' ? e.code : 1 + return finish({ exitCode: code, stdout: stdout ?? '', stderr: se }) + } + finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) + }, + ) + }), + ) +} + +// ---------- the visible-check judge (Phase A's ONLY signal) ---------- + +export interface VisibleResult { + /** total visible checks run: shown assert + generated asserts (-1 = crashed) */ + attempted: number + failed: number + failureOutput: string + pass: boolean + sAttempted?: number + sFailed?: number + gAttempted?: number + gFailed?: number +} + +/** Each visible assert runs INDIVIDUALLY in try/except so one malformed line + * cannot zero the rest; per-assert failure text is the repair feedback. The + * hidden asserts (test_list[1:]) never appear here. */ +function buildVisibleProgram(task: MbppTask, candidate: string, nonce: string, genTests: string[]): string { + const shownB64 = Buffer.from(JSON.stringify([task.shownAssert]), 'utf8').toString('base64') + const genB64 = Buffer.from(JSON.stringify(genTests), 'utf8').toString('base64') + return `${task.testImports.join('\n')}\n${candidate}\n +import base64 as _b64, json as _json, sys as _sys +_shown = _json.loads(_b64.b64decode("${shownB64}").decode("utf8")) +_gen = _json.loads(_b64.b64decode("${genB64}").decode("utf8")) +_out = [] +def _run(_tests): + _att, _fail = 0, 0 + for _t in _tests: + _att += 1 + try: + exec(_t, dict(globals())) + except Exception as _e: + _fail += 1 + _out.append("CHECK FAILED: %s -> %s: %s" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200])) + return _att, _fail +_s_att, _s_fail = _run(_shown) +_g_att, _g_fail = _run(_gen) +_att, _fail = _s_att + _g_att, _s_fail + _g_fail +print("VISIBLE-${nonce} attempted=%d failed=%d satt=%d sfail=%d gatt=%d gfail=%d" % (_att, _fail, _s_att, _s_fail, _g_att, _g_fail)) +_sys.stdout.write("\\n".join(_out)[-1500:]) +_sys.exit(0 if _att > 0 and _fail == 0 else 1) +` +} + +export async function runVisibleJudge(task: MbppTask, candidate: string, genTests: string[]): Promise { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildVisibleProgram(task, candidate, nonce, genTests)) + const summary = new RegExp(`VISIBLE-${nonce} attempted=(\\d+) failed=(\\d+) satt=(\\d+) sfail=(\\d+) gatt=(\\d+) gfail=(\\d+)`).exec(r.stdout) + if (!summary) { + const detail = (r.stderr || r.stdout).slice(-1500) || 'timed out (no output)' + return { attempted: -1, failed: -1, failureOutput: detail, pass: false } + } + const attempted = Number(summary[1]) + const failed = Number(summary[2]) + const failureOutput = r.stdout.replace(summary[0], '').slice(-1500) + return { + attempted, + failed, + failureOutput, + pass: attempted > 0 && failed === 0, + sAttempted: Number(summary[3]), + sFailed: Number(summary[4]), + gAttempted: Number(summary[5]), + gFailed: Number(summary[6]), + } +} + +function visibleScore(h: VisibleResult): number { + if (h.attempted <= 0) return h.attempted === 0 ? 0 : -1 + // The shown assert is OFFICIAL (printed in the model's prompt); generated asserts + // are the model's own guesses and run ~70% wrong on MBPP's one-sentence specs + // (measured on the pilot: 71/102 failed on officially-passing code). Rank by the + // official signal first; guesses only break ties — otherwise 6 noisy guesses + // outvote the one reliable check and selection goes NEGATIVE. + const sA = h.sAttempted ?? 0 + const gA = h.gAttempted ?? 0 + const sFrac = sA > 0 ? (sA - (h.sFailed ?? 0)) / sA : 0 + const gFrac = gA > 0 ? (gA - (h.gFailed ?? 0)) / gA : 0 + return sFrac + 0.001 * gFrac +} + +// ---------- the hidden judge (Phase B / calibration ONLY) ---------- + +function buildHiddenProgram(task: MbppTask, candidate: string, nonce: string): string { + return `${task.testImports.join('\n')}\n${candidate}\n\n${task.hiddenAsserts.join('\n')}\nprint("HIDDEN-${nonce} PASS")\n` +} + +async function runHiddenJudge(task: MbppTask, candidate: string): Promise<{ pass: number; detail?: string }> { + const nonce = randomBytes(8).toString('hex') + const r = await runJailed(buildHiddenProgram(task, candidate, nonce)) + if (r.exitCode === 0 && r.stdout.includes(`HIDDEN-${nonce} PASS`)) return { pass: 1 } + return { pass: 0, detail: (r.stderr || r.stdout).slice(-600) || 'timed out (no output)' } +} + +// ---------- model client (mirrors hev-structural) ---------- + +interface ClientCfg { + base: string + key: string + model: string + maxTokens: number + temperature: number +} + +interface Completion { + content: string + attempts: number + tokensIn: number + tokensOut: number +} + +async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: string }>): Promise { + let lastErr = '' + for (let attempt = 1; attempt <= 4; attempt += 1) { + if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) + const ctl = new AbortController() + const timer = setTimeout(() => ctl.abort(), 240_000) + try { + const res = await fetch(`${cfg.base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), + signal: ctl.signal, + }) + if (!res.ok) { + lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` + continue + } + const d = (await res.json()) as { + choices?: Array<{ message?: { content?: string } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const content = d.choices?.[0]?.message?.content ?? '' + if (content.trim() === '') { + lastErr = 'empty content' + continue + } + return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + } catch (e) { + lastErr = e instanceof Error ? e.message : String(e) + } finally { + clearTimeout(timer) + } + } + throw new Error(`completion failed after retries: ${lastErr}`) +} + +function extractRepairCode(reply: string): string { + const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? '').trim()) + for (let i = fences.length - 1; i >= 0; i -= 1) { + if (/(^|\n)\s*def\s+\w+/.test(fences[i] as string)) return fences[i] as string + } + return extractCode(reply) +} + +// ---------- TESTGEN (mirrors hev-structural; description + shown assert only) ---------- + +const testGenInstruction = (count: number, entry: string) => + `Read the following task description and example test. Write exactly ${count} single-line assert statements that test the function \`${entry}\`, based ONLY on the described behavior. Match the EXACT output type and format the example test shows (if it expects a string, expect a string; if a tuple, a tuple). Each assert must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False check). Do NOT implement the function. Do NOT repeat the example test verbatim if you can test other cases too. Output ONLY the assert lines inside a single \`\`\`python code block.` + +async function generateTests(cfg: ClientCfg, task: MbppTask, count: number): Promise<{ tests: string[]; completion: Completion }> { + const c = await complete(cfg, [ + { role: 'user', content: `${testGenInstruction(count, task.entryPoint)}\n\nTask: ${task.description}\nExample test:\n\`\`\`python\n${task.shownAssert}\n\`\`\`` }, + ]) + const block = extractCode(c.content) + const balanced = (s: string) => { + let d = 0 + for (const ch of s) { + if (ch === '(' || ch === '[' || ch === '{') d += 1 + else if (ch === ')' || ch === ']' || ch === '}') d -= 1 + if (d < 0) return false + } + return d === 0 + } + const tests = block + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('assert ') && l.includes(task.entryPoint) && balanced(l)) + .slice(0, count) + return { tests, completion: c } +} + +// ---------- Phase A: the harness (sees ONLY visible information) ---------- + +interface HarnessOutcome { + taskId: number + samples: string[] + visible: VisibleResult[] + selectedIdx: number + repairs: Array<{ code: string; visible: VisibleResult }> + finalCode: string + repairStop: string + genTests: string[] + llmCalls: number + llmAttempts: number + tokensIn: number + tokensOut: number +} + +async function runHarnessForTask(cfg: ClientCfg, task: MbppTask, k: number, maxRepairs: number, testGen: number): Promise { + let llmCalls = 0 + let llmAttempts = 0 + let tokensIn = 0 + let tokensOut = 0 + const track = (c: Completion) => { + llmCalls += 1 + llmAttempts += c.attempts + tokensIn += c.tokensIn + tokensOut += c.tokensOut + } + + let genTests: string[] = [] + if (testGen > 0) { + const g = await generateTests(cfg, task, testGen) + track(g.completion) + genTests = g.tests + } + + const samples: string[] = [] + for (let i = 0; i < k; i += 1) { + const c = await complete(cfg, [{ role: 'user', content: basePrompt(task) }]) + track(c) + samples.push(extractCode(c.content)) + } + const visible: VisibleResult[] = [] + for (const s of samples) visible.push(await runVisibleJudge(task, s, genTests)) + + let selectedIdx = 0 + for (let i = 1; i < k; i += 1) { + if (visibleScore(visible[i] as VisibleResult) > visibleScore(visible[selectedIdx] as VisibleResult)) selectedIdx = i + } + + const selVisible = visible[selectedIdx] as VisibleResult + let best = { code: samples[selectedIdx] as string, visible: selVisible } + const repairs: HarnessOutcome['repairs'] = [] + let repairStop = 'already-passing' + if (!selVisible.pass) { + if (selVisible.attempted === 0) { + repairStop = 'no-signal' + } else { + repairStop = 'rounds-exhausted' + let current = best + for (let r = 0; r < maxRepairs; r += 1) { + const repairPrompt = [ + 'Your Python function failed some of its checks.', + 'The task:', + task.description, + 'It must satisfy this example test:', + '```python', + task.shownAssert, + '```', + 'Your current attempt:', + '```python', + current.code, + '```', + 'Result of running the checks against your attempt:', + '```', + current.visible.failureOutput.trim() || '(the code crashed before the checks could run)', + '```', + 'Fix the function so the checks pass. Output the COMPLETE corrected function definition inside a single ```python code block. Do not write tests or example calls.', + ].join('\n') + const c = await complete(cfg, [{ role: 'user', content: repairPrompt }]) + track(c) + const code = extractRepairCode(c.content) + const h = await runVisibleJudge(task, code, genTests) + repairs.push({ code, visible: h }) + if (visibleScore(h) > visibleScore(current.visible)) current = { code, visible: h } + if (visibleScore(current.visible) > visibleScore(best.visible)) best = current + if (h.pass) { + repairStop = 'repaired-pass' + break + } + } + } + } + + return { taskId: task.taskId, samples, visible, selectedIdx, repairs, finalCode: best.code, repairStop, genTests, llmCalls, llmAttempts, tokensIn, tokensOut } +} + +// ---------- statistics (mirrors hev-structural) ---------- + +function signTestP(deltas: number[]): { pos: number; neg: number; p: number } { + const pos = deltas.filter((d) => d > 1e-9).length + const neg = deltas.filter((d) => d < -1e-9).length + const m = pos + neg + if (m === 0) return { pos, neg, p: 1 } + const logC: number[] = [0] + for (let i = 1; i <= m; i += 1) logC.push((logC[i - 1] as number) + Math.log(m - i + 1) - Math.log(i)) + const pmf = (x: number) => Math.exp((logC[x] as number) - m * Math.LN2) + const extreme = Math.max(pos, neg) + let p = 0 + for (let x = extreme; x <= m; x += 1) p += pmf(x) + p *= 2 + if (pos === neg) p = 1 + return { pos, neg, p: Math.min(1, p) } +} + +// ---------- calibration mode ---------- + +async function calibrate(tasks: MbppTask[]): Promise { + console.log(`=== CALIBRATION · MBPP reference solutions vs both judges · n=${tasks.length} ===`) + const rows = await pool(tasks, 16, async (t) => { + const hidden = await runHiddenJudge(t, t.referenceCode) + const visible = await runVisibleJudge(t, t.referenceCode, []) + return { id: t.taskId, hidden: hidden.pass, hiddenDetail: hidden.detail, vAttempted: visible.attempted, vFailed: visible.failed, visiblePass: visible.pass } + }) + const hiddenPass = rows.filter((r) => r.hidden === 1) + const visibleFail = rows.filter((r) => !r.visiblePass) + console.log(` hidden judge self-check: ${hiddenPass.length}/${rows.length} reference solutions pass (must be ~100%)`) + if (hiddenPass.length < rows.length) { + for (const r of rows.filter((x) => x.hidden !== 1)) console.log(` hidden FAIL ${r.id}: ${(r.hiddenDetail ?? '').replace(/\n/g, ' | ').slice(0, 160)}`) + } + console.log(` visible-check false-fail on reference: ${visibleFail.length}/${rows.length}`) + if (visibleFail.length > 0) console.log(` visible false-fail ids: ${visibleFail.map((r) => `${r.id}(${r.vFailed}/${r.vAttempted})`).join(', ')}`) +} + +// ---------- main ---------- + +const pct = (x: number) => `${(x * 100).toFixed(1)}%` +const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp` + +interface GradedRow extends HarnessOutcome { + hiddenSamples: number[] + hiddenFinal: number +} + +async function main(): Promise { + const n = Number(process.env.N ?? 427) + const k = Number(process.env.K ?? 5) + const maxRepairs = Number(process.env.REPAIRS ?? 2) + const offset = Number(process.env.OFFSET ?? 0) + const temperature = Number(process.env.TEMPERATURE ?? 0.8) + const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' + const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' + const solveConc = Number(process.env.CONCURRENCY ?? 6) + dockerSlots = Number(process.env.DOCKER_CONCURRENCY ?? 6) + const testGen = Number(process.env.TESTGEN ?? 0) + const out = process.env.OUT + + const { tasks, droppedShort, droppedEntry } = loadMbpp(n, offset) + console.log(`loaded ${tasks.length} MBPP task(s); dropped ${droppedShort.length} (<2 asserts: ${droppedShort.join(',') || '-'}), ${droppedEntry.length} (entry unresolved: ${droppedEntry.join(',') || '-'})`) + + if (process.env.CALIBRATE === '1') { + await calibrate(tasks) + return + } + + const cfg: ClientCfg = { base, key: must('TANGLE_API_KEY'), model, maxTokens: Number(process.env.MAX_TOKENS ?? 2500), temperature } + + console.log(`=== MBPP STRUCTURAL lever · visible=test_list[0]+gen · hidden=test_list[1:] · n=${tasks.length} k=${k} repairs<=${maxRepairs} temp=${temperature} testgen=${testGen} ===`) + console.log(` model=${model} base=${base} llm-conc=${solveConc} docker-conc=${dockerSlots} (global semaphore)`) + + let done = 0 + let errCount = 0 + const outcomes = await pool(tasks, solveConc, async (task): Promise => { + try { + const o = await runHarnessForTask(cfg, task, k, maxRepairs, testGen) + done += 1 + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, temperature, k, maxRepairs, ...o })}\n`) + process.stderr.write( + ` [A ${done}/${tasks.length}] mbpp/${o.taskId}: sel=${o.selectedIdx} visible=${o.visible.map((h) => visibleScore(h).toFixed(2)).join('/')} repairs=${o.repairs.length} stop=${o.repairStop}\n`, + ) + return o + } catch (e) { + errCount += 1 + const error = e instanceof Error ? e.message : String(e) + if (out) appendFileSync(`${out}.phaseA`, `${JSON.stringify({ model, taskId: task.taskId, error })}\n`) + process.stderr.write(` [A ERROR] mbpp/${task.taskId}: ${error.slice(0, 160)}\n`) + if (errCount > Math.max(3, 0.15 * tasks.length)) throw new Error(`aborting: ${errCount} task errors — harness-level fault, not task noise (last: ${error})`) + return { taskId: task.taskId, error } + } + }) + + const okOutcomes = outcomes.filter((o): o is HarnessOutcome => !('error' in o)) + const okTasks = okOutcomes.map((o) => tasks.find((t) => t.taskId === o.taskId) as MbppTask) + if (errCount > 0) console.log(` WARNING: ${errCount}/${tasks.length} task(s) errored in Phase A — excluded from stats, recorded in ${out ?? '(no OUT set)'}.phaseA`) + + console.log(`\n▶ Phase B: hidden grading (${okOutcomes.length} tasks × ${k} samples + finals)`) + const graded: GradedRow[] = await pool(okOutcomes, 16, async (o, ti) => { + const task = okTasks[ti] as MbppTask + const hiddenSamples: number[] = [] + for (const s of o.samples) hiddenSamples.push((await runHiddenJudge(task, s)).pass) + const finalIsSelected = o.finalCode === o.samples[o.selectedIdx] + const hiddenFinal = finalIsSelected ? (hiddenSamples[o.selectedIdx] as number) : (await runHiddenJudge(task, o.finalCode)).pass + const g: GradedRow = { ...o, hiddenSamples, hiddenFinal } + if (out) appendFileSync(out, `${JSON.stringify({ model, temperature, k, maxRepairs, ...g })}\n`) + return g + }) + if (out) console.log(` raw rows appended to ${out} (phase-A rows incl. errors: ${out}.phaseA)`) + + const blind1First = graded.map((g) => g.hiddenSamples[0] as number) + const blind1Mean = graded.map((g) => g.hiddenSamples.reduce((s, x) => s + x, 0) / g.hiddenSamples.length) + const selected = graded.map((g) => g.hiddenSamples[g.selectedIdx] as number) + const repaired = graded.map((g) => g.hiddenFinal) + const oracleK = graded.map((g) => (g.hiddenSamples.some((x) => x === 1) ? 1 : 0)) + const covered = graded.map((g) => (g.visible.some((h) => h.attempted > 0) ? 1 : 0)) + const rate = (xs: number[]) => xs.reduce((s, x) => s + x, 0) / xs.length + const llmCallsTotal = graded.reduce((s, g) => s + g.llmCalls, 0) + const llmAttemptsTotal = graded.reduce((s, g) => s + g.llmAttempts, 0) + const tokensInTotal = graded.reduce((s, g) => s + g.tokensIn, 0) + const tokensOutTotal = graded.reduce((s, g) => s + g.tokensOut, 0) + const repairFired = graded.filter((g) => g.repairs.length > 0) + + console.log(`\n${'='.repeat(78)}`) + console.log(`RESULTS · MBPP structural lever · n=${graded.length} · k=${k} · repairs<=${maxRepairs} · temp=${temperature} · ${model}`) + console.log('='.repeat(78)) + console.log(` visible-check coverage ${pct(rate(covered))} of tasks (shown assert always present)`) + console.log(` blind pass@1 (mean of k) ${pct(rate(blind1Mean))} [PRIMARY baseline — ${k}-rep estimator]`) + console.log(` blind pass@1 (first sample) ${pct(rate(blind1First))} [single-rep reference]`) + console.log(` selected@1 (visible argmax) ${pct(rate(selected))}`) + console.log(` repaired@1 (full harness) ${pct(rate(repaired))}`) + console.log(` oracle pass@${k} (ceiling) ${pct(rate(oracleK))}`) + console.log( + ` compute: ${llmCallsTotal} llm calls (${llmAttemptsTotal} incl. retries) = ${(llmCallsTotal / graded.length).toFixed(2)}/task; tokens in/out ${tokensInTotal}/${tokensOutTotal} (blind@1 spends 1 call/task)`, + ) + console.log(` repair fired on ${repairFired.length}/${graded.length} tasks (stop: ${['already-passing', 'no-signal', 'repaired-pass', 'rounds-exhausted'].map((s) => `${s}=${graded.filter((g) => g.repairStop === s).length}`).join(', ')})`) + + const row = (label: string, baseline: number[], treatment: number[]) => { + const l = pairedLift(baseline, treatment) + const st = signTestP(baseline.map((b, i) => (treatment[i] as number) - b)) + console.log( + ` ${label.padEnd(36)} ${pp(l.point).padStart(7)} CI [${pp(l.low)}, ${pp(l.high)}] sign-test p=${st.p < 0.001 ? st.p.toExponential(1) : st.p.toFixed(3)} (+${st.pos}/−${st.neg}) (pairs ${l.pairs})`, + ) + return { l, st } + } + + console.log(`\n PAIRED LIFTS vs blind pass@1 (mean-of-${k}) · 95% bootstrap CI (B=10000) + exact sign test:`) + const sel = row('selected@1 − blind@1 (selection)', blind1Mean, selected) + const rep = row('repaired@1 − blind@1 (full harness)', blind1Mean, repaired) + row('repaired@1 − selected@1 (repair)', selected, repaired) + row(`oracle@${k} − repaired@1 (unrealized)`, repaired, oracleK) + + const coveredIdx = graded.map((_, i) => i).filter((i) => covered[i] === 1) + if (coveredIdx.length > 0 && coveredIdx.length < graded.length) { + const pick = (xs: number[]) => coveredIdx.map((i) => xs[i] as number) + console.log(`\n COVERED-ONLY subgroup (n=${coveredIdx.length}):`) + row(' selected@1 − blind@1', pick(blind1Mean), pick(selected)) + row(' repaired@1 − blind@1', pick(blind1Mean), pick(repaired)) + } + + const verdict = (name: string, r: { l: PairedLift; st: { p: number } }) => + `${name}: ${pp(r.l.point)} — ${r.l.low > 0 && r.st.p < 0.05 ? 'POSITIVE (CI excludes 0 AND sign-test p<0.05)' : r.l.high < 0 && r.st.p < 0.05 ? 'NEGATIVE' : 'n.s.'}` + console.log(`\n VERDICT: ${verdict('full harness', rep)}; ${verdict('selection alone', sel)}`) +} + +main().catch((e) => { + reapContainers() + console.error(`mbpp-structural: ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`) + process.exit(1) +}) diff --git a/bench/src/smoke-structural-rollout.mts b/bench/src/smoke-structural-rollout.mts new file mode 100644 index 00000000..c2a42fc9 --- /dev/null +++ b/bench/src/smoke-structural-rollout.mts @@ -0,0 +1,393 @@ +/** + * smoke-structural-rollout — the ship gate for the PORTED structuralRollout strategy + * (src/runtime/structural-rollout.ts): prove the RUNTIME code path works against a REAL + * model on REAL HumanEval tasks. This runs `runAgentic` with the strategy over a + * `createVerifierEnvironment` surface — routerToolLoop, the conserved pool, the metered + * consult channel, `sandboxCheckRunner`, receipts — NOT the bench rig (hev-structural.mts). + * + * Honesty split (the rig's Phase A / Phase B, preserved): + * - The strategy sees ONLY task-visible information. The surface's check is INERT + * (always 0/1), so no hidden-test signal can reach selection or repair; the visible + * checks are the strategy's own default CheckSource (model-authored asserts, frozen + * before sampling), executed by the shipped `sandboxCheckRunner` over a docker + * `--network=none` exec channel (the thin adapter this script provides). + * - The task's own hidden check() suite grades every locked candidate HERE, in the + * script, AFTER the strategy has finished the task (`runChecker`, docker, + * --network=none). Nothing flows back. + * + * Per task we collect: the k per-sample hidden grades, the selected candidate's grade + * (argmax over samples by the exported `selectBestIndex` — the strategy's own order), + * the final (post-repair) grade (the receipt marked `selected`), authored-check counts, + * and the SelectionReceipts (cross-checked against the recorded outcomes). + * + * Run (key via dotenvx; never in the shell history): + * cd ~/company/devops/secrets && dotenvx run -f agent-state.env -- bash -c ' \ + * cd /home/drew/code/agent-runtime-swe && \ + * HUMANEVAL_GZ=/abs/HumanEval.jsonl.gz N=20 OFFSET=0 npx tsx bench/src/smoke-structural-rollout.mts' + */ + +import { execFile } from 'node:child_process' +import { appendFileSync } from 'node:fs' +import { + type AgenticRunResult, + type CheckExecChannel, + type CheckOutcome, + type CheckRunner, + createVerifierEnvironment, + defaultStructuralRolloutPolicy, + runAgentic, + type StructuralRolloutResult, + sandboxCheckRunner, + selectBestIndex, + structuralRollout, + visibleCheckScore, +} from '../../src/runtime/index' +import { basePrompt, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' + +function must(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`env ${name} is required`) + return v +} + +const N = Number(process.env.N ?? 20) +const OFFSET = Number(process.env.OFFSET ?? 0) +const MODEL = process.env.MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' +const BASE = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1' +const TEMP = Number(process.env.TEMPERATURE ?? 0.8) +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 2500) +const CONCURRENCY = Number(process.env.CONCURRENCY ?? 3) +const OUT = process.env.OUT // optional JSONL of raw per-task rows + +const systemPrompt = 'You are an expert Python programmer.' +const dockerImage = 'python:3.12-slim' + +// ── The thin adapter: a docker --network=none exec channel for sandboxCheckRunner ──── +// The runner pipes its check program as `printf '%s' '' | base64 -d | python3 -`; +// this channel runs that command inside a jailed container. Infra faults (no docker, +// daemon down) fail loud; a candidate crash/hang is a real outcome (non-zero exit / no +// sentinel), returned, never thrown. Modeled on bench/src/benchmarks/humaneval.ts. +let containerSeq = 0 +function dockerExecChannel(): CheckExecChannel { + return { + exec(command, options) { + const timeoutMs = options?.timeoutMs ?? 20000 + const name = `srck-${process.pid}-${containerSeq++}` + return new Promise((resolve, reject) => { + let settled = false + const reap = () => execFile('docker', ['rm', '-f', name], () => {}) + const finish = (r: { exitCode: number; stdout: string; stderr: string }) => { + if (settled) return + settled = true + clearTimeout(backstop) + reap() + resolve(r) + } + const fail = (e: Error) => { + if (settled) return + settled = true + clearTimeout(backstop) + reap() + reject(e) + } + // execFile's timeout kills the docker CLIENT; a hung container could leave the + // callback unfired. The backstop guarantees resolution (no sentinel ⇒ the runner + // scores it crashed) and the named reap kills the stray container. + const backstop = setTimeout( + () => finish({ exitCode: 124, stdout: '', stderr: 'timed out (backstop)' }), + timeoutMs + 3000, + ) + execFile( + 'docker', + ['run', '--rm', '--name', name, '--network=none', '--cpus=1', '--memory=512m', dockerImage, 'sh', '-c', command], + { timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 }, + (err, stdout, stderr) => { + if (err) { + const e = err as NodeJS.ErrnoException & { code?: number | string } + if (e.code === 'ENOENT') { + fail(new Error('docker binary not found on PATH — cannot run visible checks')) + return + } + if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr ?? '')) { + fail(new Error(`docker daemon unreachable: ${(stderr ?? '').slice(0, 200)}`)) + return + } + const exitCode = typeof e.code === 'number' ? e.code : 1 + finish({ exitCode, stdout: stdout ?? '', stderr: stderr ?? '' }) + return + } + finish({ exitCode: 0, stdout: stdout ?? '', stderr: stderr ?? '' }) + }, + ) + }) + }, + } +} + +// ── Recording wrapper: capture each (candidate, outcome) the strategy scores, in order. +// Delegates verbatim to the shipped runner — the code under test stays the strategy. +interface ScoredCandidate { + candidate: string + outcome: CheckOutcome +} +function recordingRunner(inner: CheckRunner, log: ScoredCandidate[]): CheckRunner { + return { + async run(candidate, checks, ctx) { + const outcome = await inner.run(candidate, checks, ctx) + log.push({ candidate, outcome }) + return outcome + }, + } +} + +interface TaskRow { + taskId: string + error?: string + officialChecks: number + authoredChecks: number + repairStop: string + shots: number + sampleHidden: number[] + blindMean: number + selectedIdx: number + selectedHidden: number + finalIdx: number + finalHidden: number + selectedVisible: number + receipts: Array<{ candidateIndex: number; selected: boolean; score: number; reason: string }> + tokens: { input: number; output: number } + usd: number + ms: number +} + +async function runTask(t: HumanEvalTask): Promise { + const scored: ScoredCandidate[] = [] + const policy = { ...defaultStructuralRolloutPolicy, temperature: TEMP } + const strategy = structuralRollout({ + policy, + checkRunner: recordingRunner(sandboxCheckRunner({ box: dockerExecChannel() }), scored), + }) + // INERT check: the strategy's harness-verified score channel must carry no hidden + // signal — hidden grading is this script's job, after the rollout locks its artifacts. + const surface = createVerifierEnvironment({ + name: 'humaneval-inert', + check: () => ({ passes: 0, total: 1, errored: 0 }), + }) + const result = (await runAgentic({ + surface, + task: { + id: t.taskId, + systemPrompt, + userPrompt: basePrompt(t), + meta: { entryPoint: t.entryPoint }, + }, + routerBaseUrl: BASE, + routerKey: must('TOGETHER_API_KEY'), + model: MODEL, + temperature: TEMP, + maxTokens: MAX_TOKENS, + innerTurns: 2, + strategy, + // The strategy's documented sizing: k samples + repair rounds + the check-author consult. + budget: policy.k + policy.repairRounds + 1, + })) as AgenticRunResult & StructuralRolloutResult + + // Receipts ↔ recorded outcomes must agree exactly (candidateIndex is the recording + // order: samples first, then repairs). A mismatch is an adapter or strategy defect. + if (result.selection.length !== scored.length) { + throw new Error( + `${t.taskId}: ${result.selection.length} receipts vs ${scored.length} scored candidates`, + ) + } + for (const r of result.selection) { + const rec = scored[r.candidateIndex] + if (!rec || Math.abs(r.score - visibleCheckScore(rec.outcome)) > 1e-9) { + throw new Error(`${t.taskId}: receipt #${r.candidateIndex} score ${r.score} does not match the recorded outcome`) + } + } + + const sampleCount = result.selection.filter((r) => r.reason.startsWith('sample')).length + const samples = scored.slice(0, sampleCount) + if (samples.length === 0) throw new Error(`${t.taskId}: no sample candidates settled`) + + // Hidden grading (script-side, docker --network=none): every distinct candidate once. + const gradeCache = new Map>() + const grade = (candidate: string): Promise => { + let p = gradeCache.get(candidate) + if (!p) { + p = runChecker(t, candidate).then((r) => r.pass) + gradeCache.set(candidate, p) + } + return p + } + const sampleHidden = await Promise.all(samples.map((s) => grade(s.candidate))) + const selectedIdx = selectBestIndex(samples.map((s) => s.outcome)) + const selectedHidden = sampleHidden[selectedIdx] as number + const winner = result.selection.find((r) => r.selected) + if (!winner) throw new Error(`${t.taskId}: no receipt marked selected`) + const finalIdx = winner.candidateIndex + const finalHidden = await grade((scored[finalIdx] as ScoredCandidate).candidate) + + return { + taskId: t.taskId, + officialChecks: result.officialChecks, + authoredChecks: result.authoredChecks, + repairStop: result.repairStop, + shots: result.shots, + sampleHidden, + blindMean: sampleHidden.reduce((a, b) => a + b, 0) / sampleHidden.length, + selectedIdx, + selectedHidden, + finalIdx, + finalHidden, + selectedVisible: visibleCheckScore((samples[selectedIdx] as ScoredCandidate).outcome), + receipts: result.selection.map((r) => ({ + candidateIndex: r.candidateIndex, + selected: r.selected, + score: r.score, + reason: r.reason, + })), + tokens: result.tokens, + usd: result.usd, + ms: result.ms, + } +} + +async function pooled(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const out: R[] = new Array(items.length) + let next = 0 + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next + next += 1 + out[i] = await fn(items[i] as T) + } + }) + await Promise.all(workers) + return out +} + +let unhandled = 0 +process.on('unhandledRejection', (e) => { + unhandled += 1 + console.error('UNHANDLED REJECTION:', e) +}) + +const pct = (x: number) => `${(100 * x).toFixed(1)}%` + +async function main(): Promise { + must('TOGETHER_API_KEY') + const tasks = await loadHumanEval(N, OFFSET) + const { k, repairRounds, testgen } = defaultStructuralRolloutPolicy + console.log( + `=== structuralRollout RUNTIME smoke · n=${tasks.length} offset=${OFFSET} · k=${k} repairs<=${repairRounds} testgen=${testgen} temp=${TEMP} ===`, + ) + console.log(` model=${MODEL} base=${BASE} maxTokens=${MAX_TOKENS} task-concurrency=${CONCURRENCY}`) + console.log(` path: runAgentic → structuralRollout(default policy) → createVerifierEnvironment(inert) → sandboxCheckRunner(docker --network=none)`) + + const started = Date.now() + const rows = await pooled(tasks, CONCURRENCY, async (t): Promise => { + try { + const row = await runTask(t) + const hid = row.sampleHidden.join(' ') + console.log( + ` ${t.taskId.padEnd(14)} auth=${row.authoredChecks} samples=[${hid}] sel=#${row.selectedIdx}:${row.selectedHidden ? 'PASS' : 'fail'} final=#${row.finalIdx}:${row.finalHidden ? 'PASS' : 'fail'} ${row.repairStop}`, + ) + if (OUT) appendFileSync(OUT, `${JSON.stringify(row)}\n`) + return row + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + console.error(` ${t.taskId.padEnd(14)} ERROR: ${msg}`) + if (OUT) appendFileSync(OUT, `${JSON.stringify({ taskId: t.taskId, error: msg })}\n`) + return { + taskId: t.taskId, + error: msg, + officialChecks: 0, + authoredChecks: 0, + repairStop: 'error', + shots: 0, + sampleHidden: [], + blindMean: 0, + selectedIdx: -1, + selectedHidden: 0, + finalIdx: -1, + finalHidden: 0, + selectedVisible: 0, + receipts: [], + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, + } + } + }) + + const ok = rows.filter((r) => !r.error) + const errored = rows.filter((r) => r.error) + + console.log('\n── per-task ──') + console.log( + 'task auth samples(hidden) blind% sel final visSel repairStop shots tok(in/out) ms', + ) + for (const r of rows) { + if (r.error) { + console.log(`${r.taskId.padEnd(15)} ERROR ${r.error.slice(0, 90)}`) + continue + } + console.log( + [ + r.taskId.padEnd(15), + String(r.authoredChecks).padEnd(5), + r.sampleHidden.join(' ').padEnd(16), + pct(r.blindMean).padEnd(8), + `#${r.selectedIdx}:${r.selectedHidden ? 'PASS' : 'fail'}`.padEnd(8), + `#${r.finalIdx}:${r.finalHidden ? 'PASS' : 'fail'}`.padEnd(8), + r.selectedVisible.toFixed(3).padEnd(7), + r.repairStop.padEnd(17), + String(r.shots).padEnd(6), + `${r.tokens.input}/${r.tokens.output}`.padEnd(15), + String(r.ms), + ].join(' '), + ) + } + + const blind = ok.reduce((a, r) => a + r.blindMean, 0) / Math.max(1, ok.length) + const selected = ok.reduce((a, r) => a + r.selectedHidden, 0) / Math.max(1, ok.length) + const final = ok.reduce((a, r) => a + r.finalHidden, 0) / Math.max(1, ok.length) + const sample0 = ok.reduce((a, r) => a + (r.sampleHidden[0] ?? 0), 0) / Math.max(1, ok.length) + const oracleAtK = ok.reduce((a, r) => a + (r.sampleHidden.some((x) => x === 1) ? 1 : 0), 0) / Math.max(1, ok.length) + const tokens = ok.reduce((a, r) => ({ input: a.input + r.tokens.input, output: a.output + r.tokens.output }), { input: 0, output: 0 }) + + console.log('\n── summary ──') + console.log(`tasks: ${ok.length} scored, ${errored.length} errored (of ${rows.length})`) + console.log(`blind mean-of-k : ${pct(blind)} (sample-0 only: ${pct(sample0)}; oracle@k ceiling: ${pct(oracleAtK)})`) + console.log(`selected@1 : ${pct(selected)} (lift over blind: ${(100 * (selected - blind)).toFixed(1)}pp)`) + console.log(`final (repaired): ${pct(final)} (lift over blind: ${(100 * (final - blind)).toFixed(1)}pp)`) + console.log(`spend: tokens ${tokens.input} in / ${tokens.output} out · wall ${((Date.now() - started) / 1000).toFixed(0)}s`) + + // ── Acceptance (mechanism, not significance — n is small) ── + const authoredTasks = ok.filter((r) => r.authoredChecks > 0).length + const receiptsSane = ok.every((r) => r.receipts.length > 0 && r.receipts.length === r.shots) + const ordering = final >= selected && selected >= blind + const liftPp = 100 * (final - blind) + const rescued = ok.filter((r) => r.finalHidden === 1 && (r.sampleHidden[0] ?? 0) === 0) + const checks: Array<[string, boolean, string]> = [ + ['authored checks on >=17/20 tasks', authoredTasks >= 17, `${authoredTasks}/${rows.length} tasks`], + ['selection receipts present, scores match recorded outcomes', receiptsSane && ok.length > 0, `verified on ${ok.length} tasks (hard-checked per receipt)`], + ['final >= selected >= blind and final-blind >= +5pp', ordering && liftPp >= 5, `blind ${pct(blind)} → selected ${pct(selected)} → final ${pct(final)} (+${liftPp.toFixed(1)}pp)`], + ['>=1 task rescued (final passes, sample 0 fails)', rescued.length >= 1, rescued.map((r) => r.taskId).join(', ') || 'none'], + ['zero crashes / unhandled rejections', errored.length === 0 && unhandled === 0, `${errored.length} task errors, ${unhandled} unhandled rejections`], + ] + console.log('\n── acceptance ──') + let allPass = true + for (const [label, pass, detail] of checks) { + if (!pass) allPass = false + console.log(` [${pass ? 'PASS' : 'FAIL'}] ${label} — ${detail}`) + } + console.log(allPass ? '\nSMOKE: PASS' : '\nSMOKE: FAIL') + process.exit(allPass ? 0 : 1) +} + +main().catch((e) => { + console.error(e) + process.exit(1) +}) diff --git a/docs/api/index.md b/docs/api/index.md index 4d93ec8d..9b4906f3 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -2546,7 +2546,7 @@ Test seam — inject the worktree-dirty check (defaults to `git status`). ### ImproveOptions -Defined in: [improvement/improve.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L57) +Defined in: [improvement/improve.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L78) #### Type Parameters @@ -2564,7 +2564,7 @@ Defined in: [improvement/improve.ts:57](https://github.com/tangle-network/agent- > `optional` **surface?**: [`ImproveSurface`](#improvesurface) -Defined in: [improvement/improve.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L60) +Defined in: [improvement/improve.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L81) Which profile lever to optimize. Default `'prompt'`. Selects the default generator + the baseline-surface extraction shape. @@ -2573,7 +2573,7 @@ Which profile lever to optimize. Default `'prompt'`. Selects the default > `optional` **generator?**: `SurfaceProposer`\<`unknown`\> -Defined in: [improvement/improve.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L64) +Defined in: [improvement/improve.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L85) The `SurfaceProposer` that mutates the surface. When unset, the facade picks the default for `surface` (`gepaProposer` for prompt, `skillOptProposer` @@ -2583,7 +2583,7 @@ The `SurfaceProposer` that mutates the surface. When unset, the facade > `optional` **gate?**: `"none"` \| `"holdout"` -Defined in: [improvement/improve.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L67) +Defined in: [improvement/improve.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L88) Gate mode. `'holdout'` (default) runs the held-out promotion gate; `'none'` is a baseline-only run (`budget.generations = 0`). @@ -2592,7 +2592,7 @@ Gate mode. `'holdout'` (default) runs the held-out promotion gate; > **scenarios**: `TScenario`[] -Defined in: [improvement/improve.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L69) +Defined in: [improvement/improve.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L90) Scenarios to evaluate against. Passthrough to `selfImprove`. @@ -2600,7 +2600,7 @@ Scenarios to evaluate against. Passthrough to `selfImprove`. > **judge**: `JudgeConfig`\<`TArtifact`, `TScenario`\> -Defined in: [improvement/improve.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L71) +Defined in: [improvement/improve.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L92) Judge that scores artifacts. Passthrough to `selfImprove`. @@ -2608,7 +2608,7 @@ Judge that scores artifacts. Passthrough to `selfImprove`. > **agent**: (`surface`, `scenario`, `ctx`) => `Promise`\<`TArtifact`\> -Defined in: [improvement/improve.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L74) +Defined in: [improvement/improve.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L95) The agent under improvement — same shape as `selfImprove.agent`: it takes the current surface + scenario + ctx and returns the artifact to judge. @@ -2635,7 +2635,7 @@ The agent under improvement — same shape as `selfImprove.agent`: it takes > `optional` **budget?**: `SelfImproveBudget` -Defined in: [improvement/improve.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L76) +Defined in: [improvement/improve.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L97) Budget + loop shape. Passthrough; `gate: 'none'` forces `generations = 0`. @@ -2643,7 +2643,7 @@ Budget + loop shape. Passthrough; `gate: 'none'` forces `generations = 0`. > `optional` **llm?**: `SelfImproveLlm` -Defined in: [improvement/improve.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L79) +Defined in: [improvement/improve.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L100) LLM config. Passthrough to `selfImprove` AND used to construct the default reflective proposer (`gepaProposer`/`skillOptProposer`) when `generator` is unset. @@ -2652,7 +2652,7 @@ LLM config. Passthrough to `selfImprove` AND used to construct the default > `optional` **allowedModels?**: readonly `string`[] -Defined in: [improvement/improve.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L83) +Defined in: [improvement/improve.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L104) Restrict the run to this subset of models. When set, the reflection model (`llm.model`, or the default when unset) must be a member, or `improve()` throws @@ -2662,7 +2662,7 @@ Restrict the run to this subset of models. When set, the reflection model > `optional` **runDir?**: `string` -Defined in: [improvement/improve.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L89) +Defined in: [improvement/improve.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L110) Run directory passthrough to `selfImprove`. Pass a REAL path to make the loop durable: campaign cells + the loop provenance record land on the filesystem as @@ -2674,7 +2674,7 @@ Run directory passthrough to `selfImprove`. Pass a REAL path to make the loop > `optional` **analyzeGeneration?**: ((`input`) => `Promise`\<`unknown`[]\>) \| `null` -Defined in: [improvement/improve.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L97) +Defined in: [improvement/improve.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L118) Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). DEFAULT: the built-in failure distiller — after each generation it turns the @@ -2688,7 +2688,7 @@ Per-generation findings producer passthrough (see selfImprove.analyzeGeneration) > `optional` **rawTraceContext?**: `boolean` -Defined in: [improvement/improve.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L107) +Defined in: [improvement/improve.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L128) META-HARNESS mode: instead of the ~400-char distilled findings, feed the proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's @@ -2704,7 +2704,7 @@ META-HARNESS mode: instead of the ~400-char distilled findings, feed the > `optional` **code?**: `ImproveCodeOptions` -Defined in: [improvement/improve.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L115) +Defined in: [improvement/improve.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L136) CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a repo, and the facade assembles the whole candidate pipeline — git worktrees @@ -2718,7 +2718,7 @@ CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a > `optional` **skills?**: `ImproveSkillsOptions` -Defined in: [improvement/improve.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L122) +Defined in: [improvement/improve.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L143) SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) @@ -2731,7 +2731,7 @@ SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, > `optional` **storage?**: `CampaignStorage` -Defined in: [improvement/improve.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L124) +Defined in: [improvement/improve.ts:145](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L145) Storage passthrough to `selfImprove`; overrides the default chosen from `runDir`. @@ -2739,7 +2739,7 @@ Storage passthrough to `selfImprove`; overrides the default chosen from `runDir` ### ImproveResult -Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L155) +Defined in: [improvement/improve.ts:176](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L176) #### Type Parameters @@ -2757,7 +2757,7 @@ Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent > **profile**: `AgentProfile` -Defined in: [improvement/improve.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L158) +Defined in: [improvement/improve.ts:179](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L179) The profile after improvement: the winner surface applied back into the matching field when the gate shipped, else the input profile unchanged. @@ -2766,7 +2766,7 @@ The profile after improvement: the winner surface applied back into the > **shipped**: `boolean` -Defined in: [improvement/improve.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L160) +Defined in: [improvement/improve.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L181) True when `gateDecision === 'ship'`. @@ -2774,7 +2774,7 @@ True when `gateDecision === 'ship'`. > **lift**: `number` -Defined in: [improvement/improve.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L162) +Defined in: [improvement/improve.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L183) Held-out lift (`winner − baseline` composite). @@ -2782,7 +2782,7 @@ Held-out lift (`winner − baseline` composite). > **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [improvement/improve.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L164) +Defined in: [improvement/improve.ts:185](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L185) The five-valued gate verdict from `selfImprove`. @@ -2790,7 +2790,7 @@ The five-valued gate verdict from `selfImprove`. > **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -Defined in: [improvement/improve.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L166) +Defined in: [improvement/improve.ts:187](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L187) Full `selfImprove` result for advanced inspection. @@ -7064,12 +7064,14 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault ### ImproveSurface -> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"code"` +> **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"code"` \| `"rollout-policy"` -Defined in: [improvement/improve.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L55) +Defined in: [improvement/improve.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L69) The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law - profile levers; `code` is the implementation-tier surface. + profile levers; `code` is the implementation-tier surface, `rollout-policy` + the inference-time structuralRollout dials + (`profile.extensions['structural-rollout']`). *** @@ -7702,6 +7704,79 @@ Hard cap on chained gateway hops; refused beyond this. Default keeps recursion b *** +### ROLLOUT\_POLICY\_EXTENSION + +> `const` **ROLLOUT\_POLICY\_EXTENSION**: `"structural-rollout"` = `'structural-rollout'` + +Defined in: [improvement/rollout-policy.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L39) + +The profile extensions namespace the policy persists under. + +*** + +### ROLLOUT\_POLICY\_BOUNDS + +> `const` **ROLLOUT\_POLICY\_BOUNDS**: `object` + +Defined in: [improvement/rollout-policy.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L45) + +Proposal bounds per dial. These are the SEARCH bounds (what the proposer may + explore), chosen so every reachable value is a measured-sane recipe: k=1 is the + low-compute preset, testgen=0 disables check authoring, repairRounds caps where + the measured increment flattens (+1–3pp beyond round 2). + +#### Type Declaration + +##### k + +> `readonly` **k**: `object` + +###### k.min + +> `readonly` **min**: `1` = `1` + +###### k.max + +> `readonly` **max**: `10` = `10` + +###### k.step + +> `readonly` **step**: `2` = `2` + +##### repairRounds + +> `readonly` **repairRounds**: `object` + +###### repairRounds.min + +> `readonly` **min**: `0` = `0` + +###### repairRounds.max + +> `readonly` **max**: `3` = `3` + +###### repairRounds.step + +> `readonly` **step**: `1` = `1` + +##### testgen + +> `readonly` **testgen**: `object` + +###### testgen.min + +> `readonly` **min**: `0` = `0` + +###### testgen.max + +> `readonly` **max**: `10` = `10` + +###### testgen.step + +> `readonly` **step**: `3` = `3` + +*** + ### RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT > `const` **RESEARCH\_SUPERVISOR\_SYSTEM\_PROMPT**: `string` @@ -8533,7 +8608,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:358](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L358) +Defined in: [improvement/improve.ts:401](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L401) Run the held-out-gated self-improvement loop on ONE profile surface. @@ -8683,6 +8758,167 @@ Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edi *** +### parseRolloutPolicy() + +> **parseRolloutPolicy**(`surface`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` + +Defined in: [improvement/rollout-policy.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L66) + +Parse a serialized policy surface. Defensive by design — the proposer reads + `ctx.currentSurface`, which the loop types as `string | CodeSurface`. Returns + `undefined` (never throws) for non-strings, malformed JSON, or a shape that + violates the policy's own invariants: the no-op signal. Unknown dials are + dropped; `diverse`/`temperature` ride through untouched (the proposer never + mutates them — `diverse` is a measured paired null). + +#### Parameters + +##### surface + +`MutableSurface` + +#### Returns + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` + +*** + +### normalizeRolloutPolicy() + +> **normalizeRolloutPolicy**(`raw`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` + +Defined in: [improvement/rollout-policy.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L82) + +Normalize an untyped policy bag (a parsed surface or a profile extension) into + a full `StructuralRolloutPolicy`, defaults merged. Returns `undefined` when any + present dial violates the policy invariants (mirrors `resolvePolicy`: integer + k ≥ 1, repairRounds ≥ 0, testgen ≥ 0) — a corrupt config must read as "not + configured", never as a fabricated recipe. + +#### Parameters + +##### raw + +`unknown` + +#### Returns + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` + +*** + +### serializeRolloutPolicy() + +> **serializeRolloutPolicy**(`policy`): `string` + +Defined in: [improvement/rollout-policy.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L102) + +Stable serialization — dial order is fixed so identical policies produce + identical surfaces (the loop dedupes/hashes candidates by surface content). + +#### Parameters + +##### policy + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) + +#### Returns + +`string` + +*** + +### structuralRolloutPolicyFromProfile() + +> **structuralRolloutPolicyFromProfile**(`profile`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` + +Defined in: [improvement/rollout-policy.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L115) + +Read the persisted policy off the profile. `undefined` when the profile does + not opt into structural rollout — the improve() surface no-ops then, because + tuning dials nothing consumes would ship dead config. + +#### Parameters + +##### profile + +`AgentProfile` + +#### Returns + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) \| `undefined` + +*** + +### applyRolloutPolicyToProfile() + +> **applyRolloutPolicyToProfile**(`profile`, `policy`): `AgentProfile` + +Defined in: [improvement/rollout-policy.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L125) + +Persist a policy into the profile's extensions namespace. Shallow copy; never + mutates the input profile (the applyWinnerToProfile contract). + +#### Parameters + +##### profile + +`AgentProfile` + +##### policy + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) + +#### Returns + +`AgentProfile` + +*** + +### enumerateNeighborPolicies() + +> **enumerateNeighborPolicies**(`policy`): [`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy)[] + +Defined in: [improvement/rollout-policy.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L146) + +All bounded single-dial neighbors of `policy`, in a fixed priority order: k + first (selection breadth carries 85–92% of the measured effect), then + repairRounds, then testgen. Steps clamp to the dial's bounds; clamped-to-no-op + and duplicate policies are dropped. + +#### Parameters + +##### policy + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy) + +#### Returns + +[`StructuralRolloutPolicy`](runtime.md#structuralrolloutpolicy)[] + +*** + +### rolloutPolicyProposer() + +> **rolloutPolicyProposer**(): `SurfaceProposer` + +Defined in: [improvement/rollout-policy.ts:188](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/rollout-policy.ts#L188) + +The deterministic `SurfaceProposer` for the `'rollout-policy'` surface. + +Each generation: parse the current policy surface, enumerate its bounded +single-dial neighbors, and return at most `min(populationSize, 4)` of them, +rotating the enumeration window by generation so successive generations explore +different neighbors when nothing promoted. Proposes NOTHING when the surface +carries no policy (the profile never opted in) — an empty proposal is the +loop-native no-op, mirroring `improvementDriver`'s no-findings behavior. + +#### Returns + +`SurfaceProposer` + +*** + ### createAgentKnowledgeReadinessCheck() > **createAgentKnowledgeReadinessCheck**(`options`): [`KnowledgeReadinessCheck`](#knowledgereadinesscheck) diff --git a/docs/api/mcp.md b/docs/api/mcp.md index ba9fc764..3aaadca5 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -2262,7 +2262,7 @@ Defined in: [runtime/types.ts:400](https://github.com/tangle-network/agent-runti ###### Inherited from -[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-3) +[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-4) ##### sandboxId? diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 2c3c6daa..47c41cb4 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,11 +15,12 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 231 exports. +Import from `@tangle-network/agent-runtime` — 240 exports. | Symbol | Kind | Summary | |---|---|---| | `agenticGenerator` | function | Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. | +| `applyRolloutPolicyToProfile` | function | Persist a policy into the profile's extensions namespace. Shallow copy; never | | `applyRunRecordDefaults` | function | Stamp cross-cutting defaults onto adapter-projected RunRecords without | | `auditLoopRunner` | function | `audit` mode — analyst loop over captured trace/run data. | | `buildForwardHeaders` | function | Build the headers to emit on an outbound participant call, given the | @@ -43,6 +44,7 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `defineConversation` | function | Declarative constructor for a multi-agent `Conversation`. Validates inputs | | `defineRuntimeHooks` | function | Identity helper that types a {@link RuntimeHooks} literal so the fields are inferred. | | `deriveExecutionId` | function | Derive a stable executionId from the run identity. The same | +| `enumerateNeighborPolicies` | function | All bounded single-dial neighbors of `policy`, in a fixed priority order: k | | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | @@ -58,9 +60,11 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `mcpServeVerifier` | function | Build a `Verifier` that boots a generated MCP server over stdio and checks it exposes tools. | | `mcpToolsForRuntimeMcp` | function | Returns the queue-bound delegation tools projected into OpenAI Chat | | `mcpToolsForRuntimeMcpSubset` | function | Subset filter — return only the projected tools whose `function.name` | +| `normalizeRolloutPolicy` | function | Normalize an untyped policy bag (a parsed surface or a profile extension) into | | `notifyRuntimeDecisionPoint` | function | Fire `hooks.onDecisionPoint`, swallowing sync throws and surfacing async failures to `onError`. | | `notifyRuntimeHookEvent` | function | Fire `hooks.onEvent`, swallowing sync throws and surfacing async failures to `onError`. | | `parseLoopRunnerArgv` | function | Parse `--mode X --config Y` from an argv tail (`process.argv.slice(2)`). | +| `parseRolloutPolicy` | function | Parse a serialized policy surface. Defensive by design — the proposer reads | | `rawTraceDistiller` | function | Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE | | `readDepth` | function | Read the depth counter off an inbound request. Missing → 0 (caller is the | | `readinessServerSentEvent` | function | Serialize a `KnowledgeReadinessReport` as a Server-Sent Event string. | @@ -69,6 +73,7 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `resolveAgentBackend` | function | Resolve the `AgentExecutionBackend` for the chosen `kind`. Reuse this instead | | `resolveChatModel` | function | Resolve a chat model by precedence: the first candidate carrying a | | `resolveRouterBaseUrl` | function | Resolve the router base URL from env, normalised — no trailing `/v1` or `/`. | +| `rolloutPolicyProposer` | function | The deterministic `SurfaceProposer` for the `'rollout-policy'` surface. | | `runAgentTask` | function | Single-shot task lifecycle for adapter-driven tasks: readiness-gated, emits the runtime lifecycle event vocabulary, session-store pluggable. | | `runAgentTaskStream` | function | Streaming task lifecycle: delegates execution to an `AgentExecutionBackend` (model API, sandbox, or custom iterable) and yields lifecycle events as they happen. | | `runConversation` | function | Conversation orchestrator. Drives N participants in turn through their own | @@ -85,10 +90,12 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `sanitizeKnowledgeReadinessReport` | function | Strip PII and large blobs from a `KnowledgeReadinessReport` for safe telemetry emission. | | `sanitizeRuntimeStreamEvent` | function | Reduce a `RuntimeStreamEvent` to a PII-safe, serializable plain object for telemetry. | | `selfImproveLoopRunner` | function | `self-improve` mode — agent-eval's one-call closed improvement loop (held-out gated). | +| `serializeRolloutPolicy` | function | Stable serialization — dial order is fixed so identical policies produce | | `sleep` | function | Resolve after `ms` milliseconds — used for retry backoff in conversation call policy. | | `slugifySpeaker` | function | Reduce a speaker name to ASCII alphanumerics + dashes. Preserves enough | | `startRuntimeRun` | function | Construct a runtime-run handle. The returned handle is mutable across its | | `streamToolLoop` | function | Streaming bounded tool loop: yields each raw turn event (the caller maps + | +| `structuralRolloutPolicyFromProfile` | function | Read the persisted policy off the profile. `undefined` when the profile does | | `toolBuildPrompt` | function | Build the starting instruction for a coder agent tasked with implementing a new tool. | | `turnId` | function | Deterministic turn identifier. Stable across retries of the same logical | | `validateChatModelId` | function | Validate a caller-supplied chat-model id. Rejects non-strings, malformed | @@ -100,6 +107,8 @@ Import from `@tangle-network/agent-runtime` — 231 exports. | `FORWARD_HEADERS` | const | Standard names — lowercased so Headers maps interop on every runtime. | | `INTELLIGENCE_WIRE_VERSION` | const | Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | +| `ROLLOUT_POLICY_BOUNDS` | const | Proposal bounds per dial. These are the SEARCH bounds (what the proposer may | +| `ROLLOUT_POLICY_EXTENSION` | const | The profile extensions namespace the policy persists under. | | `AgentEvalError` | class | Base class for every contract error this package throws — carries the stable | | `BackendTransportError` | class | A backend transport call (HTTP, gRPC, sidecar IPC) failed with a non-success | | `CircuitBreakerState` | class | Live circuit-breaker state — one instance per (participant, conversation run). | @@ -269,7 +278,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. ### Recursive atom + loop kernel (alias of ./runtime) -Import from `@tangle-network/agent-runtime/loops` — 432 exports. +Import from `@tangle-network/agent-runtime/loops` — 456 exports. | Symbol | Kind | Summary | |---|---|---| @@ -285,8 +294,11 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `authorStrategy` | function | Author + load a strategy from losses. Throws when the author emits no loadable module; | | `breadthStrategy` | function | BREADTH: K independent rollouts (each own artifact), verifier picks the best. | | `buildSteerContext` | function | Build the `SteerContext` a combinator reads to steer (its `loopUntil.until`, `widen` gate, any | +| `canDisplace` | function | The repair keep-best guard: a challenger displaces the incumbent only when it is | | `collectAgentTurn` | function | Drain a `streamAgentTurn` stream (or any `RuntimeStreamEvent` stream that | +| `compareCheckOutcomes` | function | The selection order: crash < ran; then official pass-fraction; authored guesses only | | `completionAuthorizes` | function | Decide whether a `CompletionVerdict` may end the node under the policy: authority scales with the verdict's determinism, and probabilistic verdicts must clear `minConfidence`. | +| `composeCheckSources` | function | Concatenate check sources (official first by convention — ordering does not affect | | `computeFindingId` | function | Compute the stable finding_id from the identity-defining fields. | | `contentAddress` | function | Mint the content-addressed `outRef` for a result artifact: `sha256:` over a | | `createAgentEnvironmentProviderRegistry` | function | Create a registry that resolves provider names to concrete provider instances. | @@ -308,6 +320,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `createWaterfallCollector` | function | Build a `WaterfallCollector` that records agent spans and renders them as an ASCII timeline. | | `createWorktreeCliExecutor` | function | Build a worktree-CLI leaf `Executor`. Per-spawn (a fresh worktree + abort + teardown each), so a | | `decodeToolPart` | function | Decode a part with a specific harness's adapter when known, else try every registered adapter | +| `defaultExtractCandidate` | function | The candidate a shot produced, read from its conversation: the LAST `submit_answer` | | `defaultSelectWinner` | function | The kernel's winner argmax — best-valid-score, ties broken by earliest index, | | `defaultToolDetectors` | function | The default online panel for a tool-call pipe: a worker repeating the same call, or hammering | | `defineLeaderboard` | function | Assemble a declarative spec (`cases` + `prompt` + `score`) into a runnable | @@ -323,6 +336,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `extractLlmCallEvent` | function | Extract a `RuntimeStreamEvent`-shaped `llm_call` from a sandbox event when | | `failuresAnalyst` | function | The default self-improvement LENS — authored content, not a code path. On each settled worker it hands | | `fanout` | function | `fanout(items, opts)` — spawn one child per item in a single round (bounded by the conserved | +| `filterAuthoredAsserts` | function | The proven authored-assert filter (lifted from the rigs' generateTests): keep only | | `finalizeBestDelivered` | function | Keep-best finalize under the completion-oracle: return the highest-scoring DELIVERED child's | | `flatWidenGate` | function | The flat default `ScopeWidenGate` — never widens, keeping the R2 selector≠judge collision | | `gateOnDeliverable` | function | Wrap an `Executor` so its settlement `valid` reflects the deliverable check, not the | @@ -339,8 +353,10 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `makeFinding` | function | Convenience factory: produce a fully-formed AnalystFinding with the | | `mapSandboxEvent` | function | Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary, | | `mapSandboxToolEvent` | function | Project one `SandboxEvent` onto the `tool_call` / `tool_result` variants of | +| `modelAuthoredChecks` | function | Default authored-check source: one metered LLM call per task, before sampling, | | `naiveDriver` | function | `naiveDriver` — the no-signal steering control. | | `observe` | function | The third-person trace analyst: read a worker's trace and produce steer findings for the next attempt plus durable `learned` facts for the cross-run corpus. | +| `officialChecksFromMeta` | function | Official checks the surface stashed on the task (e.g. MBPP's shown assert). Reads | | `openSandboxRun` | function | Open a sandbox run. Harness-agnostic: the harness lives in | | `pairwiseSignificance` | function | Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real | | `panel` | function | `panel(spec)` — spawn the M judge children over the SAME artifact, drain their settlements, | @@ -364,6 +380,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `renderReport` | function | Operator-facing report, split by who should act. The agent block is the | | `reportLoopUsage` | function | Forward a `LoopResult`'s aggregated cost + token usage into a campaign cost | | `resolveAgentEnvironmentProvider` | function | Resolve a provider instance or registry name, failing loudly when a name is unknown. | +| `resolveEntrySymbol` | function | The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface | | `resolveSandboxClient` | function | Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core | | `routerBrain` | function | The router as a supervisor BRAIN: the canonical `ToolLoopChat` seam backed by the router's | | `routerChatWithTools` | function | A router completion WITH tool-calling — the operator driver's LLM seam. Passes OpenAI-shape | @@ -375,8 +392,10 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `runLoop` | function | The round-synchronous loop kernel: each round `driver.plan()` fans N tasks to sandboxes (bounded concurrency), parses + validates each output, and folds results through `driver.decide`. | | `runPersonified` | function | Compose the persona + chosen shape onto a fresh keystone `Supervisor`. Resolves the shape | | `runStrategyEvolution` | function | Multi-generation strategy search: author candidates from tournament losses, play them against the incumbent at equal budget, promote via `promotionGate` on an untouched holdout slice. | +| `sandboxCheckRunner` | function | Default CheckRunner backend: pipes the check program into `python3` over the sandbox | | `sandboxClientAsProvider` | function | Adapt a `SandboxClient` into the shared `AgentEnvironmentProvider` contract. | | `sandboxSessionTraceSource` | function | The SANDBOX / fleet trace source: read a box session's message parts and decode the harness's tool | +| `selectBestIndex` | function | Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero | | `selectChampion` | function | Search-side champion selection over a tournament report. | | `selectValidWinner` | function | The single content-free valid-only winner selector. Among the gated-VALID children only | | `sentinelCompletion` | function | Completion for a sandbox-agent node: done iff the latest output carries the node's stop | @@ -385,6 +404,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `spendFromUsageEvents` | function | Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate | | `stopSentinel` | function | A unique, attributable stop sentinel for a node (ralph-loop style). Deterministic from the | | `streamAgentTurn` | function | Run ONE agent turn on any backend kind and stream its events. Yields the | +| `structuralRollout` | function | Build the structuralRollout `Strategy`: k shots → score each by the frozen visible | | `sumSandboxUsage` | function | Sum the token usage + USD cost of a sandbox turn's events — the one honest way to meter an | | `supervise` | function | One-call supervisor: build + run a supervisor from its profile with sensible defaults; the raw `supervisorAgent` + `createSupervisor().run` seams stay available for power use. | | `superviseSurface` | function | Drive a team of agents (spawned + steered by `profile`) to solve a graded `AgenticSurface` task, and | @@ -392,6 +412,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `supervisorInstructions` | function | The supervisor SKILL — the how-to the supervisor reads (its system prompt). THE optimizable | | `trajectoryReport` | function | Reconstruct the whole spawn tree for `root` with per-node + rolled-up `Spend`. Reads the | | `verify` | function | `verify(spec)` — an IMPLEMENT child produces a candidate, then a SEPARATE VERIFIER child grades | +| `visibleCheckScore` | function | Display scalar for receipts/reports (the rigs' `visibleScore` shape): crash = -1, | | `watchTrace` | function | Subscribe to a `TraceSource` and run the streaming detectors over its live spans. Returns an | | `widen` | function | `widen(spec)` — the streaming spawn-on-completion driver. Spawns the seed lineages, then REACTS | | `workerFromBackend` | function | Build the worker seam from a backend (WHERE workers run) + an optional completion oracle (the | @@ -404,6 +425,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `defaultAuditorInstruction` | const | Default system instruction for intent-auditor agents: diagnose diverged/drifting trajectories. | | `defaultDelegateBudget` | const | The conserved pool a `delegate()` call applies when the caller does not pass its own `budget`. | | `defaultProfileRichnessThresholds` | const | Default thresholds for `ProfileRichnessThresholds` — 600 chars / 6 lines minimum system prompt. | +| `defaultStructuralRolloutPolicy` | const | The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. | | `refine` | const | Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next attempt → repeat (deepen one lineage). | | `sample` | const | Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). | | `sampleThenRefine` | const | The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), | @@ -431,7 +453,12 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `Budget` | interface | A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. | | `BusEvent` | interface | Every bus event is a discriminated union member keyed by `type`. | | `BusRecord` | interface | A published event stamped for ordering and observability. `seq` is the monotonic publish index; | +| `CheckExecChannel` | interface | Minimal exec channel the default runner needs. `SandboxInstance` (and therefore | +| `CheckOutcome` | interface | How one candidate fared against the frozen visible checks, split by check kind. | | `CheckpointCapableBox` | interface | Loop-side widening of the box's optional checkpoint method. The | +| `CheckRunner` | interface | Executes the frozen checks against one candidate. Implementations MUST fail loud | +| `CheckSource` | interface | Produces the task's visible checks. MUST derive them from agent-visible information | +| `CheckSourceCtx` | interface | What a CheckSource composes with. `consult` is the strategy family's raw analyst | | `CollectedAgentTurn` | interface | A drained turn: the terminal summary plus every event the stream yielded. | | `CompletionAnalyst` | interface | Reads a node's trace → a completion verdict. Same input shape as the `analyze` hook, so | | `CompletionEvidence` | interface | Trace-derived evidence for a completion claim — an artifact (output) or a verifier metric, | @@ -536,6 +563,8 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `SteerContext` | interface | How a combinator's `act` consumes findings to steer — the SINGLE firewalled steer surface a | | `Strategy` | interface | A Strategy is HOW you spend the compute budget to beat the Environment's check — it | | `StrategyCtx` | interface | What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. | +| `StructuralRolloutPolicy` | interface | The rollout's compute recipe — promoted from the proven rigs' env vars (K/REPAIRS/ | +| `StructuralRolloutResult` | interface | The body's deliverable — a `StrategyResult` plus selection provenance. The extra | | `SuperviseSurfaceResult` | interface | The deployable outcome of a supervised surface run. | | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | | `SupervisorProfile` | interface | The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. | @@ -551,6 +580,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `UsageSink` | interface | The slice of an agent-eval campaign `DispatchContext.cost` this needs. | | `VerifierEnvironmentOptions` | interface | createVerifierEnvironment — ANY checkable task as an `Environment`, no tool surface | | `VerifySpec` | interface | `verify({ implement, verifier })` — the 2-node sequential gate: an IMPLEMENT child produces a | +| `VisibleCheck` | interface | One task-visible executable check (e.g. a single-line Python assert). | | `WatchTraceOptions` | interface | The ONLINE analyst: watch a `TraceSource` and fold each tool span through agent-eval's published | | `WaterfallSpan` | interface | createWaterfallCollector — 100% trajectory observability from the lifecycle stream: | | `WidenGate` | interface | The progressive-widening gate (MCTS-PW). Decides whether a settled child is | @@ -601,7 +631,7 @@ Import from `@tangle-network/agent-runtime/loops` — 432 exports. | `WinnerStrategy` | type | Built-in valid-only winner strategies for `selectValidWinner` (selector≠judge): best gated-valid | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 7f25ff74..1bf5d9ff 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -7860,7 +7860,7 @@ Default false: only facts tagged `audience:agent` are injected into the worker. ### AgenticRunResult -Defined in: [runtime/strategy.ts:594](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L594) +Defined in: [runtime/strategy.ts:608](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L608) #### Properties @@ -7868,7 +7868,7 @@ Defined in: [runtime/strategy.ts:594](https://github.com/tangle-network/agent-ru > **mode**: `string` -Defined in: [runtime/strategy.ts:596](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L596) +Defined in: [runtime/strategy.ts:610](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L610) The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). @@ -7876,25 +7876,25 @@ The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). > **score**: `number` -Defined in: [runtime/strategy.ts:597](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L597) +Defined in: [runtime/strategy.ts:611](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L611) ##### resolved > **resolved**: `boolean` -Defined in: [runtime/strategy.ts:598](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L598) +Defined in: [runtime/strategy.ts:612](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L612) ##### completions > **completions**: `number` -Defined in: [runtime/strategy.ts:599](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L599) +Defined in: [runtime/strategy.ts:613](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L613) ##### progression > **progression**: `number`[] -Defined in: [runtime/strategy.ts:601](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L601) +Defined in: [runtime/strategy.ts:615](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L615) DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-so-far per rollout. @@ -7902,13 +7902,13 @@ DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-s > **shots**: `number` -Defined in: [runtime/strategy.ts:602](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L602) +Defined in: [runtime/strategy.ts:616](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L616) ##### usd > **usd**: `number` -Defined in: [runtime/strategy.ts:605](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L605) +Defined in: [runtime/strategy.ts:619](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L619) The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: real router tokens, priced usd (0 when the model is unpriced — never fabricated), wall ms. @@ -7917,13 +7917,13 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r > **ms**: `number` -Defined in: [runtime/strategy.ts:606](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L606) +Defined in: [runtime/strategy.ts:620](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L620) ##### tokens > **tokens**: `object` -Defined in: [runtime/strategy.ts:607](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L607) +Defined in: [runtime/strategy.ts:621](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L621) ###### input @@ -7937,7 +7937,7 @@ Defined in: [runtime/strategy.ts:607](https://github.com/tangle-network/agent-ru ### Strategy -Defined in: [runtime/strategy.ts:744](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L744) +Defined in: [runtime/strategy.ts:758](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L758) A Strategy is HOW you spend the compute budget to beat the Environment's check — it builds the driver `Agent` the Supervisor runs. This is the OPEN extension point: a dev @@ -7954,7 +7954,7 @@ the reference implementations to copy: > `readonly` **name**: `string` -Defined in: [runtime/strategy.ts:745](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L745) +Defined in: [runtime/strategy.ts:759](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L759) #### Methods @@ -7962,7 +7962,7 @@ Defined in: [runtime/strategy.ts:745](https://github.com/tangle-network/agent-ru > **driver**(`surface`, `task`, `opts`, `budget`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:746](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L746) +Defined in: [runtime/strategy.ts:760](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L760) ###### Parameters @@ -7990,7 +7990,7 @@ Defined in: [runtime/strategy.ts:746](https://github.com/tangle-network/agent-ru ### ShotPersona -Defined in: [runtime/strategy.ts:776](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L776) +Defined in: [runtime/strategy.ts:790](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L790) A role for one shot — multi-agent loops (researcher + engineer, a panel of k researchers) give each shot its own system prompt and optionally its own model. @@ -8001,7 +8001,7 @@ A role for one shot — multi-agent loops (researcher + engineer, a panel of k > `optional` **systemPrompt?**: `string` -Defined in: [runtime/strategy.ts:779](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L779) +Defined in: [runtime/strategy.ts:793](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L793) Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it is injected as a hand-off message (the transcript's earlier roles stay intact). @@ -8010,7 +8010,7 @@ Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it > `optional` **model?**: `string` -Defined in: [runtime/strategy.ts:781](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L781) +Defined in: [runtime/strategy.ts:795](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L795) Per-shot model override (e.g. a stronger model for the engineer shot). @@ -8018,7 +8018,7 @@ Per-shot model override (e.g. a stronger model for the engineer shot). ### ShotSpec -Defined in: [runtime/strategy.ts:784](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L784) +Defined in: [runtime/strategy.ts:798](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L798) #### Properties @@ -8026,7 +8026,7 @@ Defined in: [runtime/strategy.ts:784](https://github.com/tangle-network/agent-ru > `optional` **handle?**: [`ArtifactHandle`](#artifacthandle) -Defined in: [runtime/strategy.ts:786](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L786) +Defined in: [runtime/strategy.ts:800](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L800) present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh one (sample/restart). @@ -8034,25 +8034,25 @@ present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh on > `optional` **messages?**: `Msg`[] -Defined in: [runtime/strategy.ts:787](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L787) +Defined in: [runtime/strategy.ts:801](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L801) ##### steer? > `optional` **steer?**: `string` -Defined in: [runtime/strategy.ts:788](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L788) +Defined in: [runtime/strategy.ts:802](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L802) ##### persona? > `optional` **persona?**: [`ShotPersona`](#shotpersona) -Defined in: [runtime/strategy.ts:789](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L789) +Defined in: [runtime/strategy.ts:803](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L803) ##### tools? > `optional` **tools?**: `string`[] -Defined in: [runtime/strategy.ts:792](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L792) +Defined in: [runtime/strategy.ts:806](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L806) Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot on the relevant capabilities. Restriction-only; unknown names throw. Omitted ⇒ all. @@ -8061,7 +8061,11 @@ Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot ### StrategyResult -Defined in: [runtime/strategy.ts:794](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L794) +Defined in: [runtime/strategy.ts:808](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L808) + +#### Extended by + +- [`StructuralRolloutResult`](#structuralrolloutresult) #### Properties @@ -8069,37 +8073,37 @@ Defined in: [runtime/strategy.ts:794](https://github.com/tangle-network/agent-ru > **score**: `number` -Defined in: [runtime/strategy.ts:795](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L795) +Defined in: [runtime/strategy.ts:809](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L809) ##### resolved > **resolved**: `boolean` -Defined in: [runtime/strategy.ts:796](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L796) +Defined in: [runtime/strategy.ts:810](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L810) ##### completions > **completions**: `number` -Defined in: [runtime/strategy.ts:797](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L797) +Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) ##### progression > **progression**: `number`[] -Defined in: [runtime/strategy.ts:798](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L798) +Defined in: [runtime/strategy.ts:812](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L812) ##### shots > **shots**: `number` -Defined in: [runtime/strategy.ts:799](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L799) +Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) *** ### StrategyCtx -Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) +Defined in: [runtime/strategy.ts:825](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L825) What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. @@ -8109,7 +8113,7 @@ What a strategy body composes with: the artifact lifecycle, the budget, and the > `readonly` **surface**: `StrategyArtifacts` -Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) +Defined in: [runtime/strategy.ts:827](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L827) Open/close artifacts the body manages itself (e.g. one persistent handle for depth). @@ -8117,25 +8121,25 @@ Open/close artifacts the body manages itself (e.g. one persistent handle for dep > `readonly` **task**: [`AgenticTask`](#agentictask) -Defined in: [runtime/strategy.ts:814](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L814) +Defined in: [runtime/strategy.ts:828](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L828) ##### opts > `readonly` **opts**: [`AgenticOptions`](#agenticoptions) -Defined in: [runtime/strategy.ts:815](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L815) +Defined in: [runtime/strategy.ts:829](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L829) ##### budget > `readonly` **budget**: `number` -Defined in: [runtime/strategy.ts:816](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L816) +Defined in: [runtime/strategy.ts:830](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L830) ##### scope > `readonly` **scope**: [`Scope`](#scope-1)\<[`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:817](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L817) +Defined in: [runtime/strategy.ts:831](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L831) #### Methods @@ -8143,7 +8147,7 @@ Defined in: [runtime/strategy.ts:817](https://github.com/tangle-network/agent-ru > **shot**(`spec?`): `Promise`\<`ShotResult` \| `null`\> -Defined in: [runtime/strategy.ts:819](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L819) +Defined in: [runtime/strategy.ts:833](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L833) Run ONE worker shot; its harness-scored result, or null if it went down. @@ -8161,7 +8165,7 @@ Run ONE worker shot; its harness-scored result, or null if it went down. > **critique**(`messages`): `Promise`\<`string` \| `null`\> -Defined in: [runtime/strategy.ts:821](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L821) +Defined in: [runtime/strategy.ts:835](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L835) The firewalled critic reads the trajectory → a steer string, or null on COMPLETE/down. @@ -8179,7 +8183,7 @@ The firewalled critic reads the trajectory → a steer string, or null on COMPLE > **consult**(`messages`, `instruction`): `Promise`\<`string` \| `null`\> -Defined in: [runtime/strategy.ts:826](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L826) +Defined in: [runtime/strategy.ts:840](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L840) The RAW analyst channel: the firewalled critic answers `instruction` over the trajectory verbatim — no findings extraction, so verdict-shaped formats @@ -8204,7 +8208,7 @@ The RAW analyst channel: the firewalled critic answers `instruction` over the > **listTools**(`handle`): `Promise`\<`object`[]\> -Defined in: [runtime/strategy.ts:830](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L830) +Defined in: [runtime/strategy.ts:844](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L844) The tools THIS artifact's task actually offers (names + descriptions only — never the implementations). Tool sets vary per task on heterogeneous domains; a strategy @@ -8224,7 +8228,7 @@ The tools THIS artifact's task actually offers (names + descriptions only — ne ### RunAgenticOptions -Defined in: [runtime/strategy.ts:1059](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1059) +Defined in: [runtime/strategy.ts:1073](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1073) #### Extends @@ -8392,19 +8396,19 @@ In-context learning: when set, query `corpus` before each depth shot and inject > **surface**: [`AgenticSurface`](#agenticsurface) -Defined in: [runtime/strategy.ts:1060](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1060) +Defined in: [runtime/strategy.ts:1074](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1074) ##### task > **task**: [`AgenticTask`](#agentictask) -Defined in: [runtime/strategy.ts:1061](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1061) +Defined in: [runtime/strategy.ts:1075](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1075) ##### hooks? > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [runtime/strategy.ts:1064](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1064) +Defined in: [runtime/strategy.ts:1078](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1078) Lifecycle observability — every spawn/settle (shots, analysts) streams here live. The seam online watchdogs/route-auditors subscribe to. @@ -8413,7 +8417,7 @@ Lifecycle observability — every spawn/settle (shots, analysts) streams here li > `optional` **strategy?**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:1066](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1066) +Defined in: [runtime/strategy.ts:1080](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1080) A Strategy (the open way) — author/pass your own. Overrides `mode` when present. @@ -8421,7 +8425,7 @@ A Strategy (the open way) — author/pass your own. Overrides `mode` when presen > `optional` **mode?**: `"depth"` \| `"breadth"` -Defined in: [runtime/strategy.ts:1068](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1068) +Defined in: [runtime/strategy.ts:1082](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1082) Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. @@ -8429,7 +8433,7 @@ Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. > **budget**: `number` -Defined in: [runtime/strategy.ts:1070](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1070) +Defined in: [runtime/strategy.ts:1084](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1084) budget: refine→max shots; sample→rollout width. @@ -8437,7 +8441,7 @@ budget: refine→max shots; sample→rollout width. > `optional` **rootBudget?**: [`Budget`](#budget-12) -Defined in: [runtime/strategy.ts:1071](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1071) +Defined in: [runtime/strategy.ts:1085](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1085) *** @@ -8539,79 +8543,542 @@ Defined in: [runtime/stream-agent-turn.ts:180](https://github.com/tangle-network > **output**: `number` -Defined in: [runtime/stream-agent-turn.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L181) +Defined in: [runtime/stream-agent-turn.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L181) + +**`Experimental`** + +##### costUsd? + +> `optional` **costUsd?**: `number` + +Defined in: [runtime/stream-agent-turn.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L182) + +**`Experimental`** + +##### model? + +> `optional` **model?**: `string` + +Defined in: [runtime/stream-agent-turn.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L183) + +**`Experimental`** + +*** + +### CollectedAgentTurn + +Defined in: [runtime/stream-agent-turn.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L193) + +**`Experimental`** + +A drained turn: the terminal summary plus every event the stream yielded. +`status`/`error` mirror the terminal `final` event so a failed or aborted +turn stays inspectable without re-scanning `events`. + +#### Properties + +##### finalText + +> **finalText**: `string` + +Defined in: [runtime/stream-agent-turn.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L194) + +**`Experimental`** + +##### usage + +> **usage**: [`AgentTurnUsage`](#agentturnusage) + +Defined in: [runtime/stream-agent-turn.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L195) + +**`Experimental`** + +##### events + +> **events**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] + +Defined in: [runtime/stream-agent-turn.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L196) + +**`Experimental`** + +##### status + +> **status**: [`AgentTaskStatus`](index.md#agenttaskstatus) + +Defined in: [runtime/stream-agent-turn.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L197) + +**`Experimental`** + +##### error? + +> `optional` **error?**: [`BackendErrorDetail`](index.md#backenderrordetail) + +Defined in: [runtime/stream-agent-turn.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L198) + +**`Experimental`** + +*** + +### StructuralRolloutPolicy + +Defined in: [runtime/structural-rollout.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L45) + +The rollout's compute recipe — promoted from the proven rigs' env vars (K/REPAIRS/ + TESTGEN/DIVERSE/TEMPERATURE). Defaults are the measured sweet spot: repair value + concentrates at low k (~+12pp at k=1, +1–3pp at k=5), so `k=5, repairRounds=2` is the + full recipe and `k=1, repairRounds=2` the low-compute preset. + +#### Properties + +##### k + +> **k**: `number` + +Defined in: [runtime/structural-rollout.ts:47](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L47) + +Independent samples per task (selection breadth). + +##### repairRounds + +> **repairRounds**: `number` + +Defined in: [runtime/structural-rollout.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L49) + +Repair shots after selection, each steered by the checks' failure output. + +##### testgen + +> **testgen**: `number` + +Defined in: [runtime/structural-rollout.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L51) + +Model-authored visible checks requested per task; 0 disables authoring. + +##### diverse? + +> `optional` **diverse?**: `boolean` + +Defined in: [runtime/structural-rollout.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L54) + +Per-slot strategy-lens prefixes on the k samples (attacks the all-k-fail bucket). + Measured as a paired null (+0.6pp) — kept as an optional knob, off by default. + +##### temperature? + +> `optional` **temperature?**: `number` + +Defined in: [runtime/structural-rollout.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L56) + +Sampling temperature for every shot of this strategy; omitted ⇒ the worker default. + +*** + +### VisibleCheck + +Defined in: [runtime/structural-rollout.ts:87](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L87) + +One task-visible executable check (e.g. a single-line Python assert). + +#### Properties + +##### code + +> **code**: `string` + +Defined in: [runtime/structural-rollout.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L88) + +##### kind + +> **kind**: `"authored"` \| `"official"` + +Defined in: [runtime/structural-rollout.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L91) + +'official' = shown in the task itself (docstring example, shown assert); + 'authored' = the model's own guess. Official outranks authored in selection. + +*** + +### CheckSourceCtx + +Defined in: [runtime/structural-rollout.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L97) + +What a CheckSource composes with. `consult` is the strategy family's raw analyst + channel (metered by the conserved pool, offline-injectable via `opts.complete`) — + check authoring goes through it rather than a bespoke model client. + +#### Properties + +##### count + +> **count**: `number` + +Defined in: [runtime/structural-rollout.ts:99](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L99) + +Authored-check budget for this task (`policy.testgen`). + +##### entrySymbol? + +> `optional` **entrySymbol?**: `string` + +Defined in: [runtime/structural-rollout.ts:102](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L102) + +The symbol authored checks must reference; undefined ⇒ authoring is skipped + (no guesses beats guesses pinned to nothing). + +#### Methods + +##### consult() + +> **consult**(`instruction`): `Promise`\<`string` \| `null`\> + +Defined in: [runtime/structural-rollout.ts:105](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L105) + +One metered LLM call: instruction in, reply text out, null when the channel went + down. The task's visible prompt is included by the channel itself. + +###### Parameters + +###### instruction + +`string` + +###### Returns + +`Promise`\<`string` \| `null`\> + +*** + +### CheckSource + +Defined in: [runtime/structural-rollout.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L111) + +Produces the task's visible checks. MUST derive them from agent-visible information + only, before any candidate exists — the strategy freezes the returned set for every + sample and repair round of the task. + +#### Methods + +##### generate() + +> **generate**(`task`, `ctx`): `Promise`\<[`VisibleCheck`](#visiblecheck)[]\> + +Defined in: [runtime/structural-rollout.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L112) + +###### Parameters + +###### task + +[`AgenticTask`](#agentictask) + +###### ctx + +[`CheckSourceCtx`](#checksourcectx) + +###### Returns + +`Promise`\<[`VisibleCheck`](#visiblecheck)[]\> + +*** + +### CheckOutcome + +Defined in: [runtime/structural-rollout.ts:205](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L205) + +How one candidate fared against the frozen visible checks, split by check kind. + +#### Properties + +##### passedOfficial + +> **passedOfficial**: `number` + +Defined in: [runtime/structural-rollout.ts:206](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L206) + +##### totalOfficial + +> **totalOfficial**: `number` + +Defined in: [runtime/structural-rollout.ts:207](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L207) + +##### passedAuthored + +> **passedAuthored**: `number` + +Defined in: [runtime/structural-rollout.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L208) + +##### totalAuthored + +> **totalAuthored**: `number` + +Defined in: [runtime/structural-rollout.ts:209](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L209) + +##### failureOutput + +> **failureOutput**: `string` + +Defined in: [runtime/structural-rollout.ts:211](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L211) + +The checks' failure report — the ONLY feedback the repair loop may see. + +##### crashed? + +> `optional` **crashed?**: `boolean` + +Defined in: [runtime/structural-rollout.ts:214](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L214) + +True when the candidate crashed before any check could run — ranks below a + candidate that ran and failed everything. + +*** + +### CheckExecChannel + +Defined in: [runtime/structural-rollout.ts:219](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L219) + +Minimal exec channel the default runner needs. `SandboxInstance` (and therefore + `ValidationCtx.box`) satisfies it structurally. + +#### Methods + +##### exec() + +> **exec**(`command`, `options?`): `Promise`\<\{ `exitCode`: `number`; `stdout`: `string`; `stderr`: `string`; \}\> + +Defined in: [runtime/structural-rollout.ts:220](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L220) + +###### Parameters + +###### command + +`string` + +###### options? + +###### timeoutMs? + +`number` + +###### Returns + +`Promise`\<\{ `exitCode`: `number`; `stdout`: `string`; `stderr`: `string`; \}\> + +*** + +### CheckRunContext + +Defined in: [runtime/structural-rollout.ts:226](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L226) + +#### Properties + +##### task + +> **task**: [`AgenticTask`](#agentictask) + +Defined in: [runtime/structural-rollout.ts:227](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L227) + +##### box? + +> `optional` **box?**: [`CheckExecChannel`](#checkexecchannel) + +Defined in: [runtime/structural-rollout.ts:229](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L229) + +Live exec channel for this run (`ValidationCtx.box` / a sandbox instance). + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +Defined in: [runtime/structural-rollout.ts:230](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L230) + +*** + +### CheckRunner + +Defined in: [runtime/structural-rollout.ts:235](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L235) + +Executes the frozen checks against one candidate. Implementations MUST fail loud + (throw) when they cannot execute — a silent zero poisons selection. + +#### Methods + +##### run() + +> **run**(`candidate`, `checks`, `ctx`): `Promise`\<[`CheckOutcome`](#checkoutcome)\> + +Defined in: [runtime/structural-rollout.ts:236](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L236) + +###### Parameters + +###### candidate + +`string` + +###### checks + +[`VisibleCheck`](#visiblecheck)[] + +###### ctx + +[`CheckRunContext`](#checkruncontext) + +###### Returns + +`Promise`\<[`CheckOutcome`](#checkoutcome)\> + +*** + +### StructuralRolloutResult + +Defined in: [runtime/structural-rollout.ts:489](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L489) + +The body's deliverable — a `StrategyResult` plus selection provenance. The extra + fields ride through `defineStrategy`'s deliverable spread onto `AgenticRunResult` + (score/resolved stay harness-verified, exactly as for every authored strategy). + +#### Extends + +- [`StrategyResult`](#strategyresult) + +#### Properties + +##### score + +> **score**: `number` + +Defined in: [runtime/strategy.ts:809](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L809) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`score`](#score-9) + +##### resolved + +> **resolved**: `boolean` + +Defined in: [runtime/strategy.ts:810](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L810) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`resolved`](#resolved-4) + +##### completions + +> **completions**: `number` + +Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`completions`](#completions-1) + +##### progression + +> **progression**: `number`[] + +Defined in: [runtime/strategy.ts:812](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L812) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`progression`](#progression-2) + +##### shots + +> **shots**: `number` + +Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) + +###### Inherited from + +[`StrategyResult`](#strategyresult).[`shots`](#shots-3) + +##### selection + +> **selection**: [`SelectionReceipt`](#selectionreceipt)[] + +Defined in: [runtime/structural-rollout.ts:492](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L492) + +One receipt per scored candidate (k samples, then repairs), `SelectionReceipt` + shaped like the kernel's (`types.ts`), selector 'driver'. -**`Experimental`** +##### repairStop -##### costUsd? +> **repairStop**: [`RepairStop`](#repairstop) -> `optional` **costUsd?**: `number` +Defined in: [runtime/structural-rollout.ts:493](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L493) -Defined in: [runtime/stream-agent-turn.ts:182](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L182) +##### officialChecks -**`Experimental`** +> **officialChecks**: `number` -##### model? +Defined in: [runtime/structural-rollout.ts:494](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L494) -> `optional` **model?**: `string` +##### authoredChecks -Defined in: [runtime/stream-agent-turn.ts:183](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L183) +> **authoredChecks**: `number` -**`Experimental`** +Defined in: [runtime/structural-rollout.ts:495](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L495) *** -### CollectedAgentTurn +### StructuralRolloutConfig -Defined in: [runtime/stream-agent-turn.ts:193](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L193) +Defined in: [runtime/structural-rollout.ts:498](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L498) -**`Experimental`** +#### Properties -A drained turn: the terminal summary plus every event the stream yielded. -`status`/`error` mirror the terminal `final` event so a failed or aborted -turn stays inspectable without re-scanning `events`. +##### policy? -#### Properties +> `optional` **policy?**: `Partial`\<[`StructuralRolloutPolicy`](#structuralrolloutpolicy)\> -##### finalText +Defined in: [runtime/structural-rollout.ts:500](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L500) -> **finalText**: `string` +Knobs; missing fields take the measured defaults (k=5, repairRounds=2, testgen=6). -Defined in: [runtime/stream-agent-turn.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L194) +##### checkSource? -**`Experimental`** +> `optional` **checkSource?**: [`CheckSource`](#checksource) -##### usage +Defined in: [runtime/structural-rollout.ts:503](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L503) -> **usage**: [`AgentTurnUsage`](#agentturnusage) +Where the visible checks come from. Default: official checks from + `task.meta.visibleChecks` composed with `modelAuthoredChecks()`. -Defined in: [runtime/stream-agent-turn.ts:195](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L195) +##### checkRunner? -**`Experimental`** +> `optional` **checkRunner?**: [`CheckRunner`](#checkrunner) -##### events +Defined in: [runtime/structural-rollout.ts:506](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L506) -> **events**: [`RuntimeStreamEvent`](index.md#runtimestreamevent)[] +How candidates are measured. Default `sandboxCheckRunner()` — it needs an exec + channel (bind one to the runner, or pass `box` here) and fails loud without one. -Defined in: [runtime/stream-agent-turn.ts:196](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L196) +##### box? -**`Experimental`** +> `optional` **box?**: [`CheckExecChannel`](#checkexecchannel) -##### status +Defined in: [runtime/structural-rollout.ts:510](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L510) -> **status**: [`AgentTaskStatus`](index.md#agenttaskstatus) +Exec channel threaded into every check run of this strategy (a sandbox instance / + `ValidationCtx.box`). The strategy seam itself carries no sandbox, so the caller + who owns one supplies it here or binds it into the runner. -Defined in: [runtime/stream-agent-turn.ts:197](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L197) +##### extractCandidate? -**`Experimental`** +> `optional` **extractCandidate?**: (`messages`) => `string` -##### error? +Defined in: [runtime/structural-rollout.ts:512](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L512) -> `optional` **error?**: [`BackendErrorDetail`](index.md#backenderrordetail) +Candidate extraction from a shot's conversation. Default `defaultExtractCandidate`. -Defined in: [runtime/stream-agent-turn.ts:198](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/stream-agent-turn.ts#L198) +###### Parameters -**`Experimental`** +###### messages + +readonly `Msg`[] + +###### Returns + +`string` *** @@ -15058,6 +15525,14 @@ any custom backend): the turn is one `backend.stream()` call. *** +### RepairStop + +> **RepairStop** = `"already-passing"` \| `"no-signal"` \| `"repaired-pass"` \| `"rounds-exhausted"` \| `"no-candidates"` + +Defined in: [runtime/structural-rollout.ts:479](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L479) + +*** + ### BudgetReadout > **BudgetReadout** = `Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> @@ -15496,7 +15971,7 @@ The compressed consumable a skill carries: everything an author needs to emit a > `const` **sample**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:755](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L755) +Defined in: [runtime/strategy.ts:769](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L769) Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). @@ -15506,7 +15981,7 @@ Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N > `const` **refine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:760](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L760) +Defined in: [runtime/strategy.ts:774](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L774) Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next attempt → repeat (deepen one lineage). @@ -15516,7 +15991,7 @@ Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next > `const` **adaptiveRefine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:960](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L960) +Defined in: [runtime/strategy.ts:974](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L974) A NEW strategy, authored from the steps (~20 lines): refine, but when a steered shot fails to improve the score it ABANDONS that line and restarts fresh (branch-when-stuck) @@ -15530,7 +16005,7 @@ A NEW strategy, authored from the steps (~20 lines): refine, but when a steered > `const` **sampleThenRefine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:1003](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1003) +Defined in: [runtime/strategy.ts:1017](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1017) The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), then refine the best-verifying line with the remaining budget. Sample's basin escape + @@ -15538,6 +16013,16 @@ The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept *** +### defaultStructuralRolloutPolicy + +> `const` **defaultStructuralRolloutPolicy**: [`StructuralRolloutPolicy`](#structuralrolloutpolicy) + +Defined in: [runtime/structural-rollout.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L60) + +The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. + +*** + ### defaultProfileRichnessThresholds > `const` **defaultProfileRichnessThresholds**: [`ProfileRichnessThresholds`](#profilerichnessthresholds) @@ -17767,7 +18252,7 @@ Multi-generation strategy search: author candidates from tournament losses, play > **depthStrategy**(`surface`, `task`, `opts`, `cfg`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:616](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L616) +Defined in: [runtime/strategy.ts:630](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L630) DEPTH: one persistent artifact, carried across analyst-steered shots. @@ -17801,7 +18286,7 @@ DEPTH: one persistent artifact, carried across analyst-steered shots. > **breadthStrategy**(`_surface`, `task`, `opts`, `cfg`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:687](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L687) +Defined in: [runtime/strategy.ts:701](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L701) BREADTH: K independent rollouts (each own artifact), verifier picks the best. @@ -17835,7 +18320,7 @@ BREADTH: K independent rollouts (each own artifact), verifier picks the best. > **defineStrategy**(`name`, `run`): [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:834](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L834) +Defined in: [runtime/strategy.ts:848](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L848) Author a Strategy from the composable steps — the open, compact way. @@ -17859,7 +18344,7 @@ Author a Strategy from the composable steps — the open, compact way. > **runAgentic**(`opts`): `Promise`\<[`AgenticRunResult`](#agenticrunresult)\> -Defined in: [runtime/strategy.ts:1075](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1075) +Defined in: [runtime/strategy.ts:1089](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1089) Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved-budget Scope. @@ -17935,6 +18420,306 @@ event — a stream that violates the contract must not read as an empty turn. *** +### filterAuthoredAsserts() + +> **filterAuthoredAsserts**(`reply`, `entrySymbol`, `count`): `string`[] + +Defined in: [runtime/structural-rollout.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L125) + +The proven authored-assert filter (lifted from the rigs' generateTests): keep only + single-line, paren-balanced asserts that reference the entry symbol — malformed lines + are dropped here rather than poisoning every candidate's score identically. + +#### Parameters + +##### reply + +`string` + +##### entrySymbol + +`string` + +##### count + +`number` + +#### Returns + +`string`[] + +*** + +### modelAuthoredChecks() + +> **modelAuthoredChecks**(`overrides?`): [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L149) + +Default authored-check source: one metered LLM call per task, before sampling, + filtered through `filterAuthoredAsserts`. Returns [] (no signal, never a fabricated + check) when the budget is 0, no entry symbol resolves, or the channel went down. + +#### Parameters + +##### overrides? + +###### count? + +`number` + +#### Returns + +[`CheckSource`](#checksource) + +*** + +### officialChecksFromMeta() + +> **officialChecksFromMeta**(`key?`): [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L167) + +Official checks the surface stashed on the task (e.g. MBPP's shown assert). Reads + `task.meta[key]` as a string array; anything else means no official checks. + +#### Parameters + +##### key? + +`string` = `'visibleChecks'` + +#### Returns + +[`CheckSource`](#checksource) + +*** + +### composeCheckSources() + +> **composeCheckSources**(...`sources`): [`CheckSource`](#checksource) + +Defined in: [runtime/structural-rollout.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L181) + +Concatenate check sources (official first by convention — ordering does not affect + scoring, which reads each check's `kind`). + +#### Parameters + +##### sources + +...[`CheckSource`](#checksource)[] + +#### Returns + +[`CheckSource`](#checksource) + +*** + +### resolveEntrySymbol() + +> **resolveEntrySymbol**(`task`): `string` \| `undefined` + +Defined in: [runtime/structural-rollout.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L194) + +The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface + provides it, else the LAST `def name(` in the visible prompt (a code-completion stub + lists helpers first, the entry stub last). Undefined ⇒ authoring is skipped. + +#### Parameters + +##### task + +[`AgenticTask`](#agentictask) + +#### Returns + +`string` \| `undefined` + +*** + +### sandboxCheckRunner() + +> **sandboxCheckRunner**(`options?`): [`CheckRunner`](#checkrunner) + +Defined in: [runtime/structural-rollout.ts:279](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L279) + +Default CheckRunner backend: pipes the check program into `python3` over the sandbox + exec channel (`ctx.box`, or one bound at construction). Never shells out to docker + itself — the jail is the sandbox's concern. No channel ⇒ throws; it must never + silently score 0. Empty check sets short-circuit to a no-signal outcome (nothing to + execute, so no channel is required). + +#### Parameters + +##### options? + +###### box? + +[`CheckExecChannel`](#checkexecchannel) + +###### python? + +`string` + +###### timeoutMs? + +`number` + +#### Returns + +[`CheckRunner`](#checkrunner) + +*** + +### compareCheckOutcomes() + +> **compareCheckOutcomes**(`a`, `b`): `number` + +Defined in: [runtime/structural-rollout.ts:347](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L347) + +The selection order: crash < ran; then official pass-fraction; authored guesses only + break ties. Returns > 0 when `a` outranks `b`. Strictly lexicographic — on MBPP, + letting 6 noisy guesses outvote the one official check flipped selection negative. + +#### Parameters + +##### a + +[`CheckOutcome`](#checkoutcome) + +##### b + +[`CheckOutcome`](#checkoutcome) + +#### Returns + +`number` + +*** + +### visibleCheckScore() + +> **visibleCheckScore**(`o`): `number` + +Defined in: [runtime/structural-rollout.ts:360](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L360) + +Display scalar for receipts/reports (the rigs' `visibleScore` shape): crash = -1, + else official fraction + 0.001 × authored fraction. Selection itself uses the exact + lexicographic comparator, never this scalar. + +#### Parameters + +##### o + +[`CheckOutcome`](#checkoutcome) + +#### Returns + +`number` + +*** + +### selectBestIndex() + +> **selectBestIndex**(`outcomes`): `number` + +Defined in: [runtime/structural-rollout.ts:367](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L367) + +Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero + visible coverage every candidate ties at no-signal and index 0 is the blind pick). + +#### Parameters + +##### outcomes + +readonly [`CheckOutcome`](#checkoutcome)[] + +#### Returns + +`number` + +*** + +### canDisplace() + +> **canDisplace**(`challenger`, `incumbent`): `boolean` + +Defined in: [runtime/structural-rollout.ts:382](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L382) + +The repair keep-best guard: a challenger displaces the incumbent only when it is + strictly better in the selection order AND passes at least as many official checks. + The raw-count clause is deliberate belt-and-braces over the comparator (a custom + runner can report shifted totals): repair must NEVER replace a candidate that passes + more official checks with one that passes fewer. + +#### Parameters + +##### challenger + +[`CheckOutcome`](#checkoutcome) + +##### incumbent + +[`CheckOutcome`](#checkoutcome) + +#### Returns + +`boolean` + +*** + +### defaultExtractCandidate() + +> **defaultExtractCandidate**(`messages`): `string` + +Defined in: [runtime/structural-rollout.ts:403](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L403) + +The candidate a shot produced, read from its conversation: the LAST `submit_answer` + tool-call argument (verifier environments submit the artifact explicitly), else the + latest assistant reply's fenced code block — preferring a block containing a `def`, + because repair replies echo the failure report in a bare fence BEFORE the fixed code + (the rigs' extractRepairCode lesson) — else the latest non-empty assistant text. + +#### Parameters + +##### messages + +readonly `Msg`[] + +#### Returns + +`string` + +*** + +### structuralRollout() + +> **structuralRollout**(`config?`): [`Strategy`](#strategy-3) + +Defined in: [runtime/structural-rollout.ts:525](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/structural-rollout.ts#L525) + +Build the structuralRollout `Strategy`: k shots → score each by the frozen visible +checks (official above authored, crash lowest) → argmax with first-index tie-break → +up to `repairRounds` repair shots steered by the failure output, keep-best under the +official-check guard. Authored via `defineStrategy`, so the deliverable score stays +harness-verified and every shot is metered by the conserved pool. + +Budget note: `runAgentic`'s `budget` sizes the pool — pass at least +`k + repairRounds + 1` so the samples, repairs, and the check-author consult all admit. + +#### Parameters + +##### config? + +[`StructuralRolloutConfig`](#structuralrolloutconfig) = `{}` + +#### Returns + +[`Strategy`](#strategy-3) + +*** + ### failuresAnalyst() > **failuresAnalyst**(): [`AnalystRegistry`](#analystregistry) diff --git a/docs/design/structural-rollout-integration.md b/docs/design/structural-rollout-integration.md new file mode 100644 index 00000000..52fc6946 --- /dev/null +++ b/docs/design/structural-rollout-integration.md @@ -0,0 +1,45 @@ +# Structural rollout policy — integration design + +Status: design accepted 2026-07-09; measured basis in `supervisor-lab/docs/results/structural-lever-humaneval.md`. + +## Measured basis + +Best-of-k selection + self-repair, grounded ONLY on task-visible checks (shown examples + model-authored asserts), graded on hidden tests: + +| model × dataset | baseline → full loop | lift | p (exact sign test) | +|---|---|---|---| +| Llama-3-8B × MBPP (n=427) | 51.8% → 73.1% | **+21.3pp** | 2.3e-51 (+226/−13) | +| Llama-3-8B × HumanEval (n=164) | 43.9% → 62.2% | +18.3pp | 9.2e-11 | +| Qwen2.5-7B × HumanEval (n=164) | 82.4% → 91.5% | +9.0pp | 1.0e-8 | +| Qwen2.5-7B × MBPP (n=427) | 76.7% → 85.2% | +8.5pp | 4.6e-16 | +| glm-4.5-air / glm-5.2 × HumanEval (saturated 99.4/99.7%) | null | −0.6 / −0.4pp | the only regressions are the two calibration-flagged wrong-example tasks (/47, /116) | + +Every positive cell captures ≥93% of the pass@k bound. Selection is 85–92% of the effect; repair is a small always-positive increment. Prompt-diversity per slot is a paired null (+0.6pp). Independently verified: 1,968/1,968 regrade cells, 328/328 selection replays, 0 hidden-test leaks. + +## The finding that shapes the design + +The runtime already owns five of the six pieces: +1. best-of-k + repair strategy family — `src/runtime/strategy.ts` `sample`/`refine`/`sampleThenRefine` (:755/:760/:1003), authored via `defineStrategy` (:834) +2. jailed check execution — `createVerifierEnvironment` (`src/runtime/verifier-environment.ts:68`), agent-eval `testJudge`/`runJudgeFleet`, `@tangle-network/sandbox` — retire the bench rigs' bespoke `docker run` jails +3. selection + audit — `defaultSelectWinner` + `SelectionReceipt` (`src/runtime/run-loop.ts:1131`, `types.ts:160`) +4. visible/hidden firewall as typed field routing — agent-eval `FieldDestination`: `develop-against` (the visible-check source) vs `grading-only`, with `assertNoHiddenLeak` + `gradeOnHidden` +5. config/knob plumbing — `budget` on runBenchmark/runAgentic/superviseSurface; `directives.ts` for slot prefixes + +The ONLY net-new seam: **the model authors its own visible checks** (`CheckSource`). + +## Design (no caller changes) + +New module `src/runtime/structural-rollout.ts`: +- `CheckSource` — `generate(task, ctx) → VisibleCheck[]` from agent-visible/develop-against fields only. Default impl lifted from the proven `bench/src/hev-structural.mts generateTests` (:282) with the MBPP lesson baked in: **official shown examples outrank model-authored guesses in scoring** (guesses are 17–70% wrong depending on model × spec richness; unweighted they can flip selection negative). +- `CheckRunner` — `run(candidate, checks, ctx) → { passed, total, failureOutput }`, backend = sandbox exec / agent-eval `testJudge`, result shaped to `SurfaceScore`. +- `structuralRollout({ policy, checkSource, checkRunner }) → Strategy` — a fourth member of the sample/refine family via `defineStrategy`; argmax by weighted visible score, ≤`repairRounds` repair shots steered by `failureOutput`, keep-best-by-score. Emits `SelectionReceipt`s. +- `StructuralRolloutPolicy { k, repairRounds, testgen, diverse?, temperature? }` — promoted from the rig env vars; later an optimizable surface for `improve()`. + +Placement rule: this is an INFERENCE-TIME capability (wraps the model call). It does not go into `improve()`/`selfImprove` (training-time); `improve()` may later tune the policy knobs. + +Extend-don't-fork list: strategy family, verifier-environment, selectWinner/receipts, agent-eval field routing + judges, the rigs' `generateTests`/`extractRepairCode` as default impls. + +## Known behavior to preserve/handle +- Wrong visible examples poison repair at saturation (glm regressions on /47,/116): repair must never replace a candidate that passes MORE official checks with one that passes fewer; consider a no-repair-when-only-defect-signal guard. +- Repair value concentrates at low k (~+12pp at k=1, +1–3pp at k=5): default policy `k=5, repairRounds=2, testgen=6`; low-compute preset `k=1, repairRounds=2`. +- Exact-equality float asserts are a known wrong-test class (HumanEval/2 case). diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 312b08de..cfcd9b07 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -12,6 +12,11 @@ * * - `surface: 'prompt'` → `gepaProposer` mutates `profile.prompt.systemPrompt`. * - `surface: 'skills'` → `skillOptProposer` mutates a skills document string. + * - `surface: 'rollout-policy'` → `rolloutPolicyProposer` mutates the + * inference-time `StructuralRolloutPolicy` dials ({ k, repairRounds, testgen }) + * persisted in `profile.extensions['structural-rollout']` — deterministic + * bounded neighbor enumeration; the held-out gate does the deciding. No-op + * (nothing proposed, nothing shipped) when the profile has no such extension. * - `surface` ∈ {`tools`, `mcp`, `hooks`, `code`} → no zero-config default * proposer exists (a code/config proposer needs caller-supplied wiring — a * worktree repo root, a candidate generator, a serializer). The facade @@ -49,10 +54,26 @@ import { assertModelAllowed } from '../runtime/supervise/model-policy' import { agenticGenerator, type Verifier } from './agentic-generator' import { type CandidateGenerator, improvementDriver } from './improvement-driver' import { rawTraceDistiller } from './raw-trace-distiller' +import { + applyRolloutPolicyToProfile, + normalizeRolloutPolicy, + rolloutPolicyProposer, + serializeRolloutPolicy, + structuralRolloutPolicyFromProfile, +} from './rollout-policy' /** The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law - * profile levers; `code` is the implementation-tier surface. */ -export type ImproveSurface = 'prompt' | 'skills' | 'tools' | 'mcp' | 'hooks' | 'code' + * profile levers; `code` is the implementation-tier surface, `rollout-policy` + * the inference-time structuralRollout dials + * (`profile.extensions['structural-rollout']`). */ +export type ImproveSurface = + | 'prompt' + | 'skills' + | 'tools' + | 'mcp' + | 'hooks' + | 'code' + | 'rollout-policy' export interface ImproveOptions { /** Which profile lever to optimize. Default `'prompt'`. Selects the default @@ -188,6 +209,9 @@ function defaultGeneratorFor( return gepaProposer({ llm: llmClientOptions(llm), model, target: 'agent system prompt' }) case 'skills': return skillOptProposer({ llm: llmClientOptions(llm), model, target: 'agent skill document' }) + case 'rollout-policy': + // Deterministic bounded enumeration — no LLM, so `llm` is unused here. + return rolloutPolicyProposer() default: return undefined } @@ -214,6 +238,13 @@ function baselineSurfaceFor( return JSON.stringify(profile.mcp ?? {}) case 'hooks': return JSON.stringify(profile.hooks ?? {}) + case 'rollout-policy': { + // Empty surface when the profile never opted into structural rollout: the + // proposer reads it as "propose nothing", so the loop runs baseline-only and + // holds — tuning dials nothing consumes would ship dead config. + const policy = structuralRolloutPolicyFromProfile(profile) + return policy ? serializeRolloutPolicy(policy) : '' + } case 'code': // A code surface is produced by the caller's generator from a worktree; // the facade has no worktree ref to seed, so the baseline is the empty @@ -337,6 +368,18 @@ function applyWinnerToProfile( return { ...profile, mcp: parseWinnerJson(winner, surface) } case 'hooks': return { ...profile, hooks: parseWinnerJson(winner, surface) } + case 'rollout-policy': { + // Parse + re-validate the winner against the policy's own invariants — a + // custom generator's malformed dial must fail loud, not persist silently. + const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface)) + if (!policy) { + throw new ConfigError( + `improve(): the shipped 'rollout-policy' winner is not a valid StructuralRolloutPolicy ` + + `(integer k >= 1, repairRounds >= 0, testgen >= 0), so it cannot be applied: ${winner}`, + ) + } + return applyRolloutPolicyToProfile(profile, policy) + } case 'code': return profile } diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 28f56598..321cef8d 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -38,3 +38,14 @@ export { rawTraceDistiller, } from './raw-trace-distiller' export { type ReflectiveGeneratorOptions, reflectiveGenerator } from './reflective-generator' +export { + applyRolloutPolicyToProfile, + enumerateNeighborPolicies, + normalizeRolloutPolicy, + parseRolloutPolicy, + ROLLOUT_POLICY_BOUNDS, + ROLLOUT_POLICY_EXTENSION, + rolloutPolicyProposer, + serializeRolloutPolicy, + structuralRolloutPolicyFromProfile, +} from './rollout-policy' diff --git a/src/improvement/rollout-policy.test.ts b/src/improvement/rollout-policy.test.ts new file mode 100644 index 00000000..b2fc8985 --- /dev/null +++ b/src/improvement/rollout-policy.test.ts @@ -0,0 +1,265 @@ +/** + * `'rollout-policy'` surface proof. + * + * What this guards: `improve()` can tune the inference-time structuralRollout + * dials { k, repairRounds, testgen } through agent-eval's generic string-surface + * contract — deterministic bounded candidate enumeration, held-out gate deciding, + * winner persisted into `profile.extensions['structural-rollout']`, and a strict + * no-op when the profile never opted into structural rollout. + * + * Deterministic and offline throughout: the proposer is enumeration (no LLM), the + * judge is a pure function of the candidate policy's `k` dial, and the stub agent + * reports token-bearing cost so the backend-integrity guard sees a real backend. + */ + +import { isProposedCandidate, type ProposedCandidate } from '@tangle-network/agent-eval/campaign' +import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import type { StructuralRolloutPolicy } from '../runtime/structural-rollout' +import { improve } from './improve' +import { + applyRolloutPolicyToProfile, + enumerateNeighborPolicies, + normalizeRolloutPolicy, + parseRolloutPolicy, + ROLLOUT_POLICY_BOUNDS, + ROLLOUT_POLICY_EXTENSION, + rolloutPolicyProposer, + serializeRolloutPolicy, + structuralRolloutPolicyFromProfile, +} from './rollout-policy' + +const inBounds = (p: StructuralRolloutPolicy) => + p.k >= ROLLOUT_POLICY_BOUNDS.k.min && + p.k <= ROLLOUT_POLICY_BOUNDS.k.max && + p.repairRounds >= ROLLOUT_POLICY_BOUNDS.repairRounds.min && + p.repairRounds <= ROLLOUT_POLICY_BOUNDS.repairRounds.max && + p.testgen >= ROLLOUT_POLICY_BOUNDS.testgen.min && + p.testgen <= ROLLOUT_POLICY_BOUNDS.testgen.max + +describe('enumerateNeighborPolicies — bounded single-dial moves', () => { + it('emits all six in-bounds neighbors of an interior policy, none equal to it', () => { + const base: StructuralRolloutPolicy = { k: 5, repairRounds: 2, testgen: 6 } + const neighbors = enumerateNeighborPolicies(base) + expect(neighbors.map((n) => [n.k, n.repairRounds, n.testgen])).toEqual([ + [7, 2, 6], + [3, 2, 6], + [5, 3, 6], + [5, 1, 6], + [5, 2, 9], + [5, 2, 3], + ]) + for (const n of neighbors) expect(inBounds(n)).toBe(true) + expect(neighbors.map(serializeRolloutPolicy)).not.toContain(serializeRolloutPolicy(base)) + }) + + it('clamps at the floor: downward moves that clamp to a no-op are dropped', () => { + const neighbors = enumerateNeighborPolicies({ k: 1, repairRounds: 0, testgen: 0 }) + expect(neighbors.map((n) => [n.k, n.repairRounds, n.testgen])).toEqual([ + [3, 0, 0], + [1, 1, 0], + [1, 0, 3], + ]) + for (const n of neighbors) expect(inBounds(n)).toBe(true) + }) + + it('clamps at the ceiling: upward moves that clamp to a no-op are dropped', () => { + const neighbors = enumerateNeighborPolicies({ k: 10, repairRounds: 3, testgen: 10 }) + expect(neighbors.map((n) => [n.k, n.repairRounds, n.testgen])).toEqual([ + [8, 3, 10], + [10, 2, 10], + [10, 3, 7], + ]) + for (const n of neighbors) expect(inBounds(n)).toBe(true) + }) + + it('a clamped move that lands in-bounds survives (k=2 reaches the k=1 preset)', () => { + const neighbors = enumerateNeighborPolicies({ k: 2, repairRounds: 0, testgen: 0 }) + expect(neighbors.map((n) => n.k)).toContain(1) + for (const n of neighbors) expect(inBounds(n)).toBe(true) + }) + + it('never mutates diverse/temperature — they ride through every neighbor', () => { + const neighbors = enumerateNeighborPolicies({ + k: 5, + repairRounds: 2, + testgen: 6, + diverse: true, + temperature: 0.7, + }) + for (const n of neighbors) { + expect(n.diverse).toBe(true) + expect(n.temperature).toBe(0.7) + } + }) +}) + +describe('rolloutPolicyProposer — deterministic bounded proposal', () => { + const ctx = (overrides: Record = {}) => + ({ + currentSurface: serializeRolloutPolicy({ k: 5, repairRounds: 2, testgen: 6 }), + history: [], + findings: [], + populationSize: 4, + generation: 0, + signal: new AbortController().signal, + ...overrides, + }) as never + + const asProposed = (proposals: Array): ProposedCandidate[] => + proposals.filter((p): p is ProposedCandidate => isProposedCandidate(p as ProposedCandidate)) + + it('caps at min(populationSize, 4) candidates, every surface a valid in-bounds policy', async () => { + const raw = await rolloutPolicyProposer().propose(ctx()) + const proposals = asProposed(raw) + // Every proposal carries its {label, rationale} — never a bare surface. + expect(proposals.length).toBe(raw.length) + expect(proposals.length).toBe(4) + for (const p of proposals) { + const parsed = parseRolloutPolicy(p.surface) + expect(parsed).toBeDefined() + expect(inBounds(parsed as StructuralRolloutPolicy)).toBe(true) + expect(p.label).toMatch(/^(k|repairRounds|testgen) \d+→\d+$/) + expect(p.rationale.length).toBeGreaterThan(0) + } + const two = await rolloutPolicyProposer().propose(ctx({ populationSize: 2 })) + expect(two.length).toBe(2) + }) + + it('rotates the neighbor window by generation, so a held generation explores differently', async () => { + const gen0 = asProposed(await rolloutPolicyProposer().propose(ctx({ generation: 0 }))) + const gen1 = asProposed(await rolloutPolicyProposer().propose(ctx({ generation: 1 }))) + expect(new Set(gen0.map((p) => p.surface))).not.toEqual(new Set(gen1.map((p) => p.surface))) + }) + + it('proposes nothing when the surface carries no policy (empty, malformed, or code-tier)', async () => { + const proposer = rolloutPolicyProposer() + expect(await proposer.propose(ctx({ currentSurface: '' }))).toEqual([]) + expect(await proposer.propose(ctx({ currentSurface: 'not json' }))).toEqual([]) + expect(await proposer.propose(ctx({ currentSurface: '{"k": 0}' }))).toEqual([]) + expect(await proposer.propose(ctx({ currentSurface: { worktreeRef: 'refs/x' } }))).toEqual([]) + }) +}) + +describe('policy persistence — profile extensions round-trip', () => { + it('applyRolloutPolicyToProfile → structuralRolloutPolicyFromProfile is identity', () => { + const policy: StructuralRolloutPolicy = { + k: 3, + repairRounds: 1, + testgen: 0, + diverse: false, + temperature: 0.2, + } + const profile: AgentProfile = { name: 'fixture' } + const next = applyRolloutPolicyToProfile(profile, policy) + expect(structuralRolloutPolicyFromProfile(next)).toEqual(policy) + // Never mutates the input profile. + expect(profile.extensions).toBeUndefined() + }) + + it('a partial extension resolves the missing dials to the measured defaults', () => { + const profile: AgentProfile = { + extensions: { [ROLLOUT_POLICY_EXTENSION]: { k: 1 } }, + } + expect(structuralRolloutPolicyFromProfile(profile)).toEqual({ + k: 1, + repairRounds: 2, + testgen: 6, + }) + }) + + it('a corrupt extension reads as not-configured, never a fabricated recipe', () => { + for (const bad of [{ k: 0 }, { k: 1.5 }, { repairRounds: -1 }, { testgen: 'six' }]) { + const profile: AgentProfile = { + extensions: { [ROLLOUT_POLICY_EXTENSION]: bad as never }, + } + expect(structuralRolloutPolicyFromProfile(profile)).toBeUndefined() + } + expect(normalizeRolloutPolicy(null)).toBeUndefined() + expect(normalizeRolloutPolicy([1, 2])).toBeUndefined() + }) +}) + +// ── improve() end-to-end: the gate decides, the winner persists ───────────────────── + +// Eight scenarios at holdoutFraction 0.5 → 4 train + 4 holdout: enough held-out +// cells for the gate statistic on a deterministic score gradient. +const scenarios: Scenario[] = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((id) => ({ + id, + kind: 'fixture', +})) + +// The agent parses the policy surface into the artifact; the judge rewards the k +// dial (composite = k/10). Baseline k=5 → 0.5; the k=7 neighbor → 0.7 on every +// scenario: a +0.2 held-out lift the default gate (deltaThreshold 0.05) ships. +async function policyAgent( + surface: unknown, + _scenario: Scenario, + ctx: DispatchContext, +): Promise<{ policy: StructuralRolloutPolicy | undefined }> { + ctx.cost.observe(0.0001, 'stub-agent') + ctx.cost.observeTokens({ input: 1, output: 1 }) + return { policy: parseRolloutPolicy(String(surface)) } +} + +const kJudge: JudgeConfig<{ policy: StructuralRolloutPolicy | undefined }, Scenario> = { + name: 'k-dial-judge', + dimensions: [{ key: 'q', description: 'rewards selection breadth' }], + score: ({ artifact }) => { + const composite = (artifact.policy?.k ?? 0) / 10 + return { dimensions: { q: composite }, composite, notes: '' } + }, +} + +describe("improve() surface 'rollout-policy'", () => { + it('a gated win persists the winning policy into profile.extensions', async () => { + const profile: AgentProfile = { + name: 'fixture-agent', + extensions: { + [ROLLOUT_POLICY_EXTENSION]: { k: 5, repairRounds: 2, testgen: 6 }, + }, + } + const result = await improve(profile, [], { + surface: 'rollout-policy', + scenarios, + judge: kJudge, + agent: policyAgent, + budget: { generations: 1, populationSize: 4, holdoutFraction: 0.5 }, + }) + + expect(result.gateDecision).toBe('ship') + expect(result.shipped).toBe(true) + expect(result.lift).toBeCloseTo(0.2, 5) + // The k=7 neighbor won and was written back where the runtime reads it. + expect(structuralRolloutPolicyFromProfile(result.profile)).toEqual({ + k: 7, + repairRounds: 2, + testgen: 6, + }) + // Input profile untouched (applyWinnerToProfile is copy-on-write). + expect(structuralRolloutPolicyFromProfile(profile)).toEqual({ + k: 5, + repairRounds: 2, + testgen: 6, + }) + }) + + it('no-ops when the profile has no structural rollout config', async () => { + const profile: AgentProfile = { name: 'fixture-agent', prompt: { systemPrompt: 'base' } } + const result = await improve(profile, [], { + surface: 'rollout-policy', + scenarios, + judge: kJudge, + agent: policyAgent, + budget: { generations: 1, populationSize: 4, holdoutFraction: 0.5 }, + }) + + // The proposer proposed nothing, so nothing could ship; the exact same + // profile object comes back (no fabricated extension, no dial changes). + expect(result.shipped).toBe(false) + expect(result.gateDecision).toBe('hold') + expect(result.profile).toBe(profile) + expect(structuralRolloutPolicyFromProfile(result.profile)).toBeUndefined() + }) +}) diff --git a/src/improvement/rollout-policy.ts b/src/improvement/rollout-policy.ts new file mode 100644 index 00000000..05a6edc5 --- /dev/null +++ b/src/improvement/rollout-policy.ts @@ -0,0 +1,212 @@ +/** + * `rolloutPolicyProposer` — the `'rollout-policy'` surface for `improve()`: the + * inference-time `StructuralRolloutPolicy` dials { k, repairRounds, testgen } as a + * held-out-gated optimizable surface. + * + * Why this seam: agent-eval's loop contract is already generic — `MutableSurface` + * admits any string, documented as "serialized tool config" — so the policy rides + * the SAME serialize→propose→gate→parse-back cycle the tools/mcp/hooks surfaces + * use. No agent-eval changes; the only net-new piece is this proposer. + * + * Why deterministic: prompt-wording proposals are a measured zero on this stack, + * and the policy space is tiny and fully enumerable. The proposer emits bounded + * single-dial neighbors (k±2 in [1,10], repairRounds±1 in [0,3], testgen±3 in + * [0,10], ≤4 per generation) and lets the held-out gate do ALL the deciding — an + * LLM proposer would add cost and nondeterminism with nothing to reason about. + * + * Persistence: the policy lives in `profile.extensions['structural-rollout']` + * (AgentProfile's designed slot for runtime-specific config). A gated winner is + * written back there by `improve()`, the same profile-field write-back every other + * config surface gets; `structuralRolloutPolicyFromProfile` is the read side a + * runtime caller feeds to `structuralRollout({ policy })`. + * + * @experimental + */ + +import type { + MutableSurface, + ProposeContext, + ProposedCandidate, + SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + defaultStructuralRolloutPolicy, + type StructuralRolloutPolicy, +} from '../runtime/structural-rollout' + +/** The profile extensions namespace the policy persists under. */ +export const ROLLOUT_POLICY_EXTENSION = 'structural-rollout' + +/** Proposal bounds per dial. These are the SEARCH bounds (what the proposer may + * explore), chosen so every reachable value is a measured-sane recipe: k=1 is the + * low-compute preset, testgen=0 disables check authoring, repairRounds caps where + * the measured increment flattens (+1–3pp beyond round 2). */ +export const ROLLOUT_POLICY_BOUNDS = { + k: { min: 1, max: 10, step: 2 }, + repairRounds: { min: 0, max: 3, step: 1 }, + testgen: { min: 0, max: 10, step: 3 }, +} as const + +/** Max candidates per generation — the search space is 3 dials, so a small + * neighborhood per generation converges without burning gate budget. */ +const MAX_CANDIDATES_PER_GENERATION = 4 + +const clamp = (v: number, min: number, max: number) => Math.min(max, Math.max(min, v)) + +const isBoundedInt = (v: unknown, min: number): v is number => + typeof v === 'number' && Number.isInteger(v) && v >= min + +/** Parse a serialized policy surface. Defensive by design — the proposer reads + * `ctx.currentSurface`, which the loop types as `string | CodeSurface`. Returns + * `undefined` (never throws) for non-strings, malformed JSON, or a shape that + * violates the policy's own invariants: the no-op signal. Unknown dials are + * dropped; `diverse`/`temperature` ride through untouched (the proposer never + * mutates them — `diverse` is a measured paired null). */ +export function parseRolloutPolicy(surface: MutableSurface): StructuralRolloutPolicy | undefined { + if (typeof surface !== 'string' || surface.trim().length === 0) return undefined + let raw: unknown + try { + raw = JSON.parse(surface) + } catch { + return undefined + } + return normalizeRolloutPolicy(raw) +} + +/** Normalize an untyped policy bag (a parsed surface or a profile extension) into + * a full `StructuralRolloutPolicy`, defaults merged. Returns `undefined` when any + * present dial violates the policy invariants (mirrors `resolvePolicy`: integer + * k ≥ 1, repairRounds ≥ 0, testgen ≥ 0) — a corrupt config must read as "not + * configured", never as a fabricated recipe. */ +export function normalizeRolloutPolicy(raw: unknown): StructuralRolloutPolicy | undefined { + if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) return undefined + const bag = raw as Record + const k = bag.k ?? defaultStructuralRolloutPolicy.k + const repairRounds = bag.repairRounds ?? defaultStructuralRolloutPolicy.repairRounds + const testgen = bag.testgen ?? defaultStructuralRolloutPolicy.testgen + if (!isBoundedInt(k, 1) || !isBoundedInt(repairRounds, 0) || !isBoundedInt(testgen, 0)) { + return undefined + } + return { + k, + repairRounds, + testgen, + ...(typeof bag.diverse === 'boolean' ? { diverse: bag.diverse } : {}), + ...(typeof bag.temperature === 'number' ? { temperature: bag.temperature } : {}), + } +} + +/** Stable serialization — dial order is fixed so identical policies produce + * identical surfaces (the loop dedupes/hashes candidates by surface content). */ +export function serializeRolloutPolicy(policy: StructuralRolloutPolicy): string { + return JSON.stringify({ + k: policy.k, + repairRounds: policy.repairRounds, + testgen: policy.testgen, + ...(policy.diverse !== undefined ? { diverse: policy.diverse } : {}), + ...(policy.temperature !== undefined ? { temperature: policy.temperature } : {}), + }) +} + +/** Read the persisted policy off the profile. `undefined` when the profile does + * not opt into structural rollout — the improve() surface no-ops then, because + * tuning dials nothing consumes would ship dead config. */ +export function structuralRolloutPolicyFromProfile( + profile: AgentProfile, +): StructuralRolloutPolicy | undefined { + const bag = profile.extensions?.[ROLLOUT_POLICY_EXTENSION] + if (bag === undefined) return undefined + return normalizeRolloutPolicy(bag) +} + +/** Persist a policy into the profile's extensions namespace. Shallow copy; never + * mutates the input profile (the applyWinnerToProfile contract). */ +export function applyRolloutPolicyToProfile( + profile: AgentProfile, + policy: StructuralRolloutPolicy, +): AgentProfile { + const bag: Record = { + k: policy.k, + repairRounds: policy.repairRounds, + testgen: policy.testgen, + ...(policy.diverse !== undefined ? { diverse: policy.diverse } : {}), + ...(policy.temperature !== undefined ? { temperature: policy.temperature } : {}), + } + return { + ...profile, + extensions: { ...profile.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }, + } +} + +/** All bounded single-dial neighbors of `policy`, in a fixed priority order: k + * first (selection breadth carries 85–92% of the measured effect), then + * repairRounds, then testgen. Steps clamp to the dial's bounds; clamped-to-no-op + * and duplicate policies are dropped. */ +export function enumerateNeighborPolicies( + policy: StructuralRolloutPolicy, +): StructuralRolloutPolicy[] { + const moves: Array<{ dial: 'k' | 'repairRounds' | 'testgen'; delta: 1 | -1 }> = [ + { dial: 'k', delta: 1 }, + { dial: 'k', delta: -1 }, + { dial: 'repairRounds', delta: 1 }, + { dial: 'repairRounds', delta: -1 }, + { dial: 'testgen', delta: 1 }, + { dial: 'testgen', delta: -1 }, + ] + const seen = new Set([serializeRolloutPolicy(policy)]) + const neighbors: StructuralRolloutPolicy[] = [] + for (const move of moves) { + const bounds = ROLLOUT_POLICY_BOUNDS[move.dial] + const next = clamp(policy[move.dial] + move.delta * bounds.step, bounds.min, bounds.max) + const candidate: StructuralRolloutPolicy = { ...policy, [move.dial]: next } + const key = serializeRolloutPolicy(candidate) + if (seen.has(key)) continue + seen.add(key) + neighbors.push(candidate) + } + return neighbors +} + +function candidateLabel(base: StructuralRolloutPolicy, next: StructuralRolloutPolicy): string { + for (const dial of ['k', 'repairRounds', 'testgen'] as const) { + if (next[dial] !== base[dial]) return `${dial} ${base[dial]}→${next[dial]}` + } + return 'unchanged' +} + +/** + * The deterministic `SurfaceProposer` for the `'rollout-policy'` surface. + * + * Each generation: parse the current policy surface, enumerate its bounded + * single-dial neighbors, and return at most `min(populationSize, 4)` of them, + * rotating the enumeration window by generation so successive generations explore + * different neighbors when nothing promoted. Proposes NOTHING when the surface + * carries no policy (the profile never opted in) — an empty proposal is the + * loop-native no-op, mirroring `improvementDriver`'s no-findings behavior. + */ +export function rolloutPolicyProposer(): SurfaceProposer { + return { + kind: 'rollout-policy', + async propose(ctx: ProposeContext): Promise { + const policy = parseRolloutPolicy(ctx.currentSurface) + if (!policy) return [] + const neighbors = enumerateNeighborPolicies(policy) + if (neighbors.length === 0) return [] + const cap = Math.max(1, Math.min(ctx.populationSize, MAX_CANDIDATES_PER_GENERATION)) + const start = (ctx.generation * cap) % neighbors.length + const window: StructuralRolloutPolicy[] = [] + for (let i = 0; i < Math.min(cap, neighbors.length); i += 1) { + window.push(neighbors[(start + i) % neighbors.length] as StructuralRolloutPolicy) + } + return window.map((candidate) => ({ + surface: serializeRolloutPolicy(candidate), + label: candidateLabel(policy, candidate), + rationale: + 'bounded single-dial neighbor of the current structuralRollout policy; ' + + 'the held-out gate decides (deterministic enumeration — the dial space is tiny ' + + 'and prompt-style reflective proposals are a measured zero here)', + })) + }, + } +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 2921465a..6da24b4b 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -397,6 +397,35 @@ export { type StreamAgentTurnOptions, streamAgentTurn, } from './stream-agent-turn' +// The structural lever as a strategy-family member: k samples → select by task-visible checks +// (official above authored, crash lowest) → guarded repair steered by the checks' failure output. +// Measured +8..+21pp hidden-test lift (docs/design/structural-rollout-integration.md). +export { + type CheckExecChannel, + type CheckOutcome, + type CheckRunContext, + type CheckRunner, + type CheckSource, + type CheckSourceCtx, + canDisplace, + compareCheckOutcomes, + composeCheckSources, + defaultExtractCandidate, + defaultStructuralRolloutPolicy, + filterAuthoredAsserts, + modelAuthoredChecks, + officialChecksFromMeta, + type RepairStop, + resolveEntrySymbol, + type StructuralRolloutConfig, + type StructuralRolloutPolicy, + type StructuralRolloutResult, + sandboxCheckRunner, + selectBestIndex, + structuralRollout, + type VisibleCheck, + visibleCheckScore, +} from './structural-rollout' // The supervisor's intelligence: it AUTHORS each worker's profile (instructions + model) from a // SKILL (its own system prompt) — the optimizable self-improvement surface, not the plumbing. export { diff --git a/src/runtime/strategy.ts b/src/runtime/strategy.ts index fb27cec3..b4e146be 100644 --- a/src/runtime/strategy.ts +++ b/src/runtime/strategy.ts @@ -306,17 +306,31 @@ async function consultAnalyst( const trajectory = compactTrajectory(messages) const analystModel = opts.analystModel ?? opts.model const chat = analystChat(opts, analystModel) + // With a trajectory, the analyst framing (instruction as system, behavior as the user + // turn) is the channel's shape. With NO trajectory (pre-task consults, e.g. authored + // check generation), that framing breaks: the user turn is then just the task, which + // reads as a solve request — measured on Llama-3-8B, the model ignores the system + // instruction and answers the task (authored-assert yield 6/18 reps vs 18/18 with the + // instruction and task fused into one user message, the proven rig shape). + const consultMessages = trajectory + ? [ + { role: 'system' as const, content: instruction }, + { + role: 'user' as const, + content: `TASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}`, + }, + ] + : [ + { + role: 'user' as const, + content: `${instruction}\n\nTASK:\n${task.userPrompt.slice(0, 1500)}`, + }, + ] const res = await chat.chat({ model: analystModel, temperature: 0.2, maxTokens: 1024, - messages: [ - { role: 'system', content: instruction }, - { - role: 'user', - content: `TASK: ${task.userPrompt.slice(0, 1500)}\n\nTRAJECTORY:\n${trajectory}`, - }, - ], + messages: consultMessages, }) const usage = ( res as { diff --git a/src/runtime/structural-rollout.test.ts b/src/runtime/structural-rollout.test.ts new file mode 100644 index 00000000..dcfe5b96 --- /dev/null +++ b/src/runtime/structural-rollout.test.ts @@ -0,0 +1,399 @@ +import { describe, expect, it, vi } from 'vitest' +import { type AgenticSurface, type AgenticTask, runAgentic } from './strategy' +import { + type CheckOutcome, + type CheckRunner, + canDisplace, + compareCheckOutcomes, + defaultExtractCandidate, + filterAuthoredAsserts, + modelAuthoredChecks, + officialChecksFromMeta, + type StructuralRolloutResult, + sandboxCheckRunner, + selectBestIndex, + structuralRollout, + type VisibleCheck, + visibleCheckScore, +} from './structural-rollout' + +const outcome = ( + passedOfficial: number, + totalOfficial: number, + passedAuthored: number, + totalAuthored: number, + failureOutput = '', + crashed?: boolean, +): CheckOutcome => ({ + passedOfficial, + totalOfficial, + passedAuthored, + totalAuthored, + failureOutput, + ...(crashed !== undefined ? { crashed } : {}), +}) + +describe('selection order — official above authored, crash lowest', () => { + it('one passing official check outranks six passing authored guesses', () => { + const officialWins = outcome(1, 1, 0, 6) + const authoredSweep = outcome(0, 1, 6, 6) + expect(compareCheckOutcomes(officialWins, authoredSweep)).toBeGreaterThan(0) + expect(selectBestIndex([authoredSweep, officialWins])).toBe(1) + }) + + it('authored guesses only break official ties', () => { + const a = outcome(1, 2, 2, 6) + const b = outcome(1, 2, 3, 6) + expect(compareCheckOutcomes(b, a)).toBeGreaterThan(0) + expect(selectBestIndex([a, b])).toBe(1) + }) + + it('a crashed candidate ranks below one that ran and failed everything', () => { + const crashed = outcome(0, 0, 0, 0, 'SyntaxError', true) + const ranAndFailed = outcome(0, 1, 0, 6) + expect(compareCheckOutcomes(crashed, ranAndFailed)).toBeLessThan(0) + expect(selectBestIndex([crashed, ranAndFailed])).toBe(1) + expect(visibleCheckScore(crashed)).toBe(-1) + expect(visibleCheckScore(ranAndFailed)).toBe(0) + }) + + it('argmax breaks ties by FIRST index (the blind pick under zero coverage)', () => { + const noSignal = outcome(0, 0, 0, 0) + expect(selectBestIndex([noSignal, noSignal, noSignal])).toBe(0) + const tie = outcome(1, 2, 1, 2) + expect(selectBestIndex([outcome(0, 2, 2, 2), tie, tie])).toBe(1) + }) +}) + +describe('repair keep-best guard', () => { + it('never displaces with fewer official passes, even when authored/overall look better', () => { + const incumbent = outcome(2, 3, 0, 6) + const challenger = outcome(1, 3, 6, 6) + expect(canDisplace(challenger, incumbent)).toBe(false) + }) + + it('a crashed challenger never displaces', () => { + expect(canDisplace(outcome(0, 0, 0, 0, '', true), outcome(0, 3, 0, 6))).toBe(false) + }) + + it('requires a STRICT improvement in the selection order', () => { + const tie = outcome(2, 3, 1, 6) + expect(canDisplace(tie, tie)).toBe(false) + expect(canDisplace(outcome(2, 3, 2, 6), tie)).toBe(true) + expect(canDisplace(outcome(3, 3, 0, 6), tie)).toBe(true) + }) +}) + +describe('modelAuthoredChecks — the assert filter (visible info only, frozen per task)', () => { + const task: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'Write add.' } + + it('keeps only single-line paren-balanced asserts mentioning the entry symbol, capped', async () => { + const reply = [ + 'Here are the tests:', + '```python', + 'assert add(1, 2) == 3', + 'print(add(1, 2))', // not an assert + 'assert add((1, 2) == 3', // unbalanced parens + 'assert sub(1) == 0', // wrong symbol + 'assert add(0, 0) == 0', + 'assert add(-1, 1) == 0', + 'assert add(2, 2) == 4', + 'assert add(3, 3) == 6', + 'assert add(4, 4) == 8', + 'assert add(5, 5) == 10', // over the cap of 6 + '```', + ].join('\n') + const consult = vi.fn(async () => reply) + const checks = await modelAuthoredChecks().generate(task, { + count: 6, + entrySymbol: 'add', + consult, + }) + expect(consult).toHaveBeenCalledOnce() + expect(checks.map((c) => c.code)).toEqual([ + 'assert add(1, 2) == 3', + 'assert add(0, 0) == 0', + 'assert add(-1, 1) == 0', + 'assert add(2, 2) == 4', + 'assert add(3, 3) == 6', + 'assert add(4, 4) == 8', + ]) + expect(checks.every((c) => c.kind === 'authored')).toBe(true) + }) + + it('skips authoring (no LLM call) when the budget is 0 or no entry symbol resolves', async () => { + const consult = vi.fn(async () => 'assert add(1) == 1') + expect(await modelAuthoredChecks().generate(task, { count: 0, entrySymbol: 'add', consult })) // + .toEqual([]) + expect(await modelAuthoredChecks().generate(task, { count: 6, consult })).toEqual([]) + expect(consult).not.toHaveBeenCalled() + }) + + it('filterAuthoredAsserts falls back to the raw reply when there is no fence', () => { + expect(filterAuthoredAsserts('assert add(1, 1) == 2\nnope', 'add', 6)).toEqual([ + 'assert add(1, 1) == 2', + ]) + }) +}) + +describe('officialChecksFromMeta', () => { + it('lifts string checks from task.meta as official; anything else means none', async () => { + const source = officialChecksFromMeta() + const withMeta: AgenticTask = { + id: 't', + systemPrompt: 's', + userPrompt: 'u', + meta: { visibleChecks: ['assert f() == 1', 42, ' '] }, + } + const bare: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'u' } + const ctx = { count: 0, consult: async () => null } + expect(await source.generate(withMeta, ctx)).toEqual([ + { code: 'assert f() == 1', kind: 'official' }, + ]) + expect(await source.generate(bare, ctx)).toEqual([]) + }) +}) + +describe('sandboxCheckRunner', () => { + const task: AgenticTask = { id: 't', systemPrompt: 's', userPrompt: 'u' } + const checks: VisibleCheck[] = [ + { code: 'assert f() == 1', kind: 'official' }, + { code: 'assert f() != 2', kind: 'authored' }, + ] + + it('throws a clear error with no execution channel — never a silent 0', async () => { + await expect(sandboxCheckRunner().run('def f(): return 1', checks, { task })).rejects.toThrow( + /no execution channel/, + ) + }) + + it('short-circuits an EMPTY check set to a no-signal outcome (nothing to execute)', async () => { + expect(await sandboxCheckRunner().run('def f(): return 1', [], { task })).toEqual({ + passedOfficial: 0, + totalOfficial: 0, + passedAuthored: 0, + totalAuthored: 0, + failureOutput: '', + }) + }) + + it('parses the nonce sentinel and strips it from the repair feedback', async () => { + const box = { + async exec(command: string) { + // Decode the piped program to recover the per-call nonce, as the jail would see it. + const b64 = /printf '%s' '([A-Za-z0-9+/=]+)'/.exec(command)?.[1] ?? '' + const program = Buffer.from(b64, 'base64').toString('utf8') + const nonce = /SRCK-([0-9a-f]+)/.exec(program)?.[1] + expect(nonce).toBeTruthy() + expect(program).toContain('def f(): return 1') + return { + exitCode: 1, + stdout: `SRCK-${nonce} official=1/1 authored=0/1\nCHECK FAILED: assert f() != 2 -> AssertionError: `, + stderr: '', + } + }, + } + const result = await sandboxCheckRunner({ box }).run('def f(): return 1', checks, { task }) + expect(result).toMatchObject({ + passedOfficial: 1, + totalOfficial: 1, + passedAuthored: 0, + totalAuthored: 1, + }) + expect(result.failureOutput).toContain('CHECK FAILED: assert f() != 2') + expect(result.failureOutput).not.toContain('SRCK-') + }) + + it('no sentinel in the output ⇒ crashed (ranks below ran-and-failed), with the crash detail', async () => { + const box = { + async exec() { + return { exitCode: 1, stdout: '', stderr: 'SyntaxError: invalid syntax' } + }, + } + const result = await sandboxCheckRunner({ box }).run('def f(', checks, { task }) + expect(result.crashed).toBe(true) + expect(result.failureOutput).toContain('SyntaxError') + expect(result.totalOfficial).toBe(0) + }) +}) + +describe('defaultExtractCandidate', () => { + it('prefers the LAST submit_answer tool call over fenced content', () => { + const messages = [ + { role: 'assistant', content: '```python\ndef old(): pass\n```' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'c1', + function: { name: 'submit_answer', arguments: '{"answer":"def f(): return 1"}' }, + }, + ], + }, + { role: 'tool', content: 'submission 1 recorded' }, + ] + expect(defaultExtractCandidate(messages)).toBe('def f(): return 1') + }) + + it('prefers the fenced block containing a def over an echoed bare fence', () => { + const messages = [ + { + role: 'assistant', + content: + 'The failure was:\n```\nCHECK FAILED: ...\n```\nFixed:\n```python\ndef f():\n return 2\n```', + }, + ] + expect(defaultExtractCandidate(messages)).toBe('def f():\n return 2') + }) +}) + +describe('structuralRollout — the strategy, end to end (offline transport, fake runner)', () => { + it('validates the policy', () => { + expect(() => structuralRollout({ policy: { k: 0 } })).toThrow(/policy\.k/) + expect(() => structuralRollout({ policy: { repairRounds: -1 } })).toThrow( + /policy\.repairRounds/, + ) + expect(() => structuralRollout({ policy: { testgen: 1.5 } })).toThrow(/policy\.testgen/) + }) + + it('selects by frozen visible checks, repairs on failure output, and holds the official guard', async () => { + // The domain: a verifier-environment-shaped surface (submit_answer only); each + // shot's artifact scores 0 so nothing here can leak a hidden signal into selection — + // only the fake CheckRunner speaks. + let handles = 0 + const surface: AgenticSurface = { + name: 'inert', + async open() { + handles += 1 + return { id: `h${handles}`, surface: 'inert' } + }, + async tools() { + return [ + { + type: 'function' as const, + function: { + name: 'submit_answer', + parameters: { type: 'object', properties: { answer: { type: 'string' } } }, + }, + }, + ] + }, + async call(_handle, name) { + return name === 'submit_answer' ? 'submission recorded' : `ERROR: unknown tool ${name}` + }, + async score() { + return { passes: 0, total: 1, errored: 0 } + }, + async close() {}, + } + + // The offline model: each shot is two turns — a submit_answer tool call carrying the + // candidate (the recorded channel; a content-only final reply never enters the shot's + // conversation), then DONE. Samples produce A then B; repairs produce C then D. + const letters = ['A', 'B', 'C', 'D'] + const bodies: Array> = [] + let workerCall = 0 + const complete = async (body: Record) => { + bodies.push(body) + const shotIdx = Math.min(Math.floor(workerCall / 2), letters.length - 1) + const firstTurn = workerCall % 2 === 0 + workerCall += 1 + const message = firstTurn + ? { + content: '', + tool_calls: [ + { + id: `c${workerCall}`, + function: { + name: 'submit_answer', + arguments: JSON.stringify({ + answer: `def f():\n return "${letters[shotIdx]}"`, + }), + }, + }, + ], + } + : { content: 'DONE' } + return { + choices: [{ message }], + usage: { prompt_tokens: 10, completion_tokens: 5 }, + } + } + + const frozenChecks: VisibleCheck[] = [ + { code: 'assert f() == 1', kind: 'official' }, + { code: 'assert f() == 2', kind: 'official' }, + { code: 'assert f() == 3', kind: 'official' }, + { code: 'assert f() != 0', kind: 'authored' }, + { code: 'assert f() != 9', kind: 'authored' }, + ] + const generate = vi.fn(async () => frozenChecks) + + // Scripted visible-check outcomes, keyed on the candidate the shot produced: + // A (sample) official 2/3 — the argmax pick, with a failure report; + // B (sample) official 1/3; + // C (repair) official 1/3 but authored 2/2 — MUST be held out by the guard; + // D (repair) official 3/3 — displaces, repair stops on full pass. + const failureA = 'CHECK FAILED: assert f() == 3 -> AssertionError' + const runnerCalls: Array<{ candidate: string; checks: VisibleCheck[] }> = [] + const checkRunner: CheckRunner = { + async run(candidate, checks) { + runnerCalls.push({ candidate, checks }) + if (candidate.includes('"A"')) return outcome(2, 3, 0, 2, failureA) + if (candidate.includes('"B"')) return outcome(1, 3, 1, 2, 'CHECK FAILED: ...') + if (candidate.includes('"C"')) return outcome(1, 3, 2, 2, 'CHECK FAILED: ...') + return outcome(3, 3, 2, 2) + }, + } + + const result = await runAgentic({ + surface, + task: { id: 't1', systemPrompt: 'Solve it.', userPrompt: 'Write f.\n\ndef f():\n ...' }, + routerBaseUrl: 'http://offline.test/v1', + routerKey: 'k', + model: 'stub-model', + complete, + innerTurns: 2, + maxTokens: 64, + strategy: structuralRollout({ + policy: { k: 2, repairRounds: 2, testgen: 0 }, + checkSource: { generate }, + checkRunner, + }), + budget: 6, + }) + const rollout = result as unknown as StructuralRolloutResult & { mode: string } + + expect(rollout.mode).toBe('structuralRollout') + expect(rollout.shots).toBe(4) // 2 samples + 2 repairs + expect(rollout.repairStop).toBe('repaired-pass') + + // Checks were generated ONCE and the same frozen set fed every run. + expect(generate).toHaveBeenCalledOnce() + expect(runnerCalls).toHaveLength(4) + for (const c of runnerCalls) expect(c.checks).toBe(frozenChecks) + + // Selection receipts: A won the sample argmax, C was guard-held, D displaced. + expect(rollout.selection).toHaveLength(4) + expect(rollout.selection.map((r) => r.candidateIndex)).toEqual([0, 1, 2, 3]) + expect(rollout.selection[2]?.reason).toMatch(/fewer official checks/) + expect(rollout.selection.filter((r) => r.selected)).toHaveLength(1) + expect(rollout.selection[3]?.selected).toBe(true) + expect(rollout.selection.every((r) => r.selector === 'driver')).toBe(true) + + // Both repair shots were steered by the INCUMBENT's failure output (A's — C never + // displaced), riding the winner's carried conversation: their opening turns end with + // the steer as the last user message. + const steerTurns = bodies.filter((body) => { + const messages = body.messages as Array<{ role: string; content?: string }> + const last = messages[messages.length - 1] + return last?.role === 'user' && (last.content ?? '').includes('task-visible checks') + }) + expect(steerTurns).toHaveLength(2) + for (const body of steerTurns) { + const messages = body.messages as Array<{ role: string; content?: string }> + expect(messages[messages.length - 1]?.content).toContain(failureA) + } + }) +}) diff --git a/src/runtime/structural-rollout.ts b/src/runtime/structural-rollout.ts new file mode 100644 index 00000000..acd14685 --- /dev/null +++ b/src/runtime/structural-rollout.ts @@ -0,0 +1,683 @@ +/** + * structuralRollout — the measured structural lever as a fourth member of the + * sample/refine/sampleThenRefine strategy family: k independent samples, selection by + * TASK-VISIBLE checks only, then a guarded self-repair loop steered by the checks' + * failure output. Design: docs/design/structural-rollout-integration.md; measured basis + * (bench/src/hev-structural.mts, bench/src/mbpp-structural.mts): +8.5..+21.3pp hidden-test + * lift across Llama-3-8B/Qwen2.5-7B × HumanEval/MBPP, null only at saturation. + * + * Honesty invariants carried over from the proven rigs: + * - Visible checks are generated from task-visible information only, BEFORE any + * candidate exists, and FROZEN for every sample and repair round of the task. + * - OFFICIAL checks (shown in the task itself) rank lexicographically above + * model-AUTHORED guesses. This ordering is measured, not stylistic: authored guesses + * run 17–70% wrong depending on model × spec richness, and unweighted they flipped + * selection NEGATIVE on MBPP (6 noisy guesses outvoting the one reliable check). + * - A candidate that crashed before the checks could run ranks below one that ran and + * failed everything. + * - Repair sees ONLY the checks' failure output, and never displaces a candidate that + * passes more official checks with one that passes fewer (wrong visible examples + * poison repair at saturation — the glm /47,/116 regressions). + * + * Placement rule: this is an INFERENCE-TIME capability (it wraps the model call via the + * strategy seam). It does not belong in improve()/selfImprove (training-time); improve() + * may later tune `StructuralRolloutPolicy` as an optimizable surface. + */ + +import { randomBytes } from 'node:crypto' +import { + type AgenticTask, + defineStrategy, + type Strategy, + type StrategyCtx, + type StrategyResult, +} from './strategy' +import type { SelectionReceipt } from './types' + +type Msg = Record + +// ── Policy ──────────────────────────────────────────────────────────────────────── + +/** The rollout's compute recipe — promoted from the proven rigs' env vars (K/REPAIRS/ + * TESTGEN/DIVERSE/TEMPERATURE). Defaults are the measured sweet spot: repair value + * concentrates at low k (~+12pp at k=1, +1–3pp at k=5), so `k=5, repairRounds=2` is the + * full recipe and `k=1, repairRounds=2` the low-compute preset. */ +export interface StructuralRolloutPolicy { + /** Independent samples per task (selection breadth). */ + k: number + /** Repair shots after selection, each steered by the checks' failure output. */ + repairRounds: number + /** Model-authored visible checks requested per task; 0 disables authoring. */ + testgen: number + /** Per-slot strategy-lens prefixes on the k samples (attacks the all-k-fail bucket). + * Measured as a paired null (+0.6pp) — kept as an optional knob, off by default. */ + diverse?: boolean + /** Sampling temperature for every shot of this strategy; omitted ⇒ the worker default. */ + temperature?: number +} + +/** The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. */ +export const defaultStructuralRolloutPolicy: StructuralRolloutPolicy = { + k: 5, + repairRounds: 2, + testgen: 6, +} + +function resolvePolicy(overrides?: Partial): StructuralRolloutPolicy { + const policy = { ...defaultStructuralRolloutPolicy, ...overrides } + if (!Number.isInteger(policy.k) || policy.k < 1) { + throw new Error(`structuralRollout: policy.k must be an integer >= 1, got ${policy.k}`) + } + if (!Number.isInteger(policy.repairRounds) || policy.repairRounds < 0) { + throw new Error( + `structuralRollout: policy.repairRounds must be an integer >= 0, got ${policy.repairRounds}`, + ) + } + if (!Number.isInteger(policy.testgen) || policy.testgen < 0) { + throw new Error( + `structuralRollout: policy.testgen must be an integer >= 0, got ${policy.testgen}`, + ) + } + return policy +} + +// ── Visible checks: the CheckSource seam (the only net-new seam of the design) ────── + +/** One task-visible executable check (e.g. a single-line Python assert). */ +export interface VisibleCheck { + code: string + /** 'official' = shown in the task itself (docstring example, shown assert); + * 'authored' = the model's own guess. Official outranks authored in selection. */ + kind: 'official' | 'authored' +} + +/** What a CheckSource composes with. `consult` is the strategy family's raw analyst + * channel (metered by the conserved pool, offline-injectable via `opts.complete`) — + * check authoring goes through it rather than a bespoke model client. */ +export interface CheckSourceCtx { + /** Authored-check budget for this task (`policy.testgen`). */ + count: number + /** The symbol authored checks must reference; undefined ⇒ authoring is skipped + * (no guesses beats guesses pinned to nothing). */ + entrySymbol?: string + /** One metered LLM call: instruction in, reply text out, null when the channel went + * down. The task's visible prompt is included by the channel itself. */ + consult(instruction: string): Promise +} + +/** Produces the task's visible checks. MUST derive them from agent-visible information + * only, before any candidate exists — the strategy freezes the returned set for every + * sample and repair round of the task. */ +export interface CheckSource { + generate(task: AgenticTask, ctx: CheckSourceCtx): Promise +} + +const authorInstruction = (count: number, entry: string) => + `Read the task below. Write exactly ${count} single-line assert statements that test the ` + + `function \`${entry}\`, based ONLY on the behavior the task itself describes. Each assert ` + + `must be one physical line of the form \`assert ${entry}(...) == expected\` (or a True/False ` + + 'check). Do NOT implement the function. Do NOT copy shown examples verbatim if you can test ' + + 'other cases too. Output ONLY the assert lines inside a single ```python code block.' + +/** The proven authored-assert filter (lifted from the rigs' generateTests): keep only + * single-line, paren-balanced asserts that reference the entry symbol — malformed lines + * are dropped here rather than poisoning every candidate's score identically. */ +export function filterAuthoredAsserts(reply: string, entrySymbol: string, count: number): string[] { + const fences = [...reply.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => + (m[1] ?? '').trim(), + ) + const block = fences.length > 0 ? fences.join('\n') : reply + const balanced = (s: string) => { + let d = 0 + for (const ch of s) { + if (ch === '(' || ch === '[' || ch === '{') d += 1 + else if (ch === ')' || ch === ']' || ch === '}') d -= 1 + if (d < 0) return false + } + return d === 0 + } + return block + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.startsWith('assert ') && l.includes(entrySymbol) && balanced(l)) + .slice(0, count) +} + +/** Default authored-check source: one metered LLM call per task, before sampling, + * filtered through `filterAuthoredAsserts`. Returns [] (no signal, never a fabricated + * check) when the budget is 0, no entry symbol resolves, or the channel went down. */ +export function modelAuthoredChecks(overrides: { count?: number } = {}): CheckSource { + return { + async generate(_task, ctx): Promise { + const count = overrides.count ?? ctx.count + if (count <= 0 || !ctx.entrySymbol) return [] + const entry = ctx.entrySymbol + const reply = await ctx.consult(authorInstruction(count, entry)) + if (!reply) return [] + return filterAuthoredAsserts(reply, entry, count).map((code) => ({ + code, + kind: 'authored' as const, + })) + }, + } +} + +/** Official checks the surface stashed on the task (e.g. MBPP's shown assert). Reads + * `task.meta[key]` as a string array; anything else means no official checks. */ +export function officialChecksFromMeta(key = 'visibleChecks'): CheckSource { + return { + async generate(task): Promise { + const raw = task.meta?.[key] + if (!Array.isArray(raw)) return [] + return raw + .filter((c): c is string => typeof c === 'string' && c.trim().length > 0) + .map((code) => ({ code, kind: 'official' as const })) + }, + } +} + +/** Concatenate check sources (official first by convention — ordering does not affect + * scoring, which reads each check's `kind`). */ +export function composeCheckSources(...sources: CheckSource[]): CheckSource { + return { + async generate(task, ctx): Promise { + const all: VisibleCheck[] = [] + for (const source of sources) all.push(...(await source.generate(task, ctx))) + return all + }, + } +} + +/** The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface + * provides it, else the LAST `def name(` in the visible prompt (a code-completion stub + * lists helpers first, the entry stub last). Undefined ⇒ authoring is skipped. */ +export function resolveEntrySymbol(task: AgenticTask): string | undefined { + const meta = task.meta?.entryPoint + if (typeof meta === 'string' && meta.trim().length > 0) return meta.trim() + const defs = [...task.userPrompt.matchAll(/(?:^|\n)\s*def\s+([A-Za-z_]\w*)\s*\(/g)] + const last = defs[defs.length - 1] + return last?.[1] +} + +// ── Check execution: the CheckRunner seam ──────────────────────────────────────────── + +/** How one candidate fared against the frozen visible checks, split by check kind. */ +export interface CheckOutcome { + passedOfficial: number + totalOfficial: number + passedAuthored: number + totalAuthored: number + /** The checks' failure report — the ONLY feedback the repair loop may see. */ + failureOutput: string + /** True when the candidate crashed before any check could run — ranks below a + * candidate that ran and failed everything. */ + crashed?: boolean +} + +/** Minimal exec channel the default runner needs. `SandboxInstance` (and therefore + * `ValidationCtx.box`) satisfies it structurally. */ +export interface CheckExecChannel { + exec( + command: string, + options?: { timeoutMs?: number }, + ): Promise<{ exitCode: number; stdout: string; stderr: string }> +} + +export interface CheckRunContext { + task: AgenticTask + /** Live exec channel for this run (`ValidationCtx.box` / a sandbox instance). */ + box?: CheckExecChannel + signal?: AbortSignal +} + +/** Executes the frozen checks against one candidate. Implementations MUST fail loud + * (throw) when they cannot execute — a silent zero poisons selection. */ +export interface CheckRunner { + run(candidate: string, checks: VisibleCheck[], ctx: CheckRunContext): Promise +} + +/** The check program (mirrors the rigs' visible-check judge): candidate executes at + * module level, then each check runs INDIVIDUALLY in try/except so one malformed line + * cannot zero the rest; the summary line carries a per-call NONCE so a candidate + * printing a forged summary cannot be parsed as the verdict. */ +function buildCheckProgram( + candidate: string, + official: string[], + authored: string[], + nonce: string, +): string { + const officialB64 = Buffer.from(JSON.stringify(official), 'utf8').toString('base64') + const authoredB64 = Buffer.from(JSON.stringify(authored), 'utf8').toString('base64') + return `${candidate}\n +import base64 as _b64, json as _json, sys as _sys +_official = _json.loads(_b64.b64decode("${officialB64}").decode("utf8")) +_authored = _json.loads(_b64.b64decode("${authoredB64}").decode("utf8")) +_lines = [] +def _run(_tests): + _att, _fail = 0, 0 + for _t in _tests: + _att += 1 + try: + exec(_t, dict(globals())) + except Exception as _e: + _fail += 1 + _lines.append("CHECK FAILED: %s -> %s: %s" % (_t.strip()[:200], type(_e).__name__, str(_e)[:200])) + return _att, _fail +_o_att, _o_fail = _run(_official) +_a_att, _a_fail = _run(_authored) +print("SRCK-${nonce} official=%d/%d authored=%d/%d" % (_o_att - _o_fail, _o_att, _a_att - _a_fail, _a_att)) +_sys.stdout.write("\\n".join(_lines)[-1500:]) +_sys.exit(0 if (_o_fail + _a_fail) == 0 and (_o_att + _a_att) > 0 else 1) +` +} + +/** Default CheckRunner backend: pipes the check program into `python3` over the sandbox + * exec channel (`ctx.box`, or one bound at construction). Never shells out to docker + * itself — the jail is the sandbox's concern. No channel ⇒ throws; it must never + * silently score 0. Empty check sets short-circuit to a no-signal outcome (nothing to + * execute, so no channel is required). */ +export function sandboxCheckRunner( + options: { box?: CheckExecChannel; python?: string; timeoutMs?: number } = {}, +): CheckRunner { + const python = options.python ?? 'python3' + const timeoutMs = options.timeoutMs ?? 20000 + return { + async run(candidate, checks, ctx): Promise { + if (checks.length === 0) { + return { + passedOfficial: 0, + totalOfficial: 0, + passedAuthored: 0, + totalAuthored: 0, + failureOutput: '', + } + } + const box = ctx.box ?? options.box + if (!box) { + throw new Error( + 'sandboxCheckRunner: no execution channel — bind one via sandboxCheckRunner({ box }) ' + + 'or CheckRunContext.box (ValidationCtx.box / a sandbox instance). Refusing to score ' + + 'without executing: a silent 0 would poison selection.', + ) + } + const nonce = randomBytes(8).toString('hex') + const official = checks.filter((c) => c.kind === 'official').map((c) => c.code) + const authored = checks.filter((c) => c.kind === 'authored').map((c) => c.code) + const program = buildCheckProgram(candidate, official, authored, nonce) + const b64 = Buffer.from(program, 'utf8').toString('base64') + const r = await box.exec(`printf '%s' '${b64}' | base64 -d | ${python} -`, { timeoutMs }) + const summary = new RegExp( + `SRCK-${nonce} official=(\\d+)/(\\d+) authored=(\\d+)/(\\d+)`, + ).exec(r.stdout) + if (!summary) { + // The candidate crashed (or hung) before the check scaffold could report. + const detail = + (r.stderr || r.stdout).slice(-1500) || + 'no output (crashed or timed out before the checks could run)' + return { + passedOfficial: 0, + totalOfficial: 0, + passedAuthored: 0, + totalAuthored: 0, + failureOutput: detail, + crashed: true, + } + } + // Strip the sentinel line from repair feedback — the model must never learn the + // summary format it could try to forge. + const failureOutput = r.stdout.replace(summary[0], '').slice(-1500).trim() + return { + passedOfficial: Number(summary[1]), + totalOfficial: Number(summary[2]), + passedAuthored: Number(summary[3]), + totalAuthored: Number(summary[4]), + failureOutput, + } + }, + } +} + +// ── Selection order (measured, not stylistic) ──────────────────────────────────────── + +const frac = (passed: number, total: number) => (total > 0 ? passed / total : 0) + +/** The selection order: crash < ran; then official pass-fraction; authored guesses only + * break ties. Returns > 0 when `a` outranks `b`. Strictly lexicographic — on MBPP, + * letting 6 noisy guesses outvote the one official check flipped selection negative. */ +export function compareCheckOutcomes(a: CheckOutcome, b: CheckOutcome): number { + const aCrashed = a.crashed === true + const bCrashed = b.crashed === true + if (aCrashed !== bCrashed) return aCrashed ? -1 : 1 + if (aCrashed) return 0 + const official = frac(a.passedOfficial, a.totalOfficial) - frac(b.passedOfficial, b.totalOfficial) + if (official !== 0) return official + return frac(a.passedAuthored, a.totalAuthored) - frac(b.passedAuthored, b.totalAuthored) +} + +/** Display scalar for receipts/reports (the rigs' `visibleScore` shape): crash = -1, + * else official fraction + 0.001 × authored fraction. Selection itself uses the exact + * lexicographic comparator, never this scalar. */ +export function visibleCheckScore(o: CheckOutcome): number { + if (o.crashed) return -1 + return frac(o.passedOfficial, o.totalOfficial) + 0.001 * frac(o.passedAuthored, o.totalAuthored) +} + +/** Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero + * visible coverage every candidate ties at no-signal and index 0 is the blind pick). */ +export function selectBestIndex(outcomes: ReadonlyArray): number { + let best = 0 + for (let i = 1; i < outcomes.length; i += 1) { + if (compareCheckOutcomes(outcomes[i] as CheckOutcome, outcomes[best] as CheckOutcome) > 0) { + best = i + } + } + return best +} + +/** The repair keep-best guard: a challenger displaces the incumbent only when it is + * strictly better in the selection order AND passes at least as many official checks. + * The raw-count clause is deliberate belt-and-braces over the comparator (a custom + * runner can report shifted totals): repair must NEVER replace a candidate that passes + * more official checks with one that passes fewer. */ +export function canDisplace(challenger: CheckOutcome, incumbent: CheckOutcome): boolean { + if (challenger.crashed === true) return false + if (challenger.passedOfficial < incumbent.passedOfficial) return false + return compareCheckOutcomes(challenger, incumbent) > 0 +} + +const totalChecks = (o: CheckOutcome) => o.totalOfficial + o.totalAuthored + +const passesAllChecks = (o: CheckOutcome) => + o.crashed !== true && + totalChecks(o) > 0 && + o.passedOfficial === o.totalOfficial && + o.passedAuthored === o.totalAuthored + +// ── Candidate extraction ───────────────────────────────────────────────────────────── + +/** The candidate a shot produced, read from its conversation: the LAST `submit_answer` + * tool-call argument (verifier environments submit the artifact explicitly), else the + * latest assistant reply's fenced code block — preferring a block containing a `def`, + * because repair replies echo the failure report in a bare fence BEFORE the fixed code + * (the rigs' extractRepairCode lesson) — else the latest non-empty assistant text. */ +export function defaultExtractCandidate(messages: ReadonlyArray): string { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const calls = messages[i]?.tool_calls as + | Array<{ function?: { name?: string; arguments?: string } }> + | undefined + if (!calls) continue + for (let j = calls.length - 1; j >= 0; j -= 1) { + const call = calls[j] + if (call?.function?.name !== 'submit_answer') continue + try { + const args = JSON.parse(call.function.arguments ?? '{}') as { answer?: unknown } + if (typeof args.answer === 'string' && args.answer.trim()) return args.answer.trim() + } catch { + /* malformed arguments — keep scanning */ + } + } + } + const contents: string[] = [] + for (const m of messages) { + if (m.role === 'assistant' && typeof m.content === 'string' && m.content.trim()) { + contents.push(m.content) + } + } + const fencesOf = (text: string) => + [...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/gi)].map((m) => (m[1] ?? '').trim()) + for (let i = contents.length - 1; i >= 0; i -= 1) { + const fences = fencesOf(contents[i] as string) + for (let j = fences.length - 1; j >= 0; j -= 1) { + if (/(^|\n)\s*def\s+\w+/.test(fences[j] as string)) return fences[j] as string + } + } + for (let i = contents.length - 1; i >= 0; i -= 1) { + const fences = fencesOf(contents[i] as string) + if (fences.length > 0) return fences[fences.length - 1] as string + } + return (contents[contents.length - 1] ?? '').trim() +} + +// ── The strategy ───────────────────────────────────────────────────────────────────── + +/** Per-slot approach lenses for `diverse` mode — copied from bench/src/directives.ts + * (the measured rig plumbing; a paired null there, kept as an optional knob). */ +const DIVERSE_LENSES: ReadonlyArray = [ + 'Answer directly and decisively from what you already know. State the single best answer without hedging.', + 'Decompose the question into the sub-facts it depends on. Establish each sub-fact explicitly, then compose them into the answer.', + 'Reason from first principles. Ignore the most obvious or popular guess; derive the answer from underlying facts and relationships.', + 'Name the most plausible WRONG answer and the trap that makes it tempting. Rule it out, then commit to the answer that survives.', +] + +function slotLens(slot: number): string { + const lens = DIVERSE_LENSES[slot % DIVERSE_LENSES.length] as string + const tag = + slot < DIVERSE_LENSES.length ? '' : ` (variant ${Math.floor(slot / DIVERSE_LENSES.length) + 1})` + return `${lens}${tag}` +} + +function repairSteer(outcome: CheckOutcome): string { + return [ + 'Your latest solution failed some of the task-visible checks.', + 'Result of running the visible checks against it:', + '```', + outcome.failureOutput.trim() || '(the code crashed before the checks could run)', + '```', + 'Fix the solution so the visible checks pass. Provide the COMPLETE corrected solution the', + 'same way you provided the original (same tool or format) — not a fragment or a diff.', + ].join('\n') +} + +function describeOutcome(label: string, o: CheckOutcome): string { + if (o.crashed) return `${label}: crashed before the checks could run` + return ( + `${label}: official ${o.passedOfficial}/${o.totalOfficial}, ` + + `authored ${o.passedAuthored}/${o.totalAuthored}` + ) +} + +export type RepairStop = + | 'already-passing' + | 'no-signal' + | 'repaired-pass' + | 'rounds-exhausted' + | 'no-candidates' + +/** The body's deliverable — a `StrategyResult` plus selection provenance. The extra + * fields ride through `defineStrategy`'s deliverable spread onto `AgenticRunResult` + * (score/resolved stay harness-verified, exactly as for every authored strategy). */ +export interface StructuralRolloutResult extends StrategyResult { + /** One receipt per scored candidate (k samples, then repairs), `SelectionReceipt` + * shaped like the kernel's (`types.ts`), selector 'driver'. */ + selection: SelectionReceipt[] + repairStop: RepairStop + officialChecks: number + authoredChecks: number +} + +export interface StructuralRolloutConfig { + /** Knobs; missing fields take the measured defaults (k=5, repairRounds=2, testgen=6). */ + policy?: Partial + /** Where the visible checks come from. Default: official checks from + * `task.meta.visibleChecks` composed with `modelAuthoredChecks()`. */ + checkSource?: CheckSource + /** How candidates are measured. Default `sandboxCheckRunner()` — it needs an exec + * channel (bind one to the runner, or pass `box` here) and fails loud without one. */ + checkRunner?: CheckRunner + /** Exec channel threaded into every check run of this strategy (a sandbox instance / + * `ValidationCtx.box`). The strategy seam itself carries no sandbox, so the caller + * who owns one supplies it here or binds it into the runner. */ + box?: CheckExecChannel + /** Candidate extraction from a shot's conversation. Default `defaultExtractCandidate`. */ + extractCandidate?: (messages: ReadonlyArray) => string +} + +/** + * Build the structuralRollout `Strategy`: k shots → score each by the frozen visible + * checks (official above authored, crash lowest) → argmax with first-index tie-break → + * up to `repairRounds` repair shots steered by the failure output, keep-best under the + * official-check guard. Authored via `defineStrategy`, so the deliverable score stays + * harness-verified and every shot is metered by the conserved pool. + * + * Budget note: `runAgentic`'s `budget` sizes the pool — pass at least + * `k + repairRounds + 1` so the samples, repairs, and the check-author consult all admit. + */ +export function structuralRollout(config: StructuralRolloutConfig = {}): Strategy { + const policy = resolvePolicy(config.policy) + const checkSource = + config.checkSource ?? composeCheckSources(officialChecksFromMeta(), modelAuthoredChecks()) + const checkRunner = config.checkRunner ?? sandboxCheckRunner() + const extract = config.extractCandidate ?? defaultExtractCandidate + + const inner = defineStrategy( + 'structuralRollout', + async (ctx: StrategyCtx): Promise => { + const { task, shot } = ctx + const progression: number[] = [] + const receipts: SelectionReceipt[] = [] + let completions = 0 + let shots = 0 + + // Freeze the visible checks BEFORE any candidate exists — every sample and repair + // round is measured against this same set. Authoring goes through the strategy's + // raw analyst channel, so its spend is metered and the offline transport applies. + const consult = async (instruction: string): Promise => { + const reply = await ctx.consult([], instruction) + completions += 1 + return reply + } + const entrySymbol = resolveEntrySymbol(task) + const checks = await checkSource.generate(task, { + count: policy.testgen, + ...(entrySymbol ? { entrySymbol } : {}), + consult, + }) + const officialChecks = checks.filter((c) => c.kind === 'official').length + const authoredChecks = checks.length - officialChecks + const runCtx: CheckRunContext = { task, ...(config.box ? { box: config.box } : {}) } + + interface Candidate { + index: number + messages: Msg[] + outcome: CheckOutcome + shotScore: number + shotResolved: boolean + } + + // k independent samples, each on its own artifact, scored by the frozen checks. + const candidates: Candidate[] = [] + for (let i = 0; i < policy.k; i += 1) { + const out = await shot(policy.diverse ? { steer: slotLens(i) } : undefined) + if (!out) break // pool starved — select among what settled + shots += 1 + completions += out.completions + progression.push(out.score) + const outcome = await checkRunner.run(extract(out.messages), checks, runCtx) + candidates.push({ + index: candidates.length, + messages: out.messages, + outcome, + shotScore: out.score, + shotResolved: out.total > 0 && out.passes === out.total, + }) + } + if (candidates.length === 0) { + return { + score: 0, + resolved: false, + completions, + progression, + shots, + selection: receipts, + repairStop: 'no-candidates', + officialChecks, + authoredChecks, + } + } + + // Argmax by the official-first order; first index wins ties (the blind pick when + // the task has zero visible coverage). + let best = candidates[selectBestIndex(candidates.map((c) => c.outcome))] as Candidate + for (const c of candidates) { + receipts.push({ + candidateIndex: c.index, + selected: false, + score: visibleCheckScore(c.outcome), + reason: describeOutcome('sample', c.outcome), + selector: 'driver', + }) + } + + // Guarded repair: steered ONLY by the checks' failure output, keep-best, and a + // repair never displaces a candidate that passes more official checks. + let seq = candidates.length + let repairStop: RepairStop = 'already-passing' + if (!passesAllChecks(best.outcome)) { + if (best.outcome.crashed !== true && totalChecks(best.outcome) === 0) { + repairStop = 'no-signal' // no visible checks ⇒ nothing honest to steer on + } else { + repairStop = 'rounds-exhausted' + for (let r = 0; r < policy.repairRounds; r += 1) { + const out = await shot({ messages: best.messages, steer: repairSteer(best.outcome) }) + if (!out) break + shots += 1 + completions += out.completions + progression.push(out.score) + const outcome = await checkRunner.run(extract(out.messages), checks, runCtx) + const displaced = canDisplace(outcome, best.outcome) + const label = displaced + ? 'repair (displaced the incumbent)' + : outcome.crashed !== true && outcome.passedOfficial < best.outcome.passedOfficial + ? 'repair (held out: passes fewer official checks than the incumbent)' + : 'repair (held out: no improvement)' + receipts.push({ + candidateIndex: seq, + selected: false, + score: visibleCheckScore(outcome), + reason: describeOutcome(label, outcome), + selector: 'driver', + }) + if (displaced) { + best = { + index: seq, + messages: out.messages, + outcome, + shotScore: out.score, + shotResolved: out.total > 0 && out.passes === out.total, + } + } + seq += 1 + if (passesAllChecks(best.outcome)) { + repairStop = 'repaired-pass' + break + } + } + } + } + + const winner = receipts.find((r) => r.candidateIndex === best.index) + if (winner) winner.selected = true + + return { + score: best.shotScore, + resolved: best.shotResolved, + completions, + progression, + shots, + selection: receipts, + repairStop, + officialChecks, + authoredChecks, + } + }, + ) + + if (policy.temperature === undefined) return inner + // The shot temperature is an AgenticOptions concern; the policy override threads in at + // the driver seam so the strategy stays a plain defineStrategy member. + return { + name: inner.name, + driver: (surface, task, opts, budget) => + inner.driver(surface, task, { ...opts, temperature: policy.temperature }, budget), + } +} diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index a52725df..ee1ea7a7 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -164,8 +164,15 @@ export async function runBrainLoop(opts: { opts.hooks?.onUsage?.(r.usage) } if (r.content) lastText = r.content - if (r.toolCalls.length === 0) + if (r.toolCalls.length === 0) { + // The stopping reply is part of the conversation (the contract on `messages`: seed + + // every assistant/tool turn). Without this push, a shot that answers WITHOUT tools + // returns a transcript missing its own answer — depth continuation criticizes a + // solution the model can't see, and candidate extraction (structural-rollout's + // fenced-code fallback) reads an empty conversation on non-tool-calling models. + if (r.content) messages.push({ role: 'assistant', content: r.content }) return { final: lastText, turns: turn, toolCalls, toolTrace, usage, messages } + } // Record the assistant turn verbatim (content + the tool_calls it requested), then run each // call and fold the result back as a `tool` message.