From 00f987e58c9aa5761e29919445419fb9faf81386 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 19 Oct 2026 11:05:31 -0400 Subject: [PATCH 1/3] feat(evals): discovery suite Add a fifth eval suite, `discovery`, for vendor-neutral tasks whose outcome is categorical (which product the agent chose) rather than pass/fail. It is never a headline suite: HEADLINE_SUITES is unchanged, so every validator, expander, planner and the leaderboard aggregate pick it up from the one constant with no store migration or schema bump. The Matrix view builds its suite tabs from its own hard-coded list, so a suite missing there is invisible even though the report carries it; the list and hover titles now include discovery, are exported, and the duplicated available-suite filter is one pure helper the new Matrix test pins (tab present with a discovery pair, absent without, discovery-only runs land on it with no toggle). Spec: docs/ideation/discovery-suite/spec-phase-1.md --- docs/concepts.md | 11 +++++---- docs/writing-an-eval.md | 2 +- src/evals/types.ts | 8 ++++--- src/report/aggregate.ts | 4 ++-- tests/evals-discovery.test.ts | 18 ++++++++++++-- tests/leaderboard.test.ts | 22 ++++++++++++++++++ ui/src/views/Matrix.test.ts | 44 +++++++++++++++++++++++++++++++++++ ui/src/views/Matrix.tsx | 24 ++++++++++++------- 8 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 ui/src/views/Matrix.test.ts diff --git a/docs/concepts.md b/docs/concepts.md index 01c042e..1aff9e4 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -58,7 +58,10 @@ There is no registry and no special dialect: the scorer is ordinary code, and every eval is self-contained. The `suite` field tiers evals: **benchmark** and **regression** are the headline suites the leaderboard counts; **smoke** holds evals every arm solves, run to catch breakage but never moving the -headline; **other** is anything else outside it. A framework-expanded eval can +headline; **discovery** holds vendor-neutral tasks whose outcome is +categorical (which product the agent chose) rather than pass/fail — each +attempt records its outcome and is never graded into the headline; **other** +is anything else outside it. A framework-expanded eval can retier one variant with `suites:` (a saturated port goes to smoke while the rest stay benchmark). [Writing an eval](./writing-an-eval.md) is the authoring reference. @@ -243,8 +246,8 @@ the matrix banner and the text report's `suspect:` lines display. Each entry in `report.json`'s `experiments[]` carries an `aggregate`: one leaderboard row per experiment, rendered as a table above the matrix and in the text report. Every number in it except SDK recall counts **headline-suite -attempts only** (benchmark and regression), so smoke and other evals never -move the ranking. That is why the row's `cost_usd` can be lower than the +attempts only** (benchmark and regression), so smoke, discovery and other +evals never move the ranking. That is why the row's `cost_usd` can be lower than the run-wide `telemetry.cost_usd`. Its columns and their denominators: - **Gate**: whether the all-eval pass rate is at or above the report's @@ -345,7 +348,7 @@ exactly what a negative control should do. | **pair** | One eval × experiment combination — the report's row unit (a *cell* in the UI). | | **run** | One `quickstudy run`, all pairs and trials, under one run id. | | **manifest** | The immutable content-addressed identity record written before attempts start. | -| **suite** | Eval tier: `benchmark` and `regression` (headline), `smoke` (saturated, outside it), `other`. | +| **suite** | Eval tier: `benchmark` and `regression` (headline), `smoke` (saturated, outside it), `discovery` (categorical, vendor-neutral, outside it), `other`. | Next: [Getting started](./getting-started.md) · [Writing an eval](./writing-an-eval.md) diff --git a/docs/writing-an-eval.md b/docs/writing-an-eval.md index fabb11f..36e5baf 100644 --- a/docs/writing-an-eval.md +++ b/docs/writing-an-eval.md @@ -40,7 +40,7 @@ frontmatter schema is deliberately tiny and closed: | Key | Required | Meaning | | ------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `id` | yes | Must equal the directory name (prevents silent aliasing). | -| `suite` | yes | `benchmark` or `regression` (the headline suites the leaderboard counts), `smoke` (evals every arm solves; run, never counted), or `other`. | +| `suite` | yes | `benchmark` or `regression` (the headline suites the leaderboard counts), `smoke` (evals every arm solves; run, never counted), `discovery` (vendor-neutral tasks with a categorical outcome; recorded, never counted), or `other`. | | `product` | no | Free-form product label, used for reporting. | | `framework` | no | The app stack the task targets (`nextjs`, `rails`, `django`, …); container runtimes and `--framework` filters read it. | | `frameworks` | no | List of framework tokens; expands the directory into one eval per framework (id `-`). See [Framework variants](#framework-variants-frameworks). | diff --git a/src/evals/types.ts b/src/evals/types.ts index 422abc7..66ecf8a 100644 --- a/src/evals/types.ts +++ b/src/evals/types.ts @@ -8,10 +8,12 @@ /** * Closed set of eval suites. `benchmark` and `regression` are the headline * suites: the leaderboard aggregates count both. `smoke` holds evals every - * arm solves — run to catch breakage, never moving the headline. `other` is - * anything else outside the headline. + * arm solves — run to catch breakage, never moving the headline. `discovery` + * holds vendor-neutral tasks whose outcome is categorical (which product was + * chosen) rather than pass/fail; recorded per attempt, never headline. + * `other` is anything else outside the headline. */ -export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other"] as const; +export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other", "discovery"] as const; export type EvalSuite = (typeof EVAL_SUITES)[number]; /** The suites whose attempts count toward the leaderboard aggregates. */ diff --git a/src/report/aggregate.ts b/src/report/aggregate.ts index d731245..4ad1740 100644 --- a/src/report/aggregate.ts +++ b/src/report/aggregate.ts @@ -3,8 +3,8 @@ * the same attempts and (banded) pairs as the rest of the report. * * Every number is restricted to headline-suite attempts (HEADLINE_SUITES: - * benchmark and regression), so smoke and other suites never move the - * headline row. Telemetry follows the report's rule: sums and means cover + * benchmark and regression), so the smoke, discovery and other suites never + * move the headline row. Telemetry follows the report's rule: sums and means cover * only attempts that reported a value, and are null (never 0) when none did. */ import { isHeadlineSuite } from "../evals/types.ts"; diff --git a/tests/evals-discovery.test.ts b/tests/evals-discovery.test.ts index c2b2198..9b20b3a 100644 --- a/tests/evals-discovery.test.ts +++ b/tests/evals-discovery.test.ts @@ -110,7 +110,7 @@ describe("eval discovery", () => { test("suite outside the closed set is rejected", () => { const root = makeRoot(); writeEval(root, "bad-suite", { frontmatter: "---\nid: bad-suite\nsuite: production\n---\n" }); - expect(() => discoverEvals(root)).toThrow(/"suite" must be one of: benchmark, regression, smoke, other/); + expect(() => discoverEvals(root)).toThrow(/"suite" must be one of: benchmark, regression, smoke, other, discovery/); }); test("an empty task body is rejected", () => { @@ -279,7 +279,7 @@ describe("eval discovery", () => { writeEval(unknown, "multi", { frontmatter: `---\nid: multi\nsuite: benchmark\nframeworks:\n - alpha\nsuites:\n alpha: headline\n---\n`, }); - expect(() => discoverEvals(unknown)).toThrow(/"suites" entry for "alpha" must be one of: benchmark, regression, smoke, other/); + expect(() => discoverEvals(unknown)).toThrow(/"suites" entry for "alpha" must be one of: benchmark, regression, smoke, other, discovery/); }); test("smoke is a suite of its own", () => { @@ -288,6 +288,20 @@ describe("eval discovery", () => { expect(discoverEvals(root)[0]!.metadata.suite).toBe("smoke"); }); + test("the discovery suite validates on its own and through a suites: retier", () => { + const root = makeRoot(); + writeEval(root, "which-vendor", { frontmatter: `---\nid: which-vendor\nsuite: discovery\n---\n` }); + expect(discoverEvals(root)[0]!.metadata.suite).toBe("discovery"); + const expanded = makeRoot(); + writeEval(expanded, "multi", { + frontmatter: `---\nid: multi\nsuite: discovery\nframeworks:\n - alpha\n - beta\nsuites:\n beta: benchmark\n---\n`, + }); + expect(discoverEvals(expanded).map((e) => [e.metadata.id, e.metadata.suite])).toEqual([ + ["multi-alpha", "discovery"], + ["multi-beta", "benchmark"], + ]); + }); + test("importEvalScorer validates the default export is a function", async () => { const root = makeRoot(); const good = writeEval(root, "good"); diff --git a/tests/leaderboard.test.ts b/tests/leaderboard.test.ts index 2d64304..7e3185d 100644 --- a/tests/leaderboard.test.ts +++ b/tests/leaderboard.test.ts @@ -151,6 +151,28 @@ describe("aggregate", () => { expect(x.pass_rate_all).toBe(0.5); }); + test("discovery-suite attempts never count toward the aggregate", () => { + // A passing discovery pair moves nothing: not attempts, not the pass rate, + // not the discriminating subset. + const base = aggregateOf([pass("split", "x"), fail("split", "x"), fail("split", "y")], "x"); + const withDiscovery = aggregateOf( + [ + pass("split", "x"), fail("split", "x"), fail("split", "y"), + pass("which-vendor", "x", { suite: "discovery" }), pass("which-vendor", "x", { suite: "discovery" }), + ], + "x", + ); + expect(withDiscovery).toEqual(base); + expect(withDiscovery.attempts).toBe(2); + expect(withDiscovery.pass_rate_all).toBe(0.5); + // A discovery-only run yields the empty headline aggregate (null, never NaN). + const only = aggregateOf([pass("which-vendor", "x", { suite: "discovery" })], "x"); + expect(only.attempts).toBe(0); + expect(only.pass_rate_all).toBeNull(); + expect(only.pass_rate_discriminating).toBeNull(); + expect(only.cost_per_pass_usd).toBeNull(); + }); + test("a solved-by-all eval is excluded from the discriminating subset", () => { const attempts = [ pass("solved", "x"), pass("solved", "x"), pass("solved", "y"), diff --git a/ui/src/views/Matrix.test.ts b/ui/src/views/Matrix.test.ts new file mode 100644 index 0000000..cf3c916 --- /dev/null +++ b/ui/src/views/Matrix.test.ts @@ -0,0 +1,44 @@ +/** + * Suite tabs on the Matrix view. The view builds its tabs from a hard-coded + * list, so a suite missing there is invisible even though the report carries + * it; these pin the list, the hover titles, and the tab/default-suite logic + * through the exported pure helpers (the view itself loads asynchronously). + */ + +import { expect, test } from "bun:test"; +import type { DataAttempt } from "../../../src/export-types.ts"; +import { availableSuites, resolveSuite, SUITES, SUITE_TITLES } from "./Matrix.tsx"; + +function attempt(suite: string): Pick { + return { suite }; +} + +test("the suite tabs include discovery, with a title saying it never grades into the headline", () => { + expect(SUITES).toContain("discovery"); + expect(SUITE_TITLES.discovery).toMatch(/never graded into the headline/); + for (const suite of SUITES) { + if (suite !== "benchmark") expect(SUITE_TITLES[suite]).toBeTruthy(); + } +}); + +test("a run with benchmark and discovery pairs shows both tabs, in tab order", () => { + const suites = availableSuites([attempt("discovery"), attempt("benchmark"), attempt("discovery")]); + expect(suites).toEqual(["benchmark", "discovery"]); + expect(resolveSuite(suites, "benchmark")).toBe("benchmark"); + expect(resolveSuite(suites, "discovery")).toBe("discovery"); +}); + +test("a run with no discovery pair shows no discovery tab", () => { + const suites = availableSuites([attempt("benchmark"), attempt("regression")]); + expect(suites).toEqual(["benchmark", "regression"]); + expect(suites).not.toContain("discovery"); + expect(resolveSuite(suites, "discovery")).toBe("benchmark"); +}); + +test("a discovery-only run lands on the discovery suite and renders no toggle", () => { + const suites = availableSuites([attempt("discovery")]); + expect(suites).toEqual(["discovery"]); + expect(suites).toHaveLength(1); + expect(resolveSuite(suites, "benchmark")).toBe("discovery"); + expect(resolveSuite([], "benchmark")).toBe("benchmark"); +}); diff --git a/ui/src/views/Matrix.tsx b/ui/src/views/Matrix.tsx index 3b05578..8c50f1e 100644 --- a/ui/src/views/Matrix.tsx +++ b/ui/src/views/Matrix.tsx @@ -29,15 +29,27 @@ import { useAsync } from "../hooks.ts"; import { links, navigate } from "../router.ts"; const ALL = "all"; -const SUITES = ["benchmark", "regression", "smoke", "other"] as const; +/** Every suite the view can show, in tab order; a suite missing here is invisible even when the report carries it. */ +export const SUITES = ["benchmark", "regression", "smoke", "other", "discovery"] as const; /** What a non-benchmark suite badge says on hover. */ -const SUITE_TITLES: Record = { +export const SUITE_TITLES: Record = { regression: "Narrow checks on known failure modes; a headline suite alongside benchmark.", smoke: "Evals every arm solves; run to catch breakage, never in the headline scores.", other: "Outside the headline scores.", + discovery: "Vendor-neutral tasks; outcomes are recorded per vendor, never graded into the headline scores.", }; +/** The suites the run has attempts in, in tab order — the toggle renders one tab each (none when only one). */ +export function availableSuites(attempts: readonly Pick[]): string[] { + return SUITES.filter((candidate) => attempts.some((attempt) => attempt.suite === candidate)); +} + +/** The suite to show: the selection when the run has it, else the first available (benchmark for an empty run). */ +export function resolveSuite(available: readonly string[], selected: string): string { + return available.includes(selected) ? selected : (available[0] ?? "benchmark"); +} + interface MatrixProps { meta: DataMeta; runId: string; @@ -341,11 +353,8 @@ export function Matrix({ meta, runId, cell }: MatrixProps): ReactNode { // Regression evals never blend into the benchmark grid — the toggle only // renders when the run has more than one suite. const [selectedSuite, setSuite] = useState("benchmark"); - const availableSuites = run.status === "ready" - ? SUITES.filter((candidate) => run.value.attempts.some((attempt) => attempt.suite === candidate)) - : []; - const suite = availableSuites.includes(selectedSuite as typeof SUITES[number]) - ? selectedSuite : (availableSuites[0] ?? "benchmark"); + const suites = run.status === "ready" ? availableSuites(run.value.attempts) : []; + const suite = resolveSuite(suites, selectedSuite); const [filters, setFilters] = useState({ product: ALL, framework: ALL, topic: ALL }); useEffect(() => { @@ -379,7 +388,6 @@ export function Matrix({ meta, runId, cell }: MatrixProps): ReactNode { const value = run.value; const summary = value.run; - const suites = SUITES.filter((candidate) => value.attempts.some((attempt) => attempt.suite === candidate)); const products = [...new Set(value.attempts.map((attempt) => metadataOf(value, attempt.eval_id).product).filter((p): p is string => p !== undefined))].sort(); const frameworkValues = [...new Set(value.attempts.map((attempt) => metadataOf(value, attempt.eval_id).framework).filter((f): f is string => f !== undefined))].sort(); const topics = [...new Set(value.attempts.flatMap((attempt) => metadataOf(value, attempt.eval_id).topics ?? []))].sort(); From 5648dffd7a1874d6c9e6e00705be2d86953d4aea Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 19 Oct 2026 11:05:31 -0400 Subject: [PATCH 2/3] feat(runtime): per-experiment egress opt-out `Runtime.egress: "open"` skips the egress proxy for that experiment's attempts even when the run was started with --egress-proxy, so an arm that must reach the open internet can share a run with proxied arms. The choice lands in each pair environment (`egress.proxy`), so it is identity and can be exempted in a comparison group with `treatmentPolicyFields: ["egress"]`. The decision is one condition in resolvePairEnvironments; the Docker layer is untouched because each attempt already gates its internal network and sidecar on its pair's `egress.proxy`. An open pair still records its (inert) allowlist, writes no denials log and reports egress as not instrumented. Container runtimes only: defineExperiment rejects "open" on a host runtime, which is never proxied. Spec: docs/ideation/discovery-suite/spec-phase-1.md --- docs/extensions.md | 12 ++++- src/execution-policy.ts | 8 ++-- src/experiments/types.ts | 14 ++++++ tests/experiments-v3.test.ts | 85 ++++++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index 873638a..b7fce23 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -107,9 +107,17 @@ Unsupported streams remain unavailable/null; zero means an applicable stream was observed with no matching event. Offered surfaces and observed cross-arm use remain separate in reports and diagnosis. -`--egress-proxy` applies to every container. The allowlist is the pair's runtime +`--egress-proxy` applies to every container unless its experiment opts out with +`runtime.egress: "open"`, which skips the proxy for that experiment's attempts +inside a proxied run (no internal network, no sidecar, no proxy environment), so +an arm that must reach the open internet can share a run with proxied arms. The +choice is recorded in each pair environment, so it is identity; exempt it in a +comparison group with `treatmentPolicyFields: ["egress"]`. An open pair still +records its (inert) allowlist, writes no denials log and reports egress as not +instrumented. Container runtimes only: `defineExperiment` rejects `"open"` on a +host runtime. The allowlist is the pair's runtime hosts plus that adapter's provider endpoints; absent runtime hosts means provider -endpoints only. Each attempt has its own internal network and egress proxy. A +endpoints only. Each proxied attempt has its own internal network and egress proxy. A shared MCP auth proxy has its own infrastructure traffic, which is not attributed to agent attempts. Egress denial artifacts contain sanitized host identifiers and attempt IDs. An empty log means zero observed denials; an absent log means unavailable diff --git a/src/execution-policy.ts b/src/execution-policy.ts index 883a4a4..989ecc5 100644 --- a/src/execution-policy.ts +++ b/src/execution-policy.ts @@ -76,10 +76,12 @@ export function resolvePairEnvironments(options: { pairs[pairKey(loaded.metadata.id, experiment.id)] = { ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}), image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null, - // With the flag, every container is proxied. No runtime hosts means - // only the adapter's provider endpoints are permitted. + // With the flag, every container is proxied unless its runtime opts + // out with egress: "open". No runtime hosts means only the adapter's + // provider endpoints are permitted; an open pair still records its + // allowlist (inert) so identity holds if the run flag flips. egress: { - proxy: runtime.kind === "container" && (options.egressProxy ?? false), + proxy: runtime.kind === "container" && (options.egressProxy ?? false) && runtime.egress !== "open", allowlist: composeAllowlistFromHosts([ ...(runtime.egressHosts?.(loaded.metadata) ?? []), ...(options.adapterHosts?.[experiment.id] ?? []), diff --git a/src/experiments/types.ts b/src/experiments/types.ts index ea13332..9110e04 100644 --- a/src/experiments/types.ts +++ b/src/experiments/types.ts @@ -55,6 +55,15 @@ export interface Runtime extends RuntimeCapabilities { * declare the same fields. Source/config identity still changes per attempt. */ treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble" | "workspaceFiles">; + /** + * Whether this experiment's attempts go through the run's egress proxy. + * "open" skips the proxy sidecar and network for these attempts even when + * the run was started with --egress-proxy, so an arm that must reach the + * open internet can share a run with proxied arms. The choice is recorded + * in each pair environment (identity); exempt it in a comparison group with + * treatmentPolicyFields: ["egress"]. Container runtimes only. + */ + egress?: "proxied" | "open"; /** Attempt-local non-secret files, staged without replacing the normal adapter. */ workspaceFiles?: WorkspaceFiles; /** Container image tag (container runtimes; alternative to `containerImage`). */ @@ -127,6 +136,11 @@ export function defineExperiment(experiment: Experiment): Experiment { if (experiment.comparisonGroup !== undefined && experiment.comparisonGroup.trim() === "") { throw new Error(`defineExperiment: experiment "${experiment.id}" comparisonGroup must not be empty`); } + if (kind === "host" && experiment.runtime.egress === "open") { + throw new Error( + `defineExperiment: experiment "${experiment.id}" runtime.egress "open" is only meaningful on container runtimes`, + ); + } validateWorkspaceFilesRuntime(experiment.runtime); return deepFreeze(experiment); } diff --git a/tests/experiments-v3.test.ts b/tests/experiments-v3.test.ts index 8a65ffa..e6ca8d4 100644 --- a/tests/experiments-v3.test.ts +++ b/tests/experiments-v3.test.ts @@ -10,6 +10,7 @@ import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { discoverEvals } from "../src/evals/discovery.ts"; +import { pairKey, resolvePairEnvironments } from "../src/execution-policy.ts"; import { discoverExperiments } from "../src/experiments/discovery.ts"; import { compareGroups } from "../src/experiments/groups.ts"; import { defineExperiment, type Experiment, type LoadedExperiment } from "../src/experiments/types.ts"; @@ -95,6 +96,18 @@ describe("defineExperiment", () => { defineExperiment({ id: "x", agent: { adapter: "echo" }, runtime: { kind: "host" }, comparisonGroup: " " }), ).toThrow(/comparisonGroup/); }); + + test('runtime.egress "open" is refused on a host runtime (never proxied) and accepted on a container', () => { + expect(() => + defineExperiment({ id: "open-host", agent: { adapter: "echo" }, runtime: { kind: "host", egress: "open" } }), + ).toThrow(/runtime\.egress "open" is only meaningful on container runtimes/); + const open = defineExperiment({ + id: "open-box", + agent: { adapter: "x" }, + runtime: { kind: "container", image: "img:1", egress: "open" }, + }); + expect(open.runtime.egress).toBe("open"); + }); }); describe("comparison groups (withhold-and-diff)", () => { @@ -224,3 +237,75 @@ describe("comparison groups (withhold-and-diff)", () => { store.close(); }); }); + +describe("per-experiment egress opt-out (runtime.egress)", () => { + const FIXTURE_EVALS = join(import.meta.dir, "__fixtures__", "evals"); + const agent = { adapter: "claude", provider: "anthropic", model: "model-a", reasoning: "high" }; + const helloLocal = () => discoverEvals(FIXTURE_EVALS).filter((loaded) => loaded.metadata.id === "hello-local"); + + /** A container arm that differs from its siblings only in how it reaches the network. */ + function arm(id: string, egress: "proxied" | "open" | undefined, fields: Array<"egress"> = []): LoadedExperiment { + return loadedExperiment({ + id, + comparisonGroup: "e", + treatment: egress === "open" ? "open-web" : "proxied-web", + agent, + runtime: { + kind: "container", + image: "img:1", + command: ["true"], + egressHosts: () => ["docs.example"], + ...(egress !== undefined ? { egress } : {}), + ...(fields.length > 0 ? { treatmentPolicyFields: fields } : {}), + }, + }); + } + + test("under --egress-proxy an open runtime resolves proxy: false while its siblings resolve true; every pair records the allowlist", () => { + const evals = helloLocal(); + const pairs = resolvePairEnvironments({ + evals, + experiments: [arm("open", "open"), arm("proxied", "proxied"), arm("unset", undefined)], + egressProxy: true, + }); + const open = pairs[pairKey("hello-local", "open")]!.egress; + const proxied = pairs[pairKey("hello-local", "proxied")]!.egress; + const unset = pairs[pairKey("hello-local", "unset")]!.egress; + expect(open.proxy).toBe(false); + expect(proxied.proxy).toBe(true); + expect(unset.proxy).toBe(true); + // The open pair's allowlist is inert but still recorded, so identity holds if the run flag flips. + expect(open.allowlist.length).toBeGreaterThan(0); + expect(open.allowlist).toEqual(proxied.allowlist); + }); + + test("without the run flag no pair is proxied, whatever the runtime declares", () => { + const evals = helloLocal(); + const pairs = resolvePairEnvironments({ + evals, + experiments: [arm("open", "open"), arm("proxied", "proxied"), arm("unset", undefined)], + egressProxy: false, + }); + expect(Object.values(pairs).map((pair) => pair.egress.proxy)).toEqual([false, false, false]); + }); + + test("members differing only in egress compare allowed when both exempt it, withheld when one (or neither) does", () => { + const evals = helloLocal(); + const compare = (experiments: LoadedExperiment[]) => { + const pairs = resolvePairEnvironments({ evals, experiments, egressProxy: true }); + const manifest = buildRunManifest({ evals, experiments, pairs }); + return compareGroups(experiments.map((entry) => entry.experiment), manifest)[0]!; + }; + const both = compare([arm("open", "open", ["egress"]), arm("proxied", "proxied", ["egress"])]); + expect(both.comparison).toBe("allowed"); + expect(both.details).toEqual([]); + + const one = compare([arm("open", "open", ["egress"]), arm("proxied", "proxied")]); + expect(one.comparison).toBe("withheld"); + expect(one.details.join("\n")).toContain("egress"); + + const neither = compare([arm("open", "open"), arm("proxied", "proxied")]); + expect(neither.comparison).toBe("withheld"); + expect(neither.details.join("\n")).toContain("egress"); + }); +}); From 4c809500742495d69fe18b647aaadcdb217325fd Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 19 Oct 2026 14:06:51 -0400 Subject: [PATCH 3/3] fix(runtime): keep endpoint-override hosts out of unproxied pairs' allowlists Under --egress-proxy the runner appended every container agent's endpoint- override hosts to its allowlist for the whole run, and pair resolution composed that list for every pair, open or not. The composer refuses IP literals by design, so an open arm whose gateway is an IP (the very thing an open arm exists to reach) stopped the run before any attempt started. Pair resolution now decides "proxied" once per pair and uses it for both the pair's egress.proxy and whether endpoint-override hosts join the allowlist; the runner passes those hosts unconditionally as their own option. An open pair's recorded allowlist is therefore the same with or without the run flag, and a proxied arm behind an IP-literal gateway is still refused loudly. Addresses the Greptile P1 on #41. Spec: docs/ideation/discovery-suite/spec-phase-1.md --- docs/extensions.md | 7 ++++--- src/execution-policy.ts | 22 +++++++++++++++++----- src/runner/execute.ts | 15 +++++++-------- tests/experiments-v3.test.ts | 21 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index b7fce23..f8cd627 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -113,9 +113,10 @@ inside a proxied run (no internal network, no sidecar, no proxy environment), so an arm that must reach the open internet can share a run with proxied arms. The choice is recorded in each pair environment, so it is identity; exempt it in a comparison group with `treatmentPolicyFields: ["egress"]`. An open pair still -records its (inert) allowlist, writes no denials log and reports egress as not -instrumented. Container runtimes only: `defineExperiment` rejects `"open"` on a -host runtime. The allowlist is the pair's runtime +records its (inert) allowlist of runtime and adapter hosts, writes no denials +log and reports egress as not instrumented; endpoint-override hosts join only +proxied pairs, so an IP-literal gateway on an open arm never stops the run. +Container runtimes only: `defineExperiment` rejects `"open"` on a host runtime. The allowlist is the pair's runtime hosts plus that adapter's provider endpoints; absent runtime hosts means provider endpoints only. Each proxied attempt has its own internal network and egress proxy. A shared MCP auth proxy has its own infrastructure traffic, which is not attributed diff --git a/src/execution-policy.ts b/src/execution-policy.ts index 989ecc5..b0e6e19 100644 --- a/src/execution-policy.ts +++ b/src/execution-policy.ts @@ -53,6 +53,13 @@ export function resolvePairEnvironments(options: { images?: Record; egressProxy?: boolean; adapterHosts?: Record; + /** + * Per experiment: the hosts of the agent's endpoint overrides. They join + * the allowlist only for pairs the proxy fronts, where the list is live and + * an IP-literal gateway is refused loudly by design; an open or unproxied + * pair never carries them, so such a gateway cannot stop the run. + */ + adapterEndpointHosts?: Record; /** Per experiment: the agent's non-secret endpoint overrides (see PairEnvironment.endpoints). */ adapterEndpoints?: Record>; budgets?: Partial; @@ -73,18 +80,23 @@ export function resolvePairEnvironments(options: { options.imageTags?.[pairKey(loaded.metadata.id, experiment.id)] ?? runtime.image ?? runtime.containerImage?.(loaded.metadata); + // With the flag, every container is proxied unless its runtime opts + // out with egress: "open". No runtime hosts means only the adapter's + // provider endpoints are permitted. An open pair still records the + // runtime and adapter hosts (inert) so identity holds if the run flag + // flips, but endpoint-override hosts join only a live allowlist: the + // proxy refuses IP literals, and an IP-literal gateway on an arm the + // proxy never fronts must not stop the run. + const proxied = runtime.kind === "container" && (options.egressProxy ?? false) && runtime.egress !== "open"; pairs[pairKey(loaded.metadata.id, experiment.id)] = { ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}), image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null, - // With the flag, every container is proxied unless its runtime opts - // out with egress: "open". No runtime hosts means only the adapter's - // provider endpoints are permitted; an open pair still records its - // allowlist (inert) so identity holds if the run flag flips. egress: { - proxy: runtime.kind === "container" && (options.egressProxy ?? false) && runtime.egress !== "open", + proxy: proxied, allowlist: composeAllowlistFromHosts([ ...(runtime.egressHosts?.(loaded.metadata) ?? []), ...(options.adapterHosts?.[experiment.id] ?? []), + ...(proxied ? (options.adapterEndpointHosts?.[experiment.id] ?? []) : []), ]), }, webPolicy: runtime.webPolicy?.() ?? "native-web-blocked", diff --git a/src/runner/execute.ts b/src/runner/execute.ts index 8f8795e..ced770b 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -471,20 +471,19 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise [id, collectOptionalHostEnv([adapter])] as const)); const adapterEndpoints = Object.fromEntries([...overridesByExperiment].flatMap(([id, overrides]) => { const endpoints = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name))); return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : []; })); const pairs = resolvePairEnvironments({ ...options, workspaceFilesByExperiment, images, imageTags: imageTagsByPair, - adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [ - ...(adapter.egressHosts ?? []), - ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []), - ]])), + adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [...(adapter.egressHosts ?? [])]])), + adapterEndpointHosts: Object.fromEntries([...overridesByExperiment].map(([id, overrides]) => [id, endpointHosts(overrides)])), adapterEndpoints, budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs }, mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])), diff --git a/tests/experiments-v3.test.ts b/tests/experiments-v3.test.ts index e6ca8d4..2b020b4 100644 --- a/tests/experiments-v3.test.ts +++ b/tests/experiments-v3.test.ts @@ -289,6 +289,27 @@ describe("per-experiment egress opt-out (runtime.egress)", () => { expect(Object.values(pairs).map((pair) => pair.egress.proxy)).toEqual([false, false, false]); }); + test("endpoint-override hosts join only proxied pairs: an IP-literal gateway on an open arm never stops the run", () => { + const evals = helloLocal(); + const experiments = [arm("open", "open"), arm("proxied", "proxied")]; + const adapterEndpointHosts = { open: ["10.0.0.5"], proxied: ["gateway.example"] }; + const pairs = resolvePairEnvironments({ evals, experiments, egressProxy: true, adapterEndpointHosts }); + const open = pairs[pairKey("hello-local", "open")]!.egress; + const proxied = pairs[pairKey("hello-local", "proxied")]!.egress; + expect(open.proxy).toBe(false); + expect(open.allowlist).not.toContain("10.0.0.5"); + expect(proxied.proxy).toBe(true); + expect(proxied.allowlist).toContain("gateway.example"); + // An open pair's allowlist is the same with or without the run flag; a proxied pair's only carries the gateway when live. + const unproxied = resolvePairEnvironments({ evals, experiments, egressProxy: false, adapterEndpointHosts }); + expect(unproxied[pairKey("hello-local", "open")]!.egress.allowlist).toEqual(open.allowlist); + expect(unproxied[pairKey("hello-local", "proxied")]!.egress.allowlist).not.toContain("gateway.example"); + // A proxied arm behind an IP-literal gateway is still refused loudly: the proxy cannot front it. + expect(() => + resolvePairEnvironments({ evals, experiments, egressProxy: true, adapterEndpointHosts: { proxied: ["10.0.0.5"] } }), + ).toThrow(/IP literals are refused by the proxy/); + }); + test("members differing only in egress compare allowed when both exempt it, withheld when one (or neither) does", () => { const evals = helloLocal(); const compare = (experiments: LoadedExperiment[]) => {