feat(adapters): endpoint overrides for routing through an LLM gateway

README.md

@@ -118,6 +118,32 @@ experiment module — changing them changes the experiment's identity, by
 design. See the benchmark repository's `examples/weekly-sweep.yml` for what
 a full scheduled sweep costs and looks like in CI.
 
+### Routing through an LLM gateway
+
+The vendor CLIs and the diagnosis analyst all honor endpoint overrides, so a
+gateway or proxy (Cloudflare AI Gateway, LiteLLM, a corporate egress) is an
+env change, not a code change. Each adapter declares its overrides
+(`optionalHostEnv`); the harness forwards them into attempt containers only
+when set, adds a base URL's host to the egress allowlist, and points the
+preflight key probe at it:
+
+| Agent  | Overrides                                        | How the CLI gets them                                                                  |
+| ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------- |
+| claude | `ANTHROPIC_BASE_URL`, `ANTHROPIC_CUSTOM_HEADERS` | env, read natively by Claude Code (and by the `diagnose` analyst)                      |
+| codex  | `OPENAI_BASE_URL`                                | rendered into `config.toml` as a custom model provider (codex ignores the env var)     |
+
+`ANTHROPIC_CUSTOM_HEADERS` is `Name: Value` lines and is redacted from
+artifacts like a key. With Cloudflare AI Gateway holding the provider keys,
+the gateway token *is* the API key:
+
+```bash
+export ANTHROPIC_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/anthropic
+export ANTHROPIC_API_KEY=<gateway token>
+export ANTHROPIC_CUSTOM_HEADERS="cf-aig-authorization: Bearer <gateway token>"
+export OPENAI_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/openai
+export OPENAI_API_KEY=<gateway token>
+```
+
 ## How it works
 
 Every attempt is one point in a 2-D plan:
@@ -181,7 +207,9 @@ failing pair's harness-selected evidence (per-check tallies, transcript
 excerpts, egress denials) to a structured LLM call and writes `diagnosis.json`
 beside `report.json` — findings are evidence-linked hypotheses, never
 verdicts, keeping the report itself attribution-free. Requires
-`ANTHROPIC_API_KEY`; the UI renders the result as a Diagnosis view.
+`ANTHROPIC_API_KEY` (and follows the claude adapter's gateway overrides, see
+[Routing through an LLM gateway](#routing-through-an-llm-gateway)); the UI
+renders the result as a Diagnosis view.
 
 ## Writing evals
 

docs/getting-started.md

@@ -17,7 +17,7 @@ keys**. If a term is unfamiliar, [Concepts](./concepts.md) defines it.
 | ------------------------- | -------------------------------------------------------------------- |
 | **[Bun](https://bun.sh)** | only to build the binary or run from source                          |
 | **Docker**                | only to run _real_ agents in isolated containers                     |
-| **API keys**              | only for real LLM agents (`ANTHROPIC_API_KEY`, …)                    |
+| **API keys**              | only for real LLM agents (`ANTHROPIC_API_KEY`, …); a gateway additionally needs its base URL, with the gateway token as the key variable — see the README |
 
 The keyless act needs none of Docker, keys, or network — the `echo` agent
 runs the full run → score → persist pipeline on the host.

src/adapters/claude.ts

@@ -14,6 +14,7 @@
  */
 
 import type { McpServerConfig } from "../isolation/mcp.ts";
+import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseClaudeStream } from "./parse.ts";
 import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
 
@@ -38,10 +39,21 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
   readonly identity;
   readonly versionCommand = ["claude", "--version"] as const;
   readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const;
+  /**
+   * Endpoint overrides, forwarded only when set. ANTHROPIC_BASE_URL routes
+   * the CLI (and the key probe) through an Anthropic-compatible gateway —
+   * e.g. Cloudflare AI Gateway's `/anthropic` endpoint, where the gateway
+   * token doubles as ANTHROPIC_API_KEY; ANTHROPIC_CUSTOM_HEADERS (`Name:
+   * Value` lines) carries whatever else that gateway wants, such as
+   * `cf-aig-authorization` or `cf-aig-metadata`. Claude Code reads both
+   * natively, so no flag changes.
+   */
+  readonly optionalHostEnv = ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"] as const;
   /**
    * Egress-proxy allowlist (harness-default layer). Claude Code documents
    * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is
-   * the LLM endpoint, statsig/sentry are its telemetry (allowed so denial
+   * the default LLM endpoint (an ANTHROPIC_BASE_URL override adds its own
+   * host at run time), statsig/sentry are its telemetry (allowed so denial
    * logs stay signal, not noise). Verified live by the egress-proxy regression
    * check (`bun scripts/dev-attempt.ts --experiment <id> --egress-proxy`).
    */
@@ -66,8 +78,12 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: "https://api.anthropic.com/v1/models",
-      headers: { "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", "anthropic-version": "2023-06-01" },
+      url: endpointUrl(env["ANTHROPIC_BASE_URL"], "https://api.anthropic.com", "v1/models"),
+      headers: {
+        ...parseHeaderLines(env["ANTHROPIC_CUSTOM_HEADERS"]),
+        "x-api-key": env["ANTHROPIC_API_KEY"] ?? "",
+        "anthropic-version": "2023-06-01",
+      },
     };
   }
 

src/adapters/codex.ts

@@ -9,15 +9,43 @@
  * container create, but codex >= 0.144 no longer reads it directly — the
  * command wrapper pipes it into `codex login --with-api-key` (stdin, never
  * argv) to write the auth.json the CLI requires.
+ *
+ * Endpoint override: an OPENAI_BASE_URL set on the host (an LLM gateway) is
+ * rendered into the same config.toml as a custom model provider — codex
+ * 0.149 ignores the env var for its built-in provider (verified live: it
+ * still dialed api.openai.com), while a `[model_providers.*]` entry with
+ * `env_key = "OPENAI_API_KEY"` routes every request through the override.
  */
 
 import type { McpServerConfig } from "../isolation/mcp.ts";
+import { endpointUrl } from "../secrets.ts";
 import { parseCodexStream } from "./parse.ts";
 import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
 
 /** Container path of the Codex config (containers run as root). */
 export const CODEX_CONFIG_PATH = "/root/.codex/config.toml";
 
+/** The config.toml model provider id an OPENAI_BASE_URL override is rendered as. */
+export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway";
+
+/**
+ * A custom model provider pointing at `baseUrl` (the Responses API, which
+ * codex requires), selected as the default. Top-level keys must precede any
+ * table, so this block always renders first in the file.
+ */
+function renderCodexProviderToml(baseUrl: string): string {
+  return [
+    `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`,
+    "",
+    `[model_providers.${CODEX_GATEWAY_PROVIDER}]`,
+    'name = "OpenAI (via OPENAI_BASE_URL)"',
+    `base_url = ${JSON.stringify(baseUrl)}`,
+    'env_key = "OPENAI_API_KEY"',
+    'wire_api = "responses"',
+    "",
+  ].join("\n");
+}
+
 /**
  * `config.toml` MCP table rendered from the runtime's MCP servers (url-based,
  * credential-free — authed servers already point at the token-proxy sidecar).
@@ -37,13 +65,22 @@ export class CodexAdapter implements ContainerAgentAdapter {
   readonly identity;
   readonly versionCommand = ["codex", "--version"] as const;
   readonly requiredHostEnv = ["OPENAI_API_KEY"] as const;
+  /**
+   * Endpoint override, forwarded only when set: OPENAI_BASE_URL (vendor
+   * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's
+   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY.
+   * Rendered into config.toml by setupFiles (see the module comment); the
+   * key probe follows it.
+   */
+  readonly optionalHostEnv = ["OPENAI_BASE_URL"] as const;
   /**
    * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based;
    * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) —
    * verified live by the egress-proxy regression check (the CONNECT tunnel
    * to api.openai.com round-tripped). chatgpt.com came from that run's
    * denial log: codex probes its ChatGPT-auth endpoints even under key auth.
-   * (Its github.com update-check denials are left denied on purpose.)
+   * (Its github.com update-check denials are left denied on purpose.) An
+   * OPENAI_BASE_URL override adds its own host at run time.
    */
   readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const;
 
@@ -66,14 +103,18 @@ export class CodexAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: "https://api.openai.com/v1/models",
+      url: endpointUrl(env["OPENAI_BASE_URL"], "https://api.openai.com/v1", "models"),
       headers: { Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
     };
   }
 
   setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
-    if (ctx.mcpServers === undefined) return {};
-    return { [CODEX_CONFIG_PATH]: renderCodexMcpToml(ctx.mcpServers) };
+    const sections: string[] = [];
+    const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"];
+    if (baseUrl !== undefined) sections.push(renderCodexProviderToml(baseUrl));
+    if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers));
+    if (sections.length === 0) return {};
+    return { [CODEX_CONFIG_PATH]: sections.join("\n") };
   }
 
   command(ctx: ContainerAttemptContext): string[] {

src/adapters/types.ts

@@ -132,6 +132,13 @@ export interface ContainerAttemptContext {
   mcpServers?: Readonly<Record<string, McpServerConfig>>;
   /** The runtime's native web-tool policy (default: blocked). */
   webPolicy: WebPolicy;
+  /**
+   * The adapter's endpoint overrides that are set on the host (its
+   * `optionalHostEnv` minus secret-named entries — never provider keys or
+   * header bundles), for a CLI that takes a config-file entry rather than an
+   * env var. Absent when none are set.
+   */
+  overrides?: Readonly<Record<string, string>>;
 }
 
 /**
@@ -153,17 +160,27 @@ export interface ContainerAgentAdapter {
   /** Command used during preflight to record the executable's exact version. */
   versionCommand?: readonly string[];
   /**
-   * Cheap authenticated request composed from the collected provider keys,
-   * probed at run preflight so a present-but-rejected key aborts BEFORE any
-   * container spend. A 401/403 response stops the run; network failures only
-   * warn — an unreachable probe must not gate a run the agent might complete.
+   * Cheap authenticated request composed from the collected provider keys
+   * (plus any `optionalHostEnv` overrides that are set), probed at run
+   * preflight so a present-but-rejected key aborts BEFORE any container
+   * spend. A 401/403 response stops the run; network failures only warn — an
+   * unreachable probe must not gate a run the agent might complete.
    */
   keyProbe?(env: Record<string, string>): { url: string; headers: Record<string, string> };
   /**
    * Host env vars that must be present before any container is created
    * (shared provider keys — see src/secrets.ts).
    */
   requiredHostEnv: readonly string[];
+  /**
+   * Host env vars forwarded into the container when set, never required:
+   * endpoint overrides such as a vendor base URL that routes the CLI through
+   * an LLM gateway or proxy, and the custom headers that gateway needs. A
+   * value that is an absolute URL also adds its host to the egress allowlist
+   * and retargets the key probe; secret-named values (headers carrying a
+   * bearer token) are redacted from artifacts like provider keys.
+   */
+  optionalHostEnv?: readonly string[];
   /**
    * Hosts this agent's CLI must reach (its LLM/vendor endpoints) — the
    * harness-default layer of the egress allowlist when the run uses the

src/execution-policy.ts

@@ -27,6 +27,13 @@ export interface PairEnvironment {
   /** Runtime text prepended to the eval prompt for this pair (null = prompt verbatim). */
   promptPreamble: string | null;
   budgets: LifecycleBudgets;
+  /**
+   * The agent's endpoint overrides in force (non-secret base URLs, e.g.
+   * OPENAI_BASE_URL → an LLM gateway route), recorded in full so a changed
+   * route changes identity and a resume cannot mix routes. Absent when the
+   * agent runs vendor-direct — such pairs hash exactly as before.
+   */
+  endpoints?: Record<string, string>;
 }
 
 /** The prompt an agent receives: the runtime's preamble (if any), a blank line, the eval's verbatim body. */
@@ -43,6 +50,8 @@ export function resolvePairEnvironments(options: {
   images?: Record<string, RuntimeImageIdentity>;
   egressProxy?: boolean;
   adapterHosts?: Record<string, readonly string[]>;
+  /** Per experiment: the agent's non-secret endpoint overrides (see PairEnvironment.endpoints). */
+  adapterEndpoints?: Record<string, Record<string, string>>;
   budgets?: Partial<LifecycleBudgets>;
   mcpHashes?: Record<string, string>;
   imageTags?: Record<string, string>;
@@ -77,6 +86,10 @@ export function resolvePairEnvironments(options: {
           return text === "" ? null : text;
         })(),
         budgets: { ...budgets },
+        ...(() => {
+          const endpoints = options.adapterEndpoints?.[experiment.id];
+          return endpoints !== undefined && Object.keys(endpoints).length > 0 ? { endpoints: { ...endpoints } } : {};
+        })(),
       };
     }
   return pairs;

src/isolation/docker.ts

@@ -59,15 +59,20 @@ export interface ExecResult {
   timedOut: boolean;
 }
 
-/** Run `docker <args>`, capturing output. Never throws for non-zero exits. */
+/**
+ * Run `docker <args>`, capturing output. Never throws for non-zero exits.
+ * `env` is added to the docker CLI's own environment — the channel for
+ * values named on argv by `-e KEY` alone.
+ */
 async function runDocker(
   args: string[],
-  opts: { timeoutMs?: number; stdin?: string } = {},
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
 ): Promise<ExecResult> {
   const proc = Bun.spawn(["docker", ...args], {
     stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin),
     stdout: "pipe",
     stderr: "pipe",
+    ...(opts.env !== undefined ? { env: { ...process.env, ...opts.env } } : {}),
   });
   let timedOut = false;
   let timer: ReturnType<typeof setTimeout> | undefined;
@@ -87,7 +92,10 @@ async function runDocker(
 }
 
 /** Run `docker <args>` and throw a DockerError on non-zero exit. */
-async function mustDocker(args: string[], opts: { timeoutMs?: number; stdin?: string } = {}): Promise<ExecResult> {
+async function mustDocker(
+  args: string[],
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+): Promise<ExecResult> {
   const result = await runDocker(args, opts);
   if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr);
   return result;
@@ -198,36 +206,55 @@ export interface AttemptContainer {
  * fixture into /workspace. On any failure the partial container is removed
  * before the error propagates — a failed create never leaks a container.
  */
+/**
+ * Route an attempt's env between its two argv-free channels. Single-line
+ * values become env-file lines; a value that itself contains newlines (a
+ * `Name: Value` header bundle) cannot be expressed in a line-based env-file,
+ * so it rides the docker CLI's own environment instead, named on argv by
+ * `-e KEY` alone — the CLI forwards the value intact and argv never shows it.
+ */
+export function splitAttemptEnv(env: Record<string, string>): { envFileLines: string[]; inherited: Record<string, string> } {
+  const envFileLines: string[] = [];
+  const inherited: Record<string, string> = {};
+  for (const [key, value] of Object.entries(env)) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`);
+    if (value.includes("\n")) inherited[key] = value;
+    else envFileLines.push(`${key}=${value}`);
+  }
+  return { envFileLines, inherited };
+}
+
 export async function createAttemptContainer(opts: CreateAttemptContainerOptions): Promise<AttemptContainer> {
   // Secrets travel via a transient env-file, not argv (argv is visible in
-  // host process listings while `docker run` executes).
+  // host process listings while `docker run` executes); multi-line values
+  // take the CLI-environment channel (see splitAttemptEnv).
   const envDir = await mkdtemp(join(tmpdir(), "quickstudy-env-"));
   const envFile = join(envDir, "attempt.env");
-  const lines = Object.entries(opts.env).map(([key, value]) => {
-    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`);
-    if (value.includes("\n")) throw new Error(`environment variable "${key}" contains a newline`);
-    return `${key}=${value}`;
-  });
-  await writeFile(envFile, lines.join("\n") + (lines.length > 0 ? "\n" : ""), { mode: 0o600 });
+  const { envFileLines, inherited } = splitAttemptEnv(opts.env);
+  await writeFile(envFile, envFileLines.join("\n") + (envFileLines.length > 0 ? "\n" : ""), { mode: 0o600 });
 
   let id: string;
   try {
-    const created = await mustDocker([
-      "run",
-      "-d",
-      "--label",
-      `${ATTEMPT_LABEL}=${opts.attemptId}`,
-      "--env-file",
-      envFile,
-      "-v",
-      `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`,
-      "-w",
-      CONTAINER_WORKSPACE,
-      ...(opts.network !== undefined ? ["--network", opts.network] : []),
-      opts.image,
-      "sleep",
-      "infinity",
-    ]);
+    const created = await mustDocker(
+      [
+        "run",
+        "-d",
+        "--label",
+        `${ATTEMPT_LABEL}=${opts.attemptId}`,
+        "--env-file",
+        envFile,
+        ...Object.keys(inherited).flatMap((key) => ["-e", key]),
+        "-v",
+        `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`,
+        "-w",
+        CONTAINER_WORKSPACE,
+        ...(opts.network !== undefined ? ["--network", opts.network] : []),
+        opts.image,
+        "sleep",
+        "infinity",
+      ],
+      { env: inherited },
+    );
     id = created.stdout.trim();
   } catch (err) {
     // `docker run` can fail AFTER creating the container (start failure) —