From eab175656d731ba4a7d5a9e3e32d112967896b24 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 16 Oct 2026 13:46:34 -0400 Subject: [PATCH 1/2] feat(adapters): endpoint overrides for routing through an LLM gateway Container adapters can declare `optionalHostEnv`: endpoint overrides (ANTHROPIC_BASE_URL, ANTHROPIC_CUSTOM_HEADERS; OPENAI_BASE_URL) that are forwarded into attempt containers only when set, so a gateway or proxy is an env change rather than a code change. The harness adds an override's host to the egress allowlist, retargets the preflight key probe at it, and redacts header bundles from artifacts like a key; base URLs stay diagnostic. The codex adapter renders OPENAI_BASE_URL into ~/.codex/config.toml as a custom model provider: codex 0.149.1 ignores the env var for its built-in provider (verified in the pinned image). The diagnose analyst honors the same Anthropic overrides so it takes the agent's route. --- README.md | 30 ++++++++++++- docs/getting-started.md | 2 +- src/adapters/claude.ts | 22 ++++++++-- src/adapters/codex.ts | 49 +++++++++++++++++++-- src/adapters/types.ts | 25 +++++++++-- src/llm.ts | 14 ++++-- src/runner/execute.ts | 19 ++++++-- src/secrets.ts | 83 ++++++++++++++++++++++++++++++++++- tests/adapters.test.ts | 54 ++++++++++++++++++++++- tests/docs-smoke-inventory.ts | 6 +++ tests/secrets.test.ts | 62 ++++++++++++++++++++++++++ 11 files changed, 345 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index a539861..f442bf8 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,32 @@ experiment module — changing them changes the experiment's identity, by design. See the benchmark repository's `examples/weekly-sweep.yml` for what a full scheduled sweep costs and looks like in CI. +### Routing through an LLM gateway + +The vendor CLIs and the diagnosis analyst all honor endpoint overrides, so a +gateway or proxy (Cloudflare AI Gateway, LiteLLM, a corporate egress) is an +env change, not a code change. Each adapter declares its overrides +(`optionalHostEnv`); the harness forwards them into attempt containers only +when set, adds a base URL's host to the egress allowlist, and points the +preflight key probe at it: + +| Agent | Overrides | How the CLI gets them | +| ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------- | +| claude | `ANTHROPIC_BASE_URL`, `ANTHROPIC_CUSTOM_HEADERS` | env, read natively by Claude Code (and by the `diagnose` analyst) | +| codex | `OPENAI_BASE_URL` | rendered into `config.toml` as a custom model provider (codex ignores the env var) | + +`ANTHROPIC_CUSTOM_HEADERS` is `Name: Value` lines and is redacted from +artifacts like a key. With Cloudflare AI Gateway holding the provider keys, +the gateway token *is* the API key: + +```bash +export ANTHROPIC_BASE_URL=https://gateway.ai.cloudflare.com/v1///anthropic +export ANTHROPIC_API_KEY= +export ANTHROPIC_CUSTOM_HEADERS="cf-aig-authorization: Bearer " +export OPENAI_BASE_URL=https://gateway.ai.cloudflare.com/v1///openai +export OPENAI_API_KEY= +``` + ## How it works Every attempt is one point in a 2-D plan: @@ -181,7 +207,9 @@ failing pair's harness-selected evidence (per-check tallies, transcript excerpts, egress denials) to a structured LLM call and writes `diagnosis.json` beside `report.json` — findings are evidence-linked hypotheses, never verdicts, keeping the report itself attribution-free. Requires -`ANTHROPIC_API_KEY`; the UI renders the result as a Diagnosis view. +`ANTHROPIC_API_KEY` (and follows the claude adapter's gateway overrides, see +[Routing through an LLM gateway](#routing-through-an-llm-gateway)); the UI +renders the result as a Diagnosis view. ## Writing evals diff --git a/docs/getting-started.md b/docs/getting-started.md index 7c4e322..5f5a0cb 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -17,7 +17,7 @@ keys**. If a term is unfamiliar, [Concepts](./concepts.md) defines it. | ------------------------- | -------------------------------------------------------------------- | | **[Bun](https://bun.sh)** | only to build the binary or run from source | | **Docker** | only to run _real_ agents in isolated containers | -| **API keys** | only for real LLM agents (`ANTHROPIC_API_KEY`, …) | +| **API keys** | only for real LLM agents (`ANTHROPIC_API_KEY`, …; a gateway needs only `ANTHROPIC_BASE_URL` — see the README) | The keyless act needs none of Docker, keys, or network — the `echo` agent runs the full run → score → persist pipeline on the host. diff --git a/src/adapters/claude.ts b/src/adapters/claude.ts index ec6143c..b0121c5 100644 --- a/src/adapters/claude.ts +++ b/src/adapters/claude.ts @@ -14,6 +14,7 @@ */ import type { McpServerConfig } from "../isolation/mcp.ts"; +import { endpointUrl, parseHeaderLines } from "../secrets.ts"; import { parseClaudeStream } from "./parse.ts"; import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts"; @@ -38,10 +39,21 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { readonly identity; readonly versionCommand = ["claude", "--version"] as const; readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const; + /** + * Endpoint overrides, forwarded only when set. ANTHROPIC_BASE_URL routes + * the CLI (and the key probe) through an Anthropic-compatible gateway — + * e.g. Cloudflare AI Gateway's `/anthropic` endpoint, where the gateway + * token doubles as ANTHROPIC_API_KEY; ANTHROPIC_CUSTOM_HEADERS (`Name: + * Value` lines) carries whatever else that gateway wants, such as + * `cf-aig-authorization` or `cf-aig-metadata`. Claude Code reads both + * natively, so no flag changes. + */ + readonly optionalHostEnv = ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"] as const; /** * Egress-proxy allowlist (harness-default layer). Claude Code documents * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is - * the LLM endpoint, statsig/sentry are its telemetry (allowed so denial + * the default LLM endpoint (an ANTHROPIC_BASE_URL override adds its own + * host at run time), statsig/sentry are its telemetry (allowed so denial * logs stay signal, not noise). Verified live by the egress-proxy regression * check (`bun scripts/dev-attempt.ts --experiment --egress-proxy`). */ @@ -66,8 +78,12 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter { keyProbe(env: Record): { url: string; headers: Record } { return { - url: "https://api.anthropic.com/v1/models", - headers: { "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", "anthropic-version": "2023-06-01" }, + url: endpointUrl(env["ANTHROPIC_BASE_URL"], "https://api.anthropic.com", "v1/models"), + headers: { + ...parseHeaderLines(env["ANTHROPIC_CUSTOM_HEADERS"]), + "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", + "anthropic-version": "2023-06-01", + }, }; } diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index 798f1dd..bac9c59 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -9,15 +9,43 @@ * container create, but codex >= 0.144 no longer reads it directly — the * command wrapper pipes it into `codex login --with-api-key` (stdin, never * argv) to write the auth.json the CLI requires. + * + * Endpoint override: an OPENAI_BASE_URL set on the host (an LLM gateway) is + * rendered into the same config.toml as a custom model provider — codex + * 0.149 ignores the env var for its built-in provider (verified live: it + * still dialed api.openai.com), while a `[model_providers.*]` entry with + * `env_key = "OPENAI_API_KEY"` routes every request through the override. */ import type { McpServerConfig } from "../isolation/mcp.ts"; +import { endpointUrl } from "../secrets.ts"; import { parseCodexStream } from "./parse.ts"; import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts"; /** Container path of the Codex config (containers run as root). */ export const CODEX_CONFIG_PATH = "/root/.codex/config.toml"; +/** The config.toml model provider id an OPENAI_BASE_URL override is rendered as. */ +export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway"; + +/** + * A custom model provider pointing at `baseUrl` (the Responses API, which + * codex requires), selected as the default. Top-level keys must precede any + * table, so this block always renders first in the file. + */ +function renderCodexProviderToml(baseUrl: string): string { + return [ + `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`, + "", + `[model_providers.${CODEX_GATEWAY_PROVIDER}]`, + 'name = "OpenAI (via OPENAI_BASE_URL)"', + `base_url = ${JSON.stringify(baseUrl)}`, + 'env_key = "OPENAI_API_KEY"', + 'wire_api = "responses"', + "", + ].join("\n"); +} + /** * `config.toml` MCP table rendered from the runtime's MCP servers (url-based, * credential-free — authed servers already point at the token-proxy sidecar). @@ -37,13 +65,22 @@ export class CodexAdapter implements ContainerAgentAdapter { readonly identity; readonly versionCommand = ["codex", "--version"] as const; readonly requiredHostEnv = ["OPENAI_API_KEY"] as const; + /** + * Endpoint override, forwarded only when set: OPENAI_BASE_URL (vendor + * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's + * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY. + * Rendered into config.toml by setupFiles (see the module comment); the + * key probe follows it. + */ + readonly optionalHostEnv = ["OPENAI_BASE_URL"] as const; /** * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based; * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) — * verified live by the egress-proxy regression check (the CONNECT tunnel * to api.openai.com round-tripped). chatgpt.com came from that run's * denial log: codex probes its ChatGPT-auth endpoints even under key auth. - * (Its github.com update-check denials are left denied on purpose.) + * (Its github.com update-check denials are left denied on purpose.) An + * OPENAI_BASE_URL override adds its own host at run time. */ readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const; @@ -66,14 +103,18 @@ export class CodexAdapter implements ContainerAgentAdapter { keyProbe(env: Record): { url: string; headers: Record } { return { - url: "https://api.openai.com/v1/models", + url: endpointUrl(env["OPENAI_BASE_URL"], "https://api.openai.com/v1", "models"), headers: { Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` }, }; } setupFiles(ctx: ContainerAttemptContext): Record { - if (ctx.mcpServers === undefined) return {}; - return { [CODEX_CONFIG_PATH]: renderCodexMcpToml(ctx.mcpServers) }; + const sections: string[] = []; + const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"]; + if (baseUrl !== undefined) sections.push(renderCodexProviderToml(baseUrl)); + if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers)); + if (sections.length === 0) return {}; + return { [CODEX_CONFIG_PATH]: sections.join("\n") }; } command(ctx: ContainerAttemptContext): string[] { diff --git a/src/adapters/types.ts b/src/adapters/types.ts index a17f815..eea04c7 100644 --- a/src/adapters/types.ts +++ b/src/adapters/types.ts @@ -132,6 +132,13 @@ export interface ContainerAttemptContext { mcpServers?: Readonly>; /** The runtime's native web-tool policy (default: blocked). */ webPolicy: WebPolicy; + /** + * The adapter's endpoint overrides that are set on the host (its + * `optionalHostEnv` minus secret-named entries — never provider keys or + * header bundles), for a CLI that takes a config-file entry rather than an + * env var. Absent when none are set. + */ + overrides?: Readonly>; } /** @@ -153,10 +160,11 @@ export interface ContainerAgentAdapter { /** Command used during preflight to record the executable's exact version. */ versionCommand?: readonly string[]; /** - * Cheap authenticated request composed from the collected provider keys, - * probed at run preflight so a present-but-rejected key aborts BEFORE any - * container spend. A 401/403 response stops the run; network failures only - * warn — an unreachable probe must not gate a run the agent might complete. + * Cheap authenticated request composed from the collected provider keys + * (plus any `optionalHostEnv` overrides that are set), probed at run + * preflight so a present-but-rejected key aborts BEFORE any container + * spend. A 401/403 response stops the run; network failures only warn — an + * unreachable probe must not gate a run the agent might complete. */ keyProbe?(env: Record): { url: string; headers: Record }; /** @@ -164,6 +172,15 @@ export interface ContainerAgentAdapter { * (shared provider keys — see src/secrets.ts). */ requiredHostEnv: readonly string[]; + /** + * Host env vars forwarded into the container when set, never required: + * endpoint overrides such as a vendor base URL that routes the CLI through + * an LLM gateway or proxy, and the custom headers that gateway needs. A + * value that is an absolute URL also adds its host to the egress allowlist + * and retargets the key probe; secret-named values (headers carrying a + * bearer token) are redacted from artifacts like provider keys. + */ + optionalHostEnv?: readonly string[]; /** * Hosts this agent's CLI must reach (its LLM/vendor endpoints) — the * harness-default layer of the egress allowlist when the run uses the diff --git a/src/llm.ts b/src/llm.ts index 3fa45ed..17d5f67 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -14,6 +14,7 @@ import Anthropic from "@anthropic-ai/sdk"; import type { z } from "zod"; +import { parseHeaderLines } from "./secrets.ts"; /** One structured-output request. `schema` is a JSON Schema object. */ export interface StructuredRequest { @@ -47,16 +48,23 @@ export class ModelOutputError extends Error { const API_MAX_RETRIES = 3; /** - * Anthropic-backed client. Reuses ANTHROPIC_API_KEY from the environment; - * transport-level retries are the SDK's (429/5xx/connection errors). + * Anthropic-backed client. Reuses ANTHROPIC_API_KEY and ANTHROPIC_BASE_URL + * from the environment (the SDK reads both), plus ANTHROPIC_CUSTOM_HEADERS + * (`Name: Value` lines — Claude Code's convention) as default headers, so the + * analyst takes the same route as the agent CLI when both are pointed at an + * LLM gateway. Transport-level retries are the SDK's (429/5xx/connection + * errors). */ export class AnthropicModelClient implements StructuredModelClient { private readonly client: Anthropic; private lastResolvedModel: string | null = null; - constructor(options: { apiKey?: string } = {}) { + constructor(options: { apiKey?: string; baseURL?: string; defaultHeaders?: Record } = {}) { + const defaultHeaders = options.defaultHeaders ?? parseHeaderLines(process.env["ANTHROPIC_CUSTOM_HEADERS"]); this.client = new Anthropic({ ...(options.apiKey !== undefined ? { apiKey: options.apiKey } : {}), + ...(options.baseURL !== undefined ? { baseURL: options.baseURL } : {}), + ...(Object.keys(defaultHeaders).length > 0 ? { defaultHeaders } : {}), maxRetries: API_MAX_RETRIES, }); } diff --git a/src/runner/execute.ts b/src/runner/execute.ts index 49a37d8..e3ec482 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -70,7 +70,7 @@ import { attemptIdentityHash, buildRunManifest, type RunManifest, type RuntimeIm import { isMovingModelAlias } from "../model-identity.ts"; import type { AttemptPlan } from "../plan.ts"; import type { ProvisionedEnvironment, RuntimeProvisionContext } from "../runtime/types.ts"; -import { collectProviderKeys, hostSecretEnv, redactSecrets, validateProviderKeys, type ProviderKeyProbe } from "../secrets.ts"; +import { collectOptionalHostEnv, collectProviderKeys, endpointHosts, hostSecretEnv, isSecretEnvName, redactSecrets, validateProviderKeys, type ProviderKeyProbe } from "../secrets.ts"; import { ArtifactsStore } from "../store/artifacts.ts"; import { activeAttempts, type ErrorKind, type NewAttempt, type ResultsStore, type RunConfig } from "../store/db.ts"; import { acquireRunLock } from "./run-lock.ts"; @@ -402,7 +402,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise [id, adapter.egressHosts ?? []])), + // Vendor defaults plus the host of any endpoint override (an LLM gateway) set on the host. + adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [...(adapter.egressHosts ?? []), ...endpointHosts(collectOptionalHostEnv([adapter]))]])), budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs }, mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])), }); @@ -806,12 +807,21 @@ async function runContainerAttemptV3( // every auth block and pointed authed servers at the proxy sidecar. env["QUICKSTUDY_MCP_SERVERS"] = JSON.stringify(rewrittenServers); } + // Endpoint overrides set on the host for this agent (gateway base URLs + // and the headers they need) — see ContainerAgentAdapter.optionalHostEnv. + const overrides = agentAdapter !== undefined ? collectOptionalHostEnv([agentAdapter]) : {}; + const publicOverrides = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name))); if (agentAdapter !== undefined) { // The agent's provider keys, validated at run preflight — injected // at create via the 0600 env-file channel, never argv. const providerKeys = collectProviderKeys([agentAdapter]); Object.assign(env, providerKeys); Object.assign(secretEnv, providerKeys); + // The overrides ride the same channel; only the secret-named ones + // feed redaction — a base URL in a transcript is diagnostic, a + // bearer header is not. + Object.assign(env, overrides); + for (const [name, value] of Object.entries(overrides)) if (isSecretEnvName(name)) secretEnv[name] = value; } if (egress) { const proxyEnv = egress.sidecar.proxyEnv(); @@ -874,6 +884,9 @@ async function runContainerAttemptV3( trialIndex: entry.trialIndex, ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}), webPolicy: environment?.webPolicy ?? "native-web-blocked", + // Never the keys or header bundles: adapters whose CLI wants a + // config-file entry (codex's model provider) render from here. + ...(Object.keys(publicOverrides).length > 0 ? { overrides: publicOverrides } : {}), }; // Treatment config files (e.g. the MCP treatment's vendor config). await inStage("environment", async () => { diff --git a/src/secrets.ts b/src/secrets.ts index cb3f3c0..eb1d35a 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -48,6 +48,12 @@ export interface ProviderKeyRequirement { name: string; /** Host env var names the adapter requires. */ requiredHostEnv: readonly string[]; + /** + * Host env var names forwarded when set and never required: endpoint + * overrides (a vendor base URL that routes the CLI through an LLM gateway + * or proxy) and the custom headers such a gateway needs. + */ + optionalHostEnv?: readonly string[]; } /** One preflight probe: a cheap authenticated GET a provider will 401 on a bad key. */ @@ -128,6 +134,81 @@ export function collectProviderKeys( return keys; } +/** + * Read every OPTIONAL host env var the adapters declare (see + * `ProviderKeyRequirement.optionalHostEnv`). Absent or empty variables are + * simply not forwarded — an empty `ANTHROPIC_BASE_URL=` from a CI template + * must not reach a CLI that would treat "" as a base URL. + */ +export function collectOptionalHostEnv( + requirements: readonly ProviderKeyRequirement[], + hostEnv: Record = process.env, +): Record { + const env: Record = {}; + for (const requirement of requirements) { + for (const envVar of requirement.optionalHostEnv ?? []) { + const value = hostEnv[envVar]; + if (value !== undefined && value !== "") env[envVar] = value; + } + } + return env; +} + +/** + * Hostnames of every forwarded value that is an absolute http(s) URL — the + * endpoints an override points at, which the egress allowlist must admit + * alongside the adapter's vendor defaults. Non-URL values (header lines) + * contribute nothing. + */ +export function endpointHosts(env: Record): string[] { + const hosts: string[] = []; + for (const value of Object.values(env)) { + let url: URL; + try { + url = new URL(value); + } catch { + continue; + } + if ((url.protocol === "https:" || url.protocol === "http:") && url.hostname !== "") hosts.push(url.hostname); + } + return hosts; +} + +/** + * `/` for a probe: the override when set (trailing slashes + * tolerated — gateways document both forms), else the vendor's own base. + */ +export function endpointUrl(override: string | undefined, vendorBase: string, path: string): string { + const base = override !== undefined && override !== "" ? override : vendorBase; + return `${base.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`; +} + +/** + * Parse `Name: Value` header lines (newline-separated — the format Claude + * Code reads from ANTHROPIC_CUSTOM_HEADERS). Blank lines and lines without a + * colon are skipped; the first colon splits name from value. + */ +export function parseHeaderLines(text: string | undefined): Record { + const headers: Record = {}; + for (const line of (text ?? "").split(/\r?\n/)) { + const colon = line.indexOf(":"); + if (colon <= 0) continue; + const name = line.slice(0, colon).trim(); + const value = line.slice(colon + 1).trim(); + if (name !== "") headers[name] = value; + } + return headers; +} + +/** + * Does an env var NAME look secret-bearing? Keys, tokens, passwords, + * credentials — and header bundles, which carry a gateway's bearer token. + * Endpoint URLs deliberately do not match: they are diagnostic, not secret. + */ +export function isSecretEnvName(name: string): boolean { + return /key|token|password|secret|credential|header/i.test(name); +} + /** Values shorter than this are never redacted (they would shred ordinary text). */ const MIN_REDACTABLE_LENGTH = 6; @@ -147,5 +228,5 @@ export function redactSecrets(text: string, env: Record): string /** Provider/judge secret env only; ordinary PATH and workspace values are evidence. */ export function hostSecretEnv(env: Record = process.env): Record { - return Object.fromEntries(Object.entries(env).filter(([name, value]) => /key|token|password|secret|credential/i.test(name) && typeof value === "string")) as Record; + return Object.fromEntries(Object.entries(env).filter(([name, value]) => isSecretEnvName(name) && typeof value === "string")) as Record; } diff --git a/tests/adapters.test.ts b/tests/adapters.test.ts index d9f5123..3690752 100644 --- a/tests/adapters.test.ts +++ b/tests/adapters.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { ClaudeCodeAdapter, CLAUDE_MCP_CONFIG_PATH } from "../src/adapters/claude.ts"; -import { CodexAdapter, CODEX_CONFIG_PATH } from "../src/adapters/codex.ts"; +import { CodexAdapter, CODEX_CONFIG_PATH, CODEX_GATEWAY_PROVIDER } from "../src/adapters/codex.ts"; import { parseClaudeStream, parseCodexStream, parseJsonlLines } from "../src/adapters/parse.ts"; import type { ContainerAttemptContext } from "../src/adapters/types.ts"; @@ -173,3 +173,55 @@ describe("codex adapter", () => { expect(adapter.parseStream(stream("codex-legacy.jsonl")).tokensIn).toBe(3000); }); }); + +describe("endpoint overrides (LLM gateway routing)", () => { + test("claude: declares its overrides; the probe follows ANTHROPIC_BASE_URL and carries the custom headers", () => { + const adapter = new ClaudeCodeAdapter("claude", { model: "m", reasoning: "r" }); + expect(adapter.optionalHostEnv).toEqual(["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"]); + expect(adapter.keyProbe({ ANTHROPIC_API_KEY: "k" }).url).toBe("https://api.anthropic.com/v1/models"); + + const probe = adapter.keyProbe({ + ANTHROPIC_API_KEY: "cf-token", + ANTHROPIC_BASE_URL: "https://gateway.example/v1/acct/gw/anthropic/", + ANTHROPIC_CUSTOM_HEADERS: 'cf-aig-authorization: Bearer cf-token\ncf-aig-metadata: {"app":"evals"}', + }); + expect(probe.url).toBe("https://gateway.example/v1/acct/gw/anthropic/v1/models"); + expect(probe.headers).toEqual({ + "cf-aig-authorization": "Bearer cf-token", + "cf-aig-metadata": '{"app":"evals"}', + "x-api-key": "cf-token", + "anthropic-version": "2023-06-01", + }); + }); + + test("codex: declares its override; the probe follows OPENAI_BASE_URL", () => { + const adapter = new CodexAdapter("codex", { model: "m", reasoning: "r" }); + expect(adapter.optionalHostEnv).toEqual(["OPENAI_BASE_URL"]); + expect(adapter.keyProbe({ OPENAI_API_KEY: "k" }).url).toBe("https://api.openai.com/v1/models"); + expect(adapter.keyProbe({ OPENAI_API_KEY: "k", OPENAI_BASE_URL: "https://gateway.example/v1/acct/gw/openai" }).url) + .toBe("https://gateway.example/v1/acct/gw/openai/models"); + }); + + test("codex: an OPENAI_BASE_URL override renders as a custom model provider, ahead of any MCP table", () => { + const adapter = new CodexAdapter("codex", { model: "m", reasoning: "r" }); + const base = "https://gateway.example/v1/acct/gw/openai"; + + const files = adapter.setupFiles({ ...ctxFor("none"), overrides: { OPENAI_BASE_URL: base } }); + expect(Object.keys(files)).toEqual([CODEX_CONFIG_PATH]); + const toml = files[CODEX_CONFIG_PATH] as string; + // codex 0.149 ignores the env var; the provider entry is what routes it. + expect(toml.startsWith(`model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`)).toBeTrue(); + expect(toml).toContain(`[model_providers.${CODEX_GATEWAY_PROVIDER}]`); + expect(toml).toContain(`base_url = ${JSON.stringify(base)}`); + expect(toml).toContain('env_key = "OPENAI_API_KEY"'); + expect(toml).toContain('wire_api = "responses"'); + + // TOML: the top-level key stays first; the provider table precedes the MCP tables. + const withMcp = adapter.setupFiles({ ...ctxFor("mcp"), overrides: { OPENAI_BASE_URL: base } })[CODEX_CONFIG_PATH] as string; + expect(withMcp.indexOf("model_provider = ")).toBe(0); + expect(withMcp.indexOf("[model_providers.")).toBeLessThan(withMcp.indexOf("[mcp_servers.acme]")); + + // Other overrides (or none) change nothing. + expect(adapter.setupFiles({ ...ctxFor("none"), overrides: { ANTHROPIC_BASE_URL: base } })).toEqual({}); + }); +}); diff --git a/tests/docs-smoke-inventory.ts b/tests/docs-smoke-inventory.ts index ade56f9..e6cf161 100644 --- a/tests/docs-smoke-inventory.ts +++ b/tests/docs-smoke-inventory.ts @@ -57,6 +57,12 @@ export const INVENTORY: InventoryEntry[] = [ classification: "needs-keys", note: "real-agent quickstart: image builds + paid API calls", }, + { + doc: "README.md", + match: "export ANTHROPIC_BASE_URL=https://gateway.ai.cloudflare.com/v1///anthropic", + classification: "placeholder", + note: "LLM gateway routing: env exports with // placeholders", + }, { doc: "README.md", match: "quickstudy validate", diff --git a/tests/secrets.test.ts b/tests/secrets.test.ts index 56880df..2a0835a 100644 --- a/tests/secrets.test.ts +++ b/tests/secrets.test.ts @@ -1,8 +1,14 @@ import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { + collectOptionalHostEnv, collectProviderKeys, + endpointHosts, + endpointUrl, + hostSecretEnv, InvalidProviderKeyError, + isSecretEnvName, MissingProviderKeyError, + parseHeaderLines, redactSecrets, validateProviderKeys, type ProviderKeyProbe, @@ -142,3 +148,59 @@ describe("validateProviderKeys", () => { afterEach(() => { delete process.env["QS_TEST_NEVER_SET_KEY"]; }); + +describe("endpoint overrides", () => { + const CLAUDE = { + name: "claude", + requiredHostEnv: ["ANTHROPIC_API_KEY"], + optionalHostEnv: ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"], + } as const; + + test("collectOptionalHostEnv forwards set overrides and drops absent or empty ones", () => { + expect(collectOptionalHostEnv([CLAUDE], {})).toEqual({}); + expect(collectOptionalHostEnv([CLAUDE], { ANTHROPIC_BASE_URL: "", ANTHROPIC_CUSTOM_HEADERS: "a: b" })).toEqual({ + ANTHROPIC_CUSTOM_HEADERS: "a: b", + }); + // An adapter that declares no overrides forwards nothing, whatever the host has. + expect(collectOptionalHostEnv([{ name: "x", requiredHostEnv: [] }], { ANTHROPIC_BASE_URL: "https://g.example" })).toEqual({}); + }); + + test("endpointHosts takes the host of every absolute http(s) URL and ignores everything else", () => { + expect( + endpointHosts({ + ANTHROPIC_BASE_URL: "https://gateway.ai.cloudflare.com/v1/acct/gw/anthropic", + ANTHROPIC_CUSTOM_HEADERS: "cf-aig-authorization: Bearer tok", + OPENAI_BASE_URL: "http://localhost:8080/openai", + JUNK: "not a url", + }), + ).toEqual(["gateway.ai.cloudflare.com", "localhost"]); + expect(endpointHosts({ OTHER: "ftp://files.example/x" })).toEqual([]); + }); + + test("endpointUrl prefers a set override, tolerating trailing and leading slashes", () => { + expect(endpointUrl(undefined, "https://api.anthropic.com", "v1/models")).toBe("https://api.anthropic.com/v1/models"); + expect(endpointUrl("", "https://api.anthropic.com", "v1/models")).toBe("https://api.anthropic.com/v1/models"); + expect(endpointUrl("https://g.example/anthropic/", "https://api.anthropic.com", "/v1/models")).toBe( + "https://g.example/anthropic/v1/models", + ); + }); + + test("parseHeaderLines reads Name: Value lines, splitting on the first colon", () => { + expect(parseHeaderLines(undefined)).toEqual({}); + expect(parseHeaderLines('cf-aig-authorization: Bearer tok\r\n\nno-colon\n: empty-name\ncf-aig-metadata: {"a":1}')).toEqual({ + "cf-aig-authorization": "Bearer tok", + "cf-aig-metadata": '{"a":1}', + }); + }); + + test("header bundles count as secrets (they carry a bearer); base URLs do not", () => { + expect(isSecretEnvName("ANTHROPIC_CUSTOM_HEADERS")).toBeTrue(); + expect(isSecretEnvName("ANTHROPIC_BASE_URL")).toBeFalse(); + const env = hostSecretEnv({ + ANTHROPIC_CUSTOM_HEADERS: "cf-aig-authorization: Bearer tok", + ANTHROPIC_BASE_URL: "https://g.example", + PATH: "/bin", + }); + expect(Object.keys(env)).toEqual(["ANTHROPIC_CUSTOM_HEADERS"]); + }); +}); From 5326614e71e36d5545dac692bae6d79e2514ef54 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 16 Oct 2026 14:15:22 -0400 Subject: [PATCH 2/2] fix(gateway): deliver multi-line headers, redact bundled tokens, record routes Review findings on the endpoint-override contract: - A header bundle with several lines could not reach the attempt container: the env-file channel is line-based and refused newlines. Multi-line values now ride the docker CLI's own environment, named on argv by `-e KEY` alone, so argv still never carries a value. Covered by a daemon-free split test and a Docker round-trip. - The credentials inside a header bundle (each auth header's value and the bare bearer token) are redaction targets of their own, so a transcript that prints only the token is scrubbed; hostSecretEnv expands bundles the same way. - An override's host joins the egress allowlist only under the proxy, where the list is live and an IP literal is refused loudly by design; without the proxy an IP-based gateway no longer stops an otherwise valid run. - The non-secret override URLs are recorded in full in the pair environment, so a changed gateway route changes both manifest and attempt identity and a resume cannot mix routes. Vendor-direct pairs hash exactly as before. - getting-started states that a gateway still needs the key variable. --- docs/getting-started.md | 2 +- src/execution-policy.ts | 13 +++++++ src/isolation/docker.ts | 79 ++++++++++++++++++++++++++------------- src/runner/execute.ts | 28 +++++++++++--- src/secrets.ts | 36 +++++++++++++++++- tests/docker.test.ts | 41 ++++++++++++++++++++ tests/identity-v3.test.ts | 25 +++++++++++++ tests/secrets.test.ts | 37 +++++++++++++++++- 8 files changed, 226 insertions(+), 35 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 5f5a0cb..8e077a6 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -17,7 +17,7 @@ keys**. If a term is unfamiliar, [Concepts](./concepts.md) defines it. | ------------------------- | -------------------------------------------------------------------- | | **[Bun](https://bun.sh)** | only to build the binary or run from source | | **Docker** | only to run _real_ agents in isolated containers | -| **API keys** | only for real LLM agents (`ANTHROPIC_API_KEY`, …; a gateway needs only `ANTHROPIC_BASE_URL` — see the README) | +| **API keys** | only for real LLM agents (`ANTHROPIC_API_KEY`, …); a gateway additionally needs its base URL, with the gateway token as the key variable — see the README | The keyless act needs none of Docker, keys, or network — the `echo` agent runs the full run → score → persist pipeline on the host. diff --git a/src/execution-policy.ts b/src/execution-policy.ts index cf92d31..d602439 100644 --- a/src/execution-policy.ts +++ b/src/execution-policy.ts @@ -27,6 +27,13 @@ export interface PairEnvironment { /** Runtime text prepended to the eval prompt for this pair (null = prompt verbatim). */ promptPreamble: string | null; budgets: LifecycleBudgets; + /** + * The agent's endpoint overrides in force (non-secret base URLs, e.g. + * OPENAI_BASE_URL → an LLM gateway route), recorded in full so a changed + * route changes identity and a resume cannot mix routes. Absent when the + * agent runs vendor-direct — such pairs hash exactly as before. + */ + endpoints?: Record; } /** The prompt an agent receives: the runtime's preamble (if any), a blank line, the eval's verbatim body. */ @@ -43,6 +50,8 @@ export function resolvePairEnvironments(options: { images?: Record; egressProxy?: boolean; adapterHosts?: Record; + /** Per experiment: the agent's non-secret endpoint overrides (see PairEnvironment.endpoints). */ + adapterEndpoints?: Record>; budgets?: Partial; mcpHashes?: Record; imageTags?: Record; @@ -77,6 +86,10 @@ export function resolvePairEnvironments(options: { return text === "" ? null : text; })(), budgets: { ...budgets }, + ...(() => { + const endpoints = options.adapterEndpoints?.[experiment.id]; + return endpoints !== undefined && Object.keys(endpoints).length > 0 ? { endpoints: { ...endpoints } } : {}; + })(), }; } return pairs; diff --git a/src/isolation/docker.ts b/src/isolation/docker.ts index f92c8ff..7f3f661 100644 --- a/src/isolation/docker.ts +++ b/src/isolation/docker.ts @@ -59,15 +59,20 @@ export interface ExecResult { timedOut: boolean; } -/** Run `docker `, capturing output. Never throws for non-zero exits. */ +/** + * Run `docker `, capturing output. Never throws for non-zero exits. + * `env` is added to the docker CLI's own environment — the channel for + * values named on argv by `-e KEY` alone. + */ async function runDocker( args: string[], - opts: { timeoutMs?: number; stdin?: string } = {}, + opts: { timeoutMs?: number; stdin?: string; env?: Record } = {}, ): Promise { const proc = Bun.spawn(["docker", ...args], { stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin), stdout: "pipe", stderr: "pipe", + ...(opts.env !== undefined ? { env: { ...process.env, ...opts.env } } : {}), }); let timedOut = false; let timer: ReturnType | undefined; @@ -87,7 +92,10 @@ async function runDocker( } /** Run `docker ` and throw a DockerError on non-zero exit. */ -async function mustDocker(args: string[], opts: { timeoutMs?: number; stdin?: string } = {}): Promise { +async function mustDocker( + args: string[], + opts: { timeoutMs?: number; stdin?: string; env?: Record } = {}, +): Promise { const result = await runDocker(args, opts); if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr); return result; @@ -198,36 +206,55 @@ export interface AttemptContainer { * fixture into /workspace. On any failure the partial container is removed * before the error propagates — a failed create never leaks a container. */ +/** + * Route an attempt's env between its two argv-free channels. Single-line + * values become env-file lines; a value that itself contains newlines (a + * `Name: Value` header bundle) cannot be expressed in a line-based env-file, + * so it rides the docker CLI's own environment instead, named on argv by + * `-e KEY` alone — the CLI forwards the value intact and argv never shows it. + */ +export function splitAttemptEnv(env: Record): { envFileLines: string[]; inherited: Record } { + const envFileLines: string[] = []; + const inherited: Record = {}; + for (const [key, value] of Object.entries(env)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`); + if (value.includes("\n")) inherited[key] = value; + else envFileLines.push(`${key}=${value}`); + } + return { envFileLines, inherited }; +} + export async function createAttemptContainer(opts: CreateAttemptContainerOptions): Promise { // Secrets travel via a transient env-file, not argv (argv is visible in - // host process listings while `docker run` executes). + // host process listings while `docker run` executes); multi-line values + // take the CLI-environment channel (see splitAttemptEnv). const envDir = await mkdtemp(join(tmpdir(), "quickstudy-env-")); const envFile = join(envDir, "attempt.env"); - const lines = Object.entries(opts.env).map(([key, value]) => { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`); - if (value.includes("\n")) throw new Error(`environment variable "${key}" contains a newline`); - return `${key}=${value}`; - }); - await writeFile(envFile, lines.join("\n") + (lines.length > 0 ? "\n" : ""), { mode: 0o600 }); + const { envFileLines, inherited } = splitAttemptEnv(opts.env); + await writeFile(envFile, envFileLines.join("\n") + (envFileLines.length > 0 ? "\n" : ""), { mode: 0o600 }); let id: string; try { - const created = await mustDocker([ - "run", - "-d", - "--label", - `${ATTEMPT_LABEL}=${opts.attemptId}`, - "--env-file", - envFile, - "-v", - `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`, - "-w", - CONTAINER_WORKSPACE, - ...(opts.network !== undefined ? ["--network", opts.network] : []), - opts.image, - "sleep", - "infinity", - ]); + const created = await mustDocker( + [ + "run", + "-d", + "--label", + `${ATTEMPT_LABEL}=${opts.attemptId}`, + "--env-file", + envFile, + ...Object.keys(inherited).flatMap((key) => ["-e", key]), + "-v", + `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`, + "-w", + CONTAINER_WORKSPACE, + ...(opts.network !== undefined ? ["--network", opts.network] : []), + opts.image, + "sleep", + "infinity", + ], + { env: inherited }, + ); id = created.stdout.trim(); } catch (err) { // `docker run` can fail AFTER creating the container (start failure) — diff --git a/src/runner/execute.ts b/src/runner/execute.ts index e3ec482..c6078d8 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -70,7 +70,7 @@ import { attemptIdentityHash, buildRunManifest, type RunManifest, type RuntimeIm import { isMovingModelAlias } from "../model-identity.ts"; import type { AttemptPlan } from "../plan.ts"; import type { ProvisionedEnvironment, RuntimeProvisionContext } from "../runtime/types.ts"; -import { collectOptionalHostEnv, collectProviderKeys, endpointHosts, hostSecretEnv, isSecretEnvName, redactSecrets, validateProviderKeys, type ProviderKeyProbe } from "../secrets.ts"; +import { collectOptionalHostEnv, collectProviderKeys, endpointHosts, headerBundleSecrets, hostSecretEnv, isSecretEnvName, redactSecrets, validateProviderKeys, type ProviderKeyProbe } from "../secrets.ts"; import { ArtifactsStore } from "../store/artifacts.ts"; import { activeAttempts, type ErrorKind, type NewAttempt, type ResultsStore, type RunConfig } from "../store/db.ts"; import { acquireRunLock } from "./run-lock.ts"; @@ -458,9 +458,22 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise [id, collectOptionalHostEnv([adapter])] as const)); + const adapterEndpoints = Object.fromEntries([...overridesByExperiment].flatMap(([id, overrides]) => { + const endpoints = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name))); + return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : []; + })); const pairs = resolvePairEnvironments({ ...options, images, imageTags: imageTagsByPair, - // Vendor defaults plus the host of any endpoint override (an LLM gateway) set on the host. - adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [...(adapter.egressHosts ?? []), ...endpointHosts(collectOptionalHostEnv([adapter]))]])), + adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [ + ...(adapter.egressHosts ?? []), + ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []), + ]])), + adapterEndpoints, budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs }, mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])), }); @@ -819,9 +832,14 @@ async function runContainerAttemptV3( Object.assign(secretEnv, providerKeys); // The overrides ride the same channel; only the secret-named ones // feed redaction — a base URL in a transcript is diagnostic, a - // bearer header is not. + // bearer header is not. A header bundle's credentials are targets of + // their own: a transcript that prints just the token is still scrubbed. Object.assign(env, overrides); - for (const [name, value] of Object.entries(overrides)) if (isSecretEnvName(name)) secretEnv[name] = value; + for (const [name, value] of Object.entries(overrides)) { + if (!isSecretEnvName(name)) continue; + secretEnv[name] = value; + Object.assign(secretEnv, headerBundleSecrets(name, value)); + } } if (egress) { const proxyEnv = egress.sidecar.proxyEnv(); diff --git a/src/secrets.ts b/src/secrets.ts index eb1d35a..78a1bdc 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -209,6 +209,36 @@ export function isSecretEnvName(name: string): boolean { return /key|token|password|secret|credential|header/i.test(name); } +/** Header names whose values are credentials (`cf-aig-authorization`, `x-api-key`, …). */ +const SECRET_HEADER = /auth|key|token|secret|credential|password/i; + +/** + * The redaction targets inside one header bundle (an env var holding `Name: + * Value` lines): each credential header's full value, plus the bare + * credential behind a `Bearer`/`Basic`/`Token` scheme — a transcript that + * prints only the token, not the whole bundle, must still be scrubbed. Keys + * name the bundle and header so the redaction marker says what was removed. + */ +export function headerBundleSecrets(bundleName: string, bundle: string): Record { + const derived: Record = {}; + for (const [header, value] of Object.entries(parseHeaderLines(bundle))) { + if (!SECRET_HEADER.test(header)) continue; + derived[`${bundleName}[${header}]`] = value; + const scheme = /^(?:Bearer|Basic|Token)\s+(\S+)$/i.exec(value); + if (scheme?.[1] !== undefined) derived[`${bundleName}[${header}]:credential`] = scheme[1]; + } + return derived; +} + +/** A secret env map plus the credentials carried inside any header bundle it holds. */ +export function expandSecretEnv(env: Record): Record { + const expanded = { ...env }; + for (const [name, value] of Object.entries(env)) { + if (/header/i.test(name)) Object.assign(expanded, headerBundleSecrets(name, value)); + } + return expanded; +} + /** Values shorter than this are never redacted (they would shred ordinary text). */ const MIN_REDACTABLE_LENGTH = 6; @@ -226,7 +256,9 @@ export function redactSecrets(text: string, env: Record): string return redacted; } -/** Provider/judge secret env only; ordinary PATH and workspace values are evidence. */ +/** Provider/judge secret env only (header-bundle credentials included); ordinary PATH and workspace values are evidence. */ export function hostSecretEnv(env: Record = process.env): Record { - return Object.fromEntries(Object.entries(env).filter(([name, value]) => isSecretEnvName(name) && typeof value === "string")) as Record; + return expandSecretEnv( + Object.fromEntries(Object.entries(env).filter(([name, value]) => isSecretEnvName(name) && typeof value === "string")) as Record, + ); } diff --git a/tests/docker.test.ts b/tests/docker.test.ts index addb9ed..bf39255 100644 --- a/tests/docker.test.ts +++ b/tests/docker.test.ts @@ -25,6 +25,7 @@ import { inspectAttemptLabel, inspectMounts, listAttemptContainers, + splitAttemptEnv, startMcpSidecar, } from "../src/isolation/docker.ts"; import { MCP_PROXY_IMAGE } from "../src/isolation/images.ts"; @@ -627,3 +628,43 @@ describeMcpSidecar("mcp auth-proxy sidecar", () => { LIFECYCLE_TIMEOUT, ); }); + +describe("attempt env channels (no daemon required)", () => { + test("single-line values become env-file lines; multi-line values ride the CLI environment by name", () => { + const { envFileLines, inherited } = splitAttemptEnv({ + PROVIDER_KEY: "sk-plain", + BUNDLE: "cf-aig-authorization: Bearer tok\ncf-aig-metadata: {\"app\":\"x\"}", + }); + expect(envFileLines).toEqual(["PROVIDER_KEY=sk-plain"]); + expect(inherited).toEqual({ BUNDLE: "cf-aig-authorization: Bearer tok\ncf-aig-metadata: {\"app\":\"x\"}" }); + }); + + test("an invalid variable name is refused before anything is written", () => { + expect(() => splitAttemptEnv({ "BAD-NAME": "x" })).toThrow(/invalid environment variable name/); + }); +}); + +describeDocker("multi-line env delivery", () => { + test( + "a newline-separated header bundle reaches the container intact", + async () => { + const fixtureDir = makeFixture(); + const resultsDir = mkdtempSync(join(tmpdir(), "quickstudy-results-")); + const bundle = 'cf-aig-authorization: Bearer multi-line-token\ncf-aig-metadata: {"app":"test"}'; + const container = await createAttemptContainer({ + image: TEST_IMAGE, + env: { QS_PLAIN: "plain-value", QS_BUNDLE: bundle }, + resultsDir, + attemptId: `test-${Date.now()}-multiline-env`, + fixtureDir, + }); + try { + expect((await container.execInWorkspace(["printenv", "QS_BUNDLE"])).stdout).toBe(`${bundle}\n`); + expect((await container.execInWorkspace(["printenv", "QS_PLAIN"])).stdout).toBe("plain-value\n"); + } finally { + await container.teardown(); + } + }, + LIFECYCLE_TIMEOUT, + ); +}); diff --git a/tests/identity-v3.test.ts b/tests/identity-v3.test.ts index 2a3e36f..c44e4c9 100644 --- a/tests/identity-v3.test.ts +++ b/tests/identity-v3.test.ts @@ -265,3 +265,28 @@ test("resolver images, enforced network policy and budgets change affected attem const historical = { ...original, manifestVersion: "v3" as const }; expect(attemptIdentityHash(plan, historical)).not.toBe(attemptIdentityHash(plan, original)); }); + +test("endpoint overrides are part of the pair environment: a changed gateway route changes identity; none leaves it untouched", async () => { + const { resolvePairEnvironments } = await import("../src/execution-policy.ts"); + const fixture = makeFixture(); + const evals = discoverEvals(fixture.evalsRoot); + const experiments = loadExperiments(fixture.experimentsRoot); + experiments[0]!.experiment = { ...experiments[0]!.experiment, runtime: { kind: "container", image: "test:image" } }; + const images = { "test:image": { tag: "test:image", id: "sha256:a", repoDigests: [] } }; + const make = (adapterEndpoints?: Record>) => + buildRunManifest({ evals, experiments, images, harness: HARNESS, pairs: resolvePairEnvironments({ evals, experiments, images, adapterEndpoints }) }); + const plan = { evalId: "hello", experimentId: "echo-x", trialIndex: 0 }; + + const direct = make(); + expect(Object.values(direct.pairs!)[0]).not.toHaveProperty("endpoints"); + expect(make({ "echo-x": {} }).hash).toBe(direct.hash); + + const teamA = make({ "echo-x": { OPENAI_BASE_URL: "https://gateway.example/team-a/openai" } }); + const teamB = make({ "echo-x": { OPENAI_BASE_URL: "https://gateway.example/team-b/openai" } }); + expect(Object.values(teamA.pairs!)[0]!.endpoints).toEqual({ OPENAI_BASE_URL: "https://gateway.example/team-a/openai" }); + // Same host, different route: the manifest AND the attempt identity differ, so a resume cannot mix them. + expect(teamA.hash).not.toBe(direct.hash); + expect(teamA.hash).not.toBe(teamB.hash); + expect(attemptIdentityHash(plan, teamA)).not.toBe(attemptIdentityHash(plan, teamB)); + expect(attemptIdentityHash(plan, teamA)).not.toBe(attemptIdentityHash(plan, direct)); +}); diff --git a/tests/secrets.test.ts b/tests/secrets.test.ts index 2a0835a..acdd92c 100644 --- a/tests/secrets.test.ts +++ b/tests/secrets.test.ts @@ -4,6 +4,8 @@ import { collectProviderKeys, endpointHosts, endpointUrl, + expandSecretEnv, + headerBundleSecrets, hostSecretEnv, InvalidProviderKeyError, isSecretEnvName, @@ -201,6 +203,39 @@ describe("endpoint overrides", () => { ANTHROPIC_BASE_URL: "https://g.example", PATH: "/bin", }); - expect(Object.keys(env)).toEqual(["ANTHROPIC_CUSTOM_HEADERS"]); + expect(Object.keys(env)).toContain("ANTHROPIC_CUSTOM_HEADERS"); + expect(Object.keys(env)).not.toContain("ANTHROPIC_BASE_URL"); + expect(Object.keys(env)).not.toContain("PATH"); + }); +}); + +describe("header bundle credentials", () => { + const BUNDLE = 'cf-aig-authorization: Bearer cf-gateway-token-123\ncf-aig-metadata: {"app":"evals"}'; + + test("headerBundleSecrets targets credential headers and their bare bearer tokens, not metadata", () => { + expect(headerBundleSecrets("ANTHROPIC_CUSTOM_HEADERS", BUNDLE)).toEqual({ + "ANTHROPIC_CUSTOM_HEADERS[cf-aig-authorization]": "Bearer cf-gateway-token-123", + "ANTHROPIC_CUSTOM_HEADERS[cf-aig-authorization]:credential": "cf-gateway-token-123", + }); + expect(headerBundleSecrets("H", "x-api-key: sk-plain-key-value")).toEqual({ "H[x-api-key]": "sk-plain-key-value" }); + }); + + test("a transcript that prints only the gateway token is still scrubbed", () => { + const secretEnv = expandSecretEnv({ ANTHROPIC_API_KEY: "sk-ant-provider-key-9", ANTHROPIC_CUSTOM_HEADERS: BUNDLE }); + const text = "curl -H 'Authorization: Bearer cf-gateway-token-123' and later just cf-gateway-token-123"; + const redacted = redactSecrets(text, secretEnv); + expect(redacted).not.toContain("cf-gateway-token-123"); + expect(redacted).toContain("[REDACTED:ANTHROPIC_CUSTOM_HEADERS[cf-aig-authorization]"); + // The whole bundle still redacts as one when echoed verbatim. + expect(redactSecrets(`env: ${BUNDLE}`, secretEnv)).toBe("env: [REDACTED:ANTHROPIC_CUSTOM_HEADERS]"); + }); + + test("hostSecretEnv expands bundles it finds on the host", () => { + const env = hostSecretEnv({ ANTHROPIC_CUSTOM_HEADERS: BUNDLE, ANTHROPIC_BASE_URL: "https://g.example", PATH: "/bin" }); + expect(Object.keys(env).sort()).toEqual([ + "ANTHROPIC_CUSTOM_HEADERS", + "ANTHROPIC_CUSTOM_HEADERS[cf-aig-authorization]", + "ANTHROPIC_CUSTOM_HEADERS[cf-aig-authorization]:credential", + ]); }); });