diff --git a/docs/concepts.md b/docs/concepts.md index 72a0537..01c042e 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -81,6 +81,9 @@ An experiment is one TypeScript module under the experiments root (default surface nobody mentions is rarely discovered, and an undiscovered surface measures nothing. The resolved preamble is recorded per pair in the run manifest; the eval body itself stays byte-identical across experiments. + Runtimes also offer optional non-secret + [attempt-local workspace files](workspace-files.md). These files are offered + before the normal adapter starts; verified placement is not proof of model use. - the optional **treatment** and **comparisonGroup** (below). Because agent pins, runtime configuration, and the module source are all part @@ -144,7 +147,7 @@ about the agent, not the harness. An **error** carries an `error_kind` naming the stage that failed: `environment` (provisioning, workspace prep, network, container create, -setup, export), `agent` (the agent command exited non-zero or threw), +setup, workspace-file staging/absence checks, export), `agent` (the agent command exited non-zero or threw), `scorer` (the scorer threw, returned an invalid result, or ran past its budget), or `harness` (anything else — a quickstudy bug). Infrastructure failures read apart from agent failures without opening a transcript. diff --git a/docs/extensions.md b/docs/extensions.md index cf3998a..873638a 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -53,6 +53,17 @@ agent files remain; outside/absolute symlinks are removed. Reconstruct dependenc and build outputs from recorded manifests, source, transcript and diff. Git history is excluded; the agent diff uses the recorded baseline commit before export. +## Attempt-local workspace files + +`runtime.workspaceFiles` is a bounded, versioned declaration for non-secret +files and absence assertions in a fresh container workspace. It preserves the +normal adapter, login, configuration, telemetry and lifecycle; it is **not** a +`runtime.command` hook. Both comparison arms must explicitly exempt +`"workspaceFiles"` through `treatmentPolicyFields` if availability is their +treatment. Host/scripted overlays are rejected. See the [full API, ordering, +limits, identity and evidence contract](workspace-files.md) and +[local consumer handoff](workspace-files-handoff.md). + ## Identity v4 v4 fingerprints the defining module and recursively resolved literal local @@ -78,7 +89,7 @@ Every pair records the exact resolved image ID, egress policy, web policy, PATH treatment, MCP configuration digest and lifecycle budgets. Execution uses those resolved values, including the image ID instead of its mutable tag. Experiment comparison checks per-eval images and budgets. A runtime may explicitly declare -`treatmentPolicyFields` for egress/webPolicy/pathTreatment/mcp/promptPreamble; all group members +`treatmentPolicyFields` for egress/webPolicy/pathTreatment/mcp/promptPreamble/workspaceFiles; all group members must declare the same exemptions. Images and budgets cannot be exempted. v3 manifests and historical reports remain readable with their original semantics. diff --git a/docs/workspace-files-handoff.md b/docs/workspace-files-handoff.md new file mode 100644 index 0000000..ae092e6 --- /dev/null +++ b/docs/workspace-files-handoff.md @@ -0,0 +1,194 @@ +# Workspace-file overlay: consumer handoff + +## Revisions and boundary + +- Inspected/source base: `30928456a7154ab64be9b7a4c4a914e2df7bf8d8` (0.5.0). +- Final harness **implementation** commit: + `bc6a803469c915f58bd494189006a65cc224e5c4`. +- Offline MCP-test isolation correction: `c7483a1` (no production change). +- The documentation commit containing this handoff follows those commits. Use + the final branch tip Riker supplies, and record `git rev-parse HEAD` for the + exact local validation revision. No release/version bump is implied. + +This implements the prerequisite described in the existing consumer blocker +report, not that consumer's treatment. No upstream skill was fetched, no +consumer recipes were changed, and no publication/live run was performed. +The existing approved source pin, full reference-tree/license requirements, +separate experiment root/group, and discovery-compatibility questions from that +report still apply. Availability, discovery/read evidence, and task success +remain separate claims. + +## Exact API for the existing consumer + +The installed package in that job is named `@workos/quickstudy`; import from +that public entry point, **not** runner, manifest or adapter internals: + +```ts +import { + defineExperiment, + defineWorkspaceFiles, + WORKSPACE_FILES_LIMITS, + type Runtime, + type WorkspaceFile, + type WorkspaceFiles, + type WorkspaceFilesIdentity, + type WorkspaceFilesEvidence, +} from "@workos/quickstudy"; +``` + +The field is `runtime.workspaceFiles: WorkspaceFiles`. Its complete shape is: + +```ts +{ + version: 1, + files: [{ + path: ".agents/skills/harbor/SKILL.md", // workspace-relative, generic example + encoding: "base64", // or explicit "utf8" + content: locallyReadBytes.toString("base64"), + source: { id: "harbor-guide", version: "commit-abc123" }, + }], + absent: [], +} +``` + +Replace that fictional payload/source with the audited, locally vendored, +approved pin; list every required reference and attribution file. Read exact +bytes (not line-normalized text), then call `defineWorkspaceFiles` to snapshot +and validate. No manually declared SHA field is needed. There is no arbitrary +callback, pre-command, installation script, host path, download, or permission +option. The [API reference](workspace-files.md) defines all limits and errors. + +For the control, keep the normal adapter and environment identical: + +```ts +const noFiles = defineWorkspaceFiles({ + version: 1, + files: [], + absent: [".agents/skills/harbor"], // replace with the actual reserved tree +}); +``` + +Both arms use exactly `treatmentPolicyFields: ["workspaceFiles"]`, a new +comparison group and distinct experiment ids/treatments. Do not inherit the +existing surface study's web/MCP/PATH exemptions. Do not set `runtime.command`. +Use container adapters, not host or scripted containers. This does not change +network authorization, PATH, native-web policy, image, model, CLI pins or budgets. + +## What to inspect + +- **Intended**: the stored run manifest's + `pairs[JSON.stringify([evalId, experimentId])].workspaceFiles`. It records + sorted destination paths, decoded byte sizes/SHA-256, declared source/version, + absence expectations and staging version. Full experiment/attempt identities + include this data. No payload or absolute host source path enters the manifest. +- **Actually checked**: `results///workspace-files.json`, typed as + `WorkspaceFilesEvidence`: `{version: 1, verified, verifiedAt}`. `verified` + must equal that pair's intended inventory. Inspect both files and control + absence evidence. A missing artifact is not a successful absence check. +- **Failure**: attempt `status: error`, `error_kind: environment`, null agent + timestamps, and `error.txt`; cleanup failures remain additional notes. +- **Model use**: separate transcript/tool evidence. A staged skill can be + ignored or not discovered. Neither hashes nor pass/fail prove discovery. + +## Offline cross-repo validation before a release + +Use a **new disposable consumer worktree**, not the active checkout. The +following is a suggested operator workflow, not commands run by this harness +job. Set `CONSUMER_REPO` and `CONSUMER_REVISION` to the intended consumer source +revision; do not borrow another job's unpublished working changes. + +```sh +# Start in the final harness worktree supplied by Riker. +export HARNESS_ROOT="$PWD" +git rev-parse HEAD +export CHECK_ROOT="$(mktemp -d)" +git -C "$CONSUMER_REPO" worktree add --detach "$CHECK_ROOT/consumer" "$CONSUMER_REVISION" +cd "$CHECK_ROOT/consumer" + +# Disposable override only; never commit this absolute path or its lockfile. +bun -e ' + const fs = require("node:fs"); + const p = JSON.parse(fs.readFileSync("package.json", "utf8")); + const name = "@workos/quickstudy"; + const section = p.dependencies?.[name] ? "dependencies" : "devDependencies"; + p[section] ??= {}; + p[section][name] = "file:" + process.env.HARNESS_ROOT; + fs.writeFileSync("package.json", JSON.stringify(p, null, 2) + "\n"); +' +bun install --ignore-scripts + +export QUICKSTUDY_LIVE_WORKOS=0 QUICKSTUDY_LIVE_TESTS=0 QUICKSTUDY_REQUIRE_DOCKER=0 +export DOCKER_HOST=unix:///nonexistent/quickstudy-offline.sock +export QUICKSTUDY_WORKOS_PROVISION=offline +bun -e 'import {defineWorkspaceFiles} from "@workos/quickstudy"; console.log(defineWorkspaceFiles({version:1,files:[],absent:["reserved-guide"]}));' +bun run typecheck +bun test + +# AFTER authoring the opt-in consumer recipes/payload in this isolated worktree: +bun node_modules/@workos/quickstudy/src/cli.ts validate \ + --evals-root evals --experiments-root experiments-skills +``` + +`experiments-skills` does not exist merely because the harness feature exists. +Author and audit that root first. Keep it out of default sweeps. `validate` +checks real generated identities/comparison groups without running an agent; +require the files-only group to be allowed. Add offline consumer tests for +its complete canonical inventory, source pin, license, references and chosen +reserved-tree control. Do not import harness internals to stage files or replace +the adapter. The harness's offline runner suite already exercises the generic +placement/lifecycle contract with generated manifests. + +Review consumer changes selectively. Never commit the temporary `package.json` +`file:` override or regenerated lockfile. Remove those disposable changes before +preparing a consumer diff, or copy only explicitly reviewed payload/recipe/tests +to a separate review branch. After preserving wanted work, the operator can +remove the disposable worktree. Nothing here requires modifying the active +consumer checkout or its dependency lockfile. + +A `file:` dependency check proves local source integration, not a published +package. Keep the consumer PR draft until a **real** harness release exists, +then update to that actual version and rerun frozen-lockfile installation and +consumer checks. Do not invent a registry version, commit an absolute dependency +path, or add release machinery as part of this prerequisite. + +## Checks and remaining gates + +Executed in the assigned harness worktree, with live flags off and Docker pointed +at a nonexistent socket: + +- `bun install --frozen-lockfile`: installed only in this worktree; no lockfile + changes or new dependencies. +- Targeted existing suites plus the initial overlay suite: + **113 pass, 9 skip, 0 fail**. +- Expanded `bun test workspace-files`: **50 pass, 0 fail**; includes the actual + 30-second staging deadline, read-back corruption, unsafe paths/types, immutable + payloads, control contamination, generated identities/resume/comparison, and + the production runner with offline container fakes. +- Initial full check: **469 pass, 22 skip, 1 fail**. The pre-existing MCP + missing-credentials test reached Docker image discovery before its assertion. + With Riker's approval, it now injects a locally generated manifest using the + existing test seam; production credential/preflight order is unchanged. +- `bun test mcp-proxy` after that test-only correction: **14 pass, 0 fail**. +- Final required command: typecheck and lint passed; **470 pass, 22 skip, + 0 fail**, 2,349 assertions across 44 files: + +```sh +env QUICKSTUDY_LIVE_WORKOS=0 QUICKSTUDY_LIVE_TESTS=0 \ + QUICKSTUDY_REQUIRE_DOCKER=0 \ + DOCKER_HOST=unix:///nonexistent/quickstudy-offline.sock \ + sh -c 'bun run typecheck && bun run lint && bun test' +``` + +All 22 skips were Docker-gated integrations. Docker access was deliberately +disabled; installed-image availability was not inspected. The new offline fakes run the real staging program on +local temporary files plus the production attempt runner and ordinary Codex +composition/parser; they do **not** prove Linux Docker transport, real CLI +execution, upstream skill discovery, model use or hosted acceptance. The Linux +no-follow descriptor path still needs the authorized real-container gate (the +local filesystem fake uses the host-platform path). + +Remaining: review and publish the harness through Riker, a real package release, +isolated consumer validation with its audited pinned payload, and separately +authorized existing-image/container then paid-agent smoke checks. No image build, +paid agent/judge, live provider probe, real-secret read, release, deployment, +push or PR creation was performed by this job. diff --git a/docs/workspace-files.md b/docs/workspace-files.md new file mode 100644 index 0000000..6caf158 --- /dev/null +++ b/docs/workspace-files.md @@ -0,0 +1,213 @@ +# Attempt-local workspace files + +`Runtime.workspaceFiles` offers non-secret skill/document files without replacing +Claude, Codex, or another built-in container adapter. **Do not use +`runtime.command` as a staging hook:** a nonempty command bypasses adapter +resolution, authentication, configuration, and vendor stream parsing. + +The public API exports `WorkspaceFile`, `WorkspaceFiles`, +`WorkspaceFilesIdentity`, `WorkspaceFilesEvidence`, `defineWorkspaceFiles`, and +`WORKSPACE_FILES_LIMITS`. `defineWorkspaceFiles` is an optional validating, +copying, deeply freezing helper. `defineExperiment` validates and freezes its +runtime declaration too. Imports below use the documentation's `quickstudy` +package alias; use your installed dependency name. + +## Files versus no files + +```ts +import { defineExperiment, defineWorkspaceFiles } from "quickstudy"; +import { readFileSync } from "node:fs"; + +// Read an audited, locally vendored, pinned file at experiment load time. +// No network fetch, installation command, host-directory mount, or callback. +const offered = defineWorkspaceFiles({ + version: 1, + files: [{ + path: ".agents/skills/harbor/SKILL.md", + encoding: "base64", + content: readFileSync(new URL("./payload/SKILL.md", import.meta.url)).toString("base64"), + source: { id: "harbor-guide", version: "commit-abc123" }, + }], + absent: [], +}); +const control = defineWorkspaceFiles({ + version: 1, + files: [], + absent: [".agents/skills/harbor"], +}); + +// Export each arm from its own experiment module. Keep all other settings equal. +const common = { + agent: { adapter: "codex", model: "gpt-5.4-2026-03-05", reasoning: "high" }, + comparisonGroup: "harbor-files", +}; +export default defineExperiment({ + ...common, + id: "codex-files", // control module: codex-no-files + treatment: "files", + runtime: { + kind: "container", + image: "quickstudy/agent-runtime:local", // use your actual resolved image + workspaceFiles: offered, // control module: control + treatmentPolicyFields: ["workspaceFiles"], + }, +}); +``` + +A real payload includes its required reference files and license/attribution, +each as another file with its own destination and declared source. The harness +does not infer references, recursively read directories, fetch upstream content, +or prove that a source label is authentic. Do not use this fictional guide or +source pin as a substitute for an approved upstream payload. + +Omitting the declaration means *no overlay instrumentation*, not proven absence. +A no-file control should assert the entire reserved tree absent. Existing +contamination fails the control; it is never deleted to manufacture cleanliness. +The assertion does not search other workspace locations or the image/home. + +## v1 data and limits + +All fields are plain data: + +- `version: 1` selects the staging semantics below. Unknown versions fail + preflight; they are never interpreted as v1. +- `files: readonly WorkspaceFile[]` lists regular, non-executable files. + Each file has `path`, `content`, explicit `encoding: "utf8" | "base64"`, and + `source: { id, version }`. +- `absent: readonly string[]` lists file-or-tree paths that must not exist. +- Source id/version are opaque labels: 1–160 ASCII characters, starting with + an alphanumeric character, followed by alphanumerics or `_.:@+-`. Use an + immutable revision or content version. URLs, absolute source paths and + credentials do not belong here. + +Limits: **128 files**, **128 absence assertions**, **256 KiB per file**, and +**1 MiB total decoded payload**. Empty files are supported. UTF-8 converts with +`Buffer.from(content, "utf8")`; unpaired surrogates are rejected. Base64 must be +canonical, padded RFC 4648 (the empty string is valid). It preserves arbitrary +bytes, including NUL, invalid UTF-8, CRLF and trailing newlines. Equivalent UTF-8 +and base64 declarations produce the same byte identity. No shell interpolation +or newline normalization occurs. Files are created with mode `0644`; there is +no executable flag, symlink, hardlink, device, directory-payload or mount type. +Nothing in a payload is executed by staging. + +Paths are relative to `/workspace`, at most **240 ASCII characters** and **16 +segments**, separated by `/`. Each nonempty segment uses only `A-Za-z0-9_.-`. +Absolute paths, `.`/`..`, trailing dots, empty segments, backslashes, colons, +Windows drive/UNC/device names, whitespace and control characters are rejected. +Case-insensitive duplicate or ancestor/descendant destinations are rejected, +including overlaps with absence assertions. Sibling files may share parents. + +Reserved segments include `.git*`, `.hg*`, `.svn*`, `.quickstudy*`, credential-like +names (the harness's credential-file exclusion rules), and export-excluded +names: `node_modules`, `vendor`, `.venv`, `.next`, `.cache`, `__pycache__`. +These conservative rules apply at every depth. This is not a credential delivery +channel: use runtime provisioning/environment injection for secrets instead. +Payloads are explicitly non-secret and can appear in the exported workspace and +diff; hashes are not a safe way to publish low-entropy secrets either. + +## Ordering and filesystem safety + +Supported: **container adapters**, including the container oracle (whose +fixture/reference starting state is prepared as usual). Images must provide +Git and Linux Node at `/usr/local/bin/node` (the standard agent-runtime does). +The absolute Node executable avoids PATH-treatment shadowing. Missing tooling +is an environment error. Host adapters and nonempty scripted `runtime.command` +combinations are rejected at preflight, including absence-only declarations. +Without an overlay, existing host/scripted/oracle behavior is unchanged. + +Each attempt: + +1. Copies the starting state into a fresh source and container; commits the + fixture/reference baseline. Overlays require a successful Git baseline. +2. Computes normal adapter setup file declarations; checks for overlapping + adapter destinations before any overlay writes. +3. Validates all overlay destinations and absence assertions in the actual + sandbox, then creates the files exclusively. Existing destination files, + directories, dangling links, and symlink ancestors are errors. Existing + ordinary parent directories are allowed; extra sibling files are not deleted. +4. Reads bytes back and compares sizes/hashes to the identity-bound inventory. +5. Writes the adapter's ordinary setup files (for example, Codex MCP TOML). +6. Rechecks file type, mode, link count, sizes/hashes and every absence assertion, + then writes successful staging evidence. Adapter setup cannot silently + invalidate an overlay. Setup paths that alias through symlinks are rejected. +7. Invokes the **unchanged adapter command**, including its normal login, + configuration flags, transcript/metrics parsing, export, scoring and teardown. + +Writes travel as JSON/base64 on stdin, not through argv or a shell. The Linux +helper walks ancestors using no-follow directory descriptors anchored under +`/workspace`; reads/writes also use no-follow opens. Staging has no reason to +write the consumer fixture, another attempt, the shared image or host home. +Trusted image executables and harness adapters remain part of the trust boundary; +this is not attestation against a malicious image or hostile daemon. + +The whole stage/setup/final-verification sequence has a fixed **30-second** v1 +budget. Cancellation stops its transport; the runner awaits the stopped operation +and tears down the sandbox rather than racing a late write against agent start. +No agent execution follows a timeout. This budget is part of versioned semantics, +not the agent budget or a new CLI flag. + +Invalid metadata/unsupported combinations fail at definition or run preflight. +Filesystem, adapter setup, write/read-back, absence, timeout and cancellation +failures produce `status: error`, `error_kind: environment`, no scorer result, +and null agent timestamps. They are not genuine agent/scorer failures. Payloads +are omitted from staging diagnostics; cleanup failures are recorded alongside, +not instead of, the primary error. + +## Intended identity versus placement evidence + +The sorted inventory is in the v4 manifest at: + +```text +pairs[JSON.stringify([evalId, experimentId])].workspaceFiles + version + files[]: { path, size, sha256, source: { id, version } } + absent[] +``` + +It contributes to both runtime/experiment and attempt identity. The actual +payload bytes are hashed automatically—even bytes read dynamically outside +literal imports or `quickstudy.identity.json` resource trees. There is no manual +second hash field to drift. At run entry the harness snapshots/canonicalizes the +payload before awaiting external work, and staging uses that immutable snapshot. +Injected/stale manifests must match it. Bytes, destination, provenance, absence +expectations, and staging version are identity-bearing. Unsupported versions +are refused rather than generating a runnable identity. + +After successful final verification, the attempt directory contains: + +```text +results///workspace-files.json + version: 1 + verified: + verifiedAt: +``` + +This artifact is separate from the intended manifest. No successful evidence is +written on staging failure. Evidence can still exist if cancellation happens +immediately afterward or the agent later fails. It proves placement/checks **at +that time**, not agent discovery, reading, obedience, or successful task execution. +The agent may subsequently edit these files. Inspect transcript/tool evidence +separately before claiming use. + +No-overlay v4 manifests retain their old shape; v3 and old v4 results are not +rewritten. Normal harness/source drift still refuses resume. Overlay identity +drift also refuses stale manifests/resume; completed attempts are never silently +reused under new bytes. + +## Comparison contract + +Both arms explicitly list `"workspaceFiles"` in +`runtime.treatmentPolicyFields` to exempt only this resolved overlay inventory +from group comparison. Both must agree on the **entire** exemption list. Full +identities remain distinct. Without this declaration, differing files or absence +assertions withhold comparison. Use exactly `["workspaceFiles"]` for a files-only +study; do not inherit unrelated exemptions from another treatment. + +This does not exempt image, model, reasoning, CLI, fixture/scorer identity, +command, lifecycle budgets, runtime configuration outside the treatment subtree, +or network/MCP/PATH/web policy. The existing separate policy exemptions retain +their explicit meaning; adding files grants no additional tool/network access. +Comparison is agreement on declared configuration, not causality or proof of use. + +See [the consumer handoff](workspace-files-handoff.md) for isolated local +integration before a real release. diff --git a/docs/writing-an-eval.md b/docs/writing-an-eval.md index 68ecb24..fabb11f 100644 --- a/docs/writing-an-eval.md +++ b/docs/writing-an-eval.md @@ -239,6 +239,12 @@ That is the entire authoring loop: edit the prompt or scorer, `run`, `report`. Iterating on a real agent's single attempt from a source checkout: `bun scripts/dev-attempt.ts --eval --experiment `. +For a container agent experiment that offers skills or documents, use +[`runtime.workspaceFiles`](workspace-files.md), not edits to the shared `local/` +fixture or a wrapper `runtime.command`. The declaration binds bytes to identity +and preserves the adapter. A no-files control can require the reserved tree to +be absent. This feature is not supported by the host Echo smoke above. + ## Composing shared check helpers A benchmark repository grows shared check helpers next to its evals — a diff --git a/src/execution-policy.ts b/src/execution-policy.ts index d602439..883a4a4 100644 --- a/src/execution-policy.ts +++ b/src/execution-policy.ts @@ -1,3 +1,4 @@ +import { workspaceFilesIdentity, type WorkspaceFiles, type WorkspaceFilesIdentity } from "./runtime/workspace-files.ts"; import type { LoadedEval } from "./evals/types.ts"; import type { LoadedExperiment } from "./experiments/types.ts"; import type { RuntimeImageIdentity } from "./manifest.ts"; @@ -19,6 +20,8 @@ export const DEFAULT_BUDGETS: LifecycleBudgets = { cleanupMs: 30_000, }; export interface PairEnvironment { + /** Omitted on historical/no-overlay pairs; payload bytes never enter manifests. */ + workspaceFiles?: WorkspaceFilesIdentity; image: RuntimeImageIdentity | null; egress: { proxy: boolean; allowlist: string[] }; webPolicy: WebPolicy; @@ -55,6 +58,8 @@ export function resolvePairEnvironments(options: { budgets?: Partial; mcpHashes?: Record; imageTags?: Record; + /** Runner-captured declarations; entries with undefined explicitly mean no overlay. */ + workspaceFilesByExperiment?: ReadonlyMap; }): Record { const budgets = { ...DEFAULT_BUDGETS, ...options.budgets }; for (const [name, value] of Object.entries(budgets)) @@ -63,11 +68,13 @@ export function resolvePairEnvironments(options: { for (const loaded of options.evals) for (const { experiment } of options.experiments) { const runtime = experiment.runtime; + const files = options.workspaceFilesByExperiment ? options.workspaceFilesByExperiment.get(experiment.id) : runtime.workspaceFiles; const tag = options.imageTags?.[pairKey(loaded.metadata.id, experiment.id)] ?? runtime.image ?? runtime.containerImage?.(loaded.metadata); pairs[pairKey(loaded.metadata.id, experiment.id)] = { + ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}), image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null, // With the flag, every container is proxied. No runtime hosts means // only the adapter's provider endpoints are permitted. diff --git a/src/experiments/types.ts b/src/experiments/types.ts index d8a443e..ea13332 100644 --- a/src/experiments/types.ts +++ b/src/experiments/types.ts @@ -7,6 +7,7 @@ */ import type { EvalMetadata } from "../evals/types.ts"; +import { validateWorkspaceFilesRuntime, type WorkspaceFiles } from "../runtime/workspace-files.ts"; import type { SurfaceUsageConfig } from "../surface-usage.ts"; import type { RuntimeCapabilities } from "../runtime/types.ts"; @@ -53,7 +54,9 @@ export interface Runtime extends RuntimeCapabilities { * Images and lifecycle budgets can never be exempted. All group members must * declare the same fields. Source/config identity still changes per attempt. */ - treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble">; + treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble" | "workspaceFiles">; + /** Attempt-local non-secret files, staged without replacing the normal adapter. */ + workspaceFiles?: WorkspaceFiles; /** Container image tag (container runtimes; alternative to `containerImage`). */ image?: string; /** Scripted command run in /workspace (container runtimes, this phase). */ @@ -124,5 +127,6 @@ export function defineExperiment(experiment: Experiment): Experiment { if (experiment.comparisonGroup !== undefined && experiment.comparisonGroup.trim() === "") { throw new Error(`defineExperiment: experiment "${experiment.id}" comparisonGroup must not be empty`); } + validateWorkspaceFilesRuntime(experiment.runtime); return deepFreeze(experiment); } diff --git a/src/index.ts b/src/index.ts index eac4cdf..1022d55 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,9 @@ export type { LoadedEval, } from "./evals/types.ts"; export { EVAL_SUITES } from "./evals/types.ts"; +export { defineWorkspaceFiles, WORKSPACE_FILES_LIMITS } from "./runtime/workspace-files.ts"; +export type { WorkspaceFile, WorkspaceFiles, WorkspaceFilesIdentity } from "./runtime/workspace-files.ts"; +export type { WorkspaceFilesEvidence } from "./isolation/workspace-files.ts"; export { defineExperiment } from "./experiments/types.ts"; export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts"; export type { diff --git a/src/isolation/docker.ts b/src/isolation/docker.ts index 7f3f661..4195d52 100644 --- a/src/isolation/docker.ts +++ b/src/isolation/docker.ts @@ -21,6 +21,8 @@ import { mkdir, mkdtemp, rm, writeFile, cp, readdir, lstat, readlink, realpath } import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { sanitizeServerName, type McpUpstream } from "./mcp-proxy/server.ts"; +import { WORKSPACE_FILES_PROGRAM, type WorkspaceFilesRequest } from "./workspace-files.ts"; +import type { WorkspaceFilesIdentity } from "../runtime/workspace-files.ts"; import { EXCLUDED_WORKSPACE_DIRS } from "../store/artifacts.ts"; /** The label key every attempt container carries. */ @@ -66,14 +68,17 @@ export interface ExecResult { */ async function runDocker( args: string[], - opts: { timeoutMs?: number; stdin?: string; env?: Record } = {}, + opts: { timeoutMs?: number; stdin?: string; env?: Record; signal?: AbortSignal } = {}, ): Promise { + opts.signal?.throwIfAborted(); const proc = Bun.spawn(["docker", ...args], { stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin), stdout: "pipe", stderr: "pipe", ...(opts.env !== undefined ? { env: { ...process.env, ...opts.env } } : {}), }); + const abort = () => proc.kill("SIGKILL"); + opts.signal?.addEventListener("abort", abort, { once: true }); let timedOut = false; let timer: ReturnType | undefined; { @@ -88,13 +93,14 @@ async function runDocker( proc.exited, ]); if (timer !== undefined) clearTimeout(timer); + opts.signal?.removeEventListener("abort", abort); return { exitCode, stdout, stderr, timedOut }; } /** Run `docker ` and throw a DockerError on non-zero exit. */ async function mustDocker( args: string[], - opts: { timeoutMs?: number; stdin?: string; env?: Record } = {}, + opts: { timeoutMs?: number; stdin?: string; env?: Record; signal?: AbortSignal } = {}, ): Promise { const result = await runDocker(args, opts); if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr); @@ -193,7 +199,9 @@ export interface AttemptContainer { /** Run a command in /workspace inside the container. Never throws for non-zero exits. */ execInWorkspace(cmd: string[], opts?: { timeoutMs?: number }): Promise; /** Write a file inside the container (parent directories created). */ - writeFile(path: string, content: string): Promise; + writeFile(path: string, content: string, opts?: { signal?: AbortSignal }): Promise; + /** Bounded create-only overlay transport; input rides stdin, never argv. */ + workspaceFiles(request: WorkspaceFilesRequest, signal: AbortSignal): Promise; /** Copy /workspace to a host directory, excluding node_modules/vendor/.venv. */ exportWorkspace(toDir: string): Promise; /** `docker rm -f` — kill and remove. Idempotent. */ @@ -282,10 +290,22 @@ export async function createAttemptContainer(opts: CreateAttemptContainerOptions } return result; }, - writeFile: async (path, content) => { + workspaceFiles: async (request, signal) => { + try { + const result = await runDocker(["exec", "-i", "-w", CONTAINER_WORKSPACE, id, "/usr/local/bin/node", "-e", WORKSPACE_FILES_PROGRAM, CONTAINER_WORKSPACE], { + stdin: JSON.stringify(request), signal, + }); + if (signal.aborted || result.timedOut || result.exitCode !== 0) throw new Error("staging failed"); + return JSON.parse(result.stdout) as WorkspaceFilesIdentity; + } catch { + // No raw stdout/stderr, command, or payload may enter diagnostics. + throw new Error("workspaceFiles: filesystem operation failed or was cancelled/timed out"); + } + }, + writeFile: async (path, content, writeOpts = {}) => { // `$1` positional keeps the target path out of shell interpolation. await mustDocker(["exec", "-i", id, "sh", "-c", 'mkdir -p "$(dirname "$1")" && cat > "$1"', "sh", path], { - stdin: content, + stdin: content, signal: writeOpts.signal, }); }, exportWorkspace: async (toDir) => { diff --git a/src/isolation/workspace-files.ts b/src/isolation/workspace-files.ts new file mode 100644 index 0000000..f2361fe --- /dev/null +++ b/src/isolation/workspace-files.ts @@ -0,0 +1,92 @@ +import type { WorkspaceFiles, WorkspaceFilesIdentity } from "../runtime/workspace-files.ts"; + +/** The same program runs in the container and offline filesystem tests. No shell or payload in argv. */ +export const WORKSPACE_FILES_PROGRAM = String.raw` +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const path = require('node:path'); +function apply(root, request) { + const { files, absent } = request.declaration; + const fail = () => { throw new Error('unsafe workspace state'); }; + const stat = (p) => { try { return fs.lstatSync(p); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } }; + if (!stat(root)?.isDirectory() || stat(root).isSymbolicLink()) fail(); + // No processes controlled by the agent run during preparation. On Linux, + // directory descriptors also anchor traversal against concurrent renames. + const descriptors = []; + const directory = (p) => { + const s = stat(p); + if (!s?.isDirectory() || s.isSymbolicLink()) fail(); + const fd = fs.openSync(p, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW); + descriptors.push(fd); + return process.platform === 'linux' ? '/proc/self/fd/' + fd : p; + }; + const base = directory(root); + const leaf = (relative, create = false) => { + while (descriptors.length > 1) fs.closeSync(descriptors.pop()); + const parts = relative.split('/'); + let parent = base; + for (const part of parts.slice(0, -1)) { + const next = parent + '/' + part; + if (!stat(next)) { + if (!create) return null; + fs.mkdirSync(next, { mode: 0o755 }); + } + parent = directory(next); + } + return parent + '/' + parts.at(-1); + }; + const checkAbsent = (relative) => { const p = leaf(relative); if (p !== null && stat(p)) fail(); }; + try { + // Setup paths may be outside the workspace, but cannot alias it through + // symlinks. Lexical overlap is separately rejected before any write. + for (const target of request.setupPaths) { + const absolute = path.resolve(root, target); + let current = '/'; + for (const part of absolute.split('/').filter(Boolean)) { + current = path.join(current, part); + const s = stat(current); + if (!s) break; + if (s.isSymbolicLink()) fail(); + } + } + for (const relative of absent) checkAbsent(relative); + if (request.phase === 'stage') { + // Validate the entire inventory before writing even the first file. + for (const file of files) checkAbsent(file.path); + for (const file of files) { + const p = leaf(file.path, true); + const fd = fs.openSync(p, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o644); + try { fs.writeFileSync(fd, Buffer.from(file.content, 'base64')); fs.fchmodSync(fd, 0o644); } + finally { fs.closeSync(fd); } + } + } + const inventory = files.map((file) => { + const p = leaf(file.path); + if (!p) fail(); + const fd = fs.openSync(p, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + try { + const s = fs.fstatSync(fd); + if (!s.isFile() || s.nlink !== 1 || (s.mode & 0o777) !== 0o644 || s.size > 262144) fail(); + const bytes = fs.readFileSync(fd); + return { path: file.path, size: bytes.length, sha256: crypto.createHash('sha256').update(bytes).digest('hex'), source: file.source }; + } finally { fs.closeSync(fd); } + }); + return { version: 1, files: inventory, absent }; + } finally { for (const fd of descriptors.reverse()) fs.closeSync(fd); } +} +try { + const request = JSON.parse(fs.readFileSync(0, 'utf8')); + process.stdout.write(JSON.stringify(apply(process.argv[1], request))); +} catch { process.stderr.write('workspaceFiles: filesystem validation/write/read-back failed'); process.exitCode = 1; } +`; + +export interface WorkspaceFilesRequest { + declaration: WorkspaceFiles; + phase: "stage" | "verify"; + setupPaths: string[]; +} +export interface WorkspaceFilesEvidence { + version: 1; + verified: WorkspaceFilesIdentity; + verifiedAt: number; +} diff --git a/src/manifest.ts b/src/manifest.ts index f2dcc49..d4b1858 100644 --- a/src/manifest.ts +++ b/src/manifest.ts @@ -10,6 +10,7 @@ * adding or removing a component is a deliberate, test-visible act. */ +import { validateWorkspaceFilesRuntime, workspaceFilesIdentity, type WorkspaceFiles } from "./runtime/workspace-files.ts"; import { readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { EMBEDDED_BUILD_INFO } from "./build-info.generated.ts"; @@ -151,6 +152,8 @@ export interface BuildRunManifestOptions { harness?: HarnessIdentity; pairs?: Record; scorerIdentities?: Record>; + /** Share the runner's immutable payload snapshot without copying runtime instances. */ + workspaceFilesByExperiment?: ReadonlyMap; } /** Build before the first paid attempt; all reads are deterministic and local. */ @@ -178,6 +181,8 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest const experiments: Record = {}; for (const { experiment, path } of options.experiments) { + const files = options.workspaceFilesByExperiment ? options.workspaceFilesByExperiment.get(experiment.id) : experiment.runtime.workspaceFiles; + validateWorkspaceFilesRuntime({ kind: experiment.runtime.kind, command: experiment.runtime.command, workspaceFiles: files }); const imageTag = experiment.runtime.image; const components: ExperimentIdentityComponents = { source: sourceIdentity(path), @@ -190,6 +195,7 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest }, runtime: contentHash({ kind: experiment.runtime.kind, + ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}), // A capacity resolver is a capability function: per-eval, never identity. capacity: typeof experiment.runtime.capacity === "function" ? null : (experiment.runtime.capacity ?? null), observationTargets: experiment.runtime.observationTargets ?? null, diff --git a/src/runner/execute.ts b/src/runner/execute.ts index ea509a5..8f8795e 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -21,6 +21,8 @@ * and never allowed to mask the attempt's result. */ +import { stageWorkspaceFiles } from "./workspace-files.ts"; +import { defineWorkspaceFiles, validateWorkspaceFilesRuntime, workspaceFilesIdentity, type WorkspaceFiles } from "../runtime/workspace-files.ts"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -103,6 +105,8 @@ export interface ExecuteRunV3Options { config?: Partial; /** Host adapters by name. Defaults to the built-in echo adapter. */ hostAdapters?: Record; + /** Internal offline-test seam; not part of the public consumer API. */ + createContainer?: typeof createAttemptContainer; /** * Container agent adapter factories by name, constructed per experiment * from its `agent` spec. Defaults to the built-in claude/codex adapters. @@ -315,6 +319,13 @@ export function collectOfferedSurfaces(options: { } export async function executeRunV3(options: ExecuteRunV3Options): Promise { + // Snapshot only the payload: runtime methods must keep their original receiver, + // including prototype methods and private fields. Capture before the first await. + const workspaceFilesByExperiment = new Map(options.experiments.map(({ experiment }) => { + validateWorkspaceFilesRuntime(experiment.runtime); + const files = experiment.runtime.workspaceFiles; + return [experiment.id, files === undefined ? undefined : defineWorkspaceFiles(files)] as const; + })); const { store, artifacts, plan } = options; const log = options.log ?? ((line: string) => console.log(line)); const attemptTimeoutMs = options.attemptTimeoutMs ?? options.budgets?.agentMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS; @@ -469,7 +480,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise !isSecretEnvName(name))); return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : []; })); - const pairs = resolvePairEnvironments({ ...options, images, imageTags: imageTagsByPair, + const pairs = resolvePairEnvironments({ ...options, workspaceFilesByExperiment, images, imageTags: imageTagsByPair, adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [ ...(adapter.egressHosts ?? []), ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []), @@ -478,7 +489,16 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise [id, hashString(canonicalJson(servers))])), }); - manifest = buildRunManifest({ evals: options.evals, experiments: options.experiments, images, cliVersions, pairs, scorerIdentities }); + manifest = buildRunManifest({ evals: options.evals, experiments: options.experiments, workspaceFilesByExperiment, images, cliVersions, pairs, scorerIdentities }); + } + // Injected/stale manifests must not authorize a different payload (or silently + // omit one). Historical no-overlay manifests retain their original shape. + for (const entry of plan) { + const files = workspaceFilesByExperiment.get(entry.experimentId); + const intended = files ? workspaceFilesIdentity(files) : undefined; + if (canonicalJson(intended ?? null) !== canonicalJson(manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)]?.workspaceFiles ?? null)) { + throw new Error("workspaceFiles: manifest payload identity mismatch"); + } } const recorded = options.resumeRunId ? store.getRun(options.resumeRunId) : null; if (options.resumeRunId && !recorded) throw new Error(`cannot resume unknown run ${options.resumeRunId}`); @@ -583,7 +603,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise artifacts.prepareAttempt(runId, attemptId)); const fixtureDir = sourceDir; - const container = await inStage("environment", () => createAttemptContainer({ + const container = await inStage("environment", () => (options.createContainer ?? createAttemptContainer)({ image, env, resultsDir: paths.attemptDir, @@ -890,6 +912,7 @@ async function runContainerAttemptV3( return (await mustExec(container, ["git", "rev-parse", "HEAD"])).stdout.trim(); }); } else { + if (workspaceFiles) throw new StageFailure("environment", new Error("workspaceFiles: git baseline is required")); (options.log ?? console.log)( `attempt ${attemptId}: git is unavailable in image "${image}" — diff.patch will be empty ` + `(the agent-runtime image ships git; only bare test images lack it)`, @@ -912,8 +935,13 @@ async function runContainerAttemptV3( }; // Treatment config files (e.g. the MCP treatment's vendor config). await inStage("environment", async () => { - for (const [path, content] of Object.entries(agentAdapter.setupFiles(promptCtx))) { - await container.writeFile(path, content); + const setup = agentAdapter.setupFiles(promptCtx); + if (workspaceFiles) { + const evidence = await stageWorkspaceFiles(container, workspaceFiles, environment!.workspaceFiles!, setup, options.signal); + await artifacts.writeWorkspaceFilesEvidence(runId, attemptId, evidence); + if (options.signal?.aborted) throw new Error("workspaceFiles: staging cancelled"); + } else { + for (const [path, content] of Object.entries(setup)) await container.writeFile(path, content); } }); command = agentAdapter.command(promptCtx); @@ -921,6 +949,7 @@ async function runContainerAttemptV3( command = runtime.command as string[]; } + if (workspaceFiles && options.signal?.aborted) throw new StageFailure("environment", new Error("workspaceFiles: staging cancelled")); persisted.agentStartedAt = Date.now(); let agentResult: Awaited>; try { @@ -1043,7 +1072,10 @@ async function runContainerAttemptV3( try { await egress.sidecar.teardown(); } catch { containerTeardownFailure = "egress sidecar teardown failed"; } try { await removeRunNetwork(egress.networkName); } catch { containerTeardownFailure = "egress network teardown failed"; } } - if (sourceDir) await rm(sourceDir, { recursive: true, force: true }); + if (sourceDir) { + try { await rm(sourceDir, { recursive: true, force: true }); } + catch { containerTeardownFailure = [containerTeardownFailure, "workspace source cleanup failed"].filter(Boolean).join("; "); } + } } // Explicit teardown accounting: the attempt keeps its status and result; diff --git a/src/runner/workspace-files.ts b/src/runner/workspace-files.ts new file mode 100644 index 0000000..871331b --- /dev/null +++ b/src/runner/workspace-files.ts @@ -0,0 +1,52 @@ +import { posix } from "node:path"; +import { canonicalJson } from "../identity-diff.ts"; +import type { AttemptContainer } from "../isolation/docker.ts"; +import type { WorkspaceFilesEvidence } from "../isolation/workspace-files.ts"; +import { pathsOverlap, WORKSPACE_FILES_LIMITS, type WorkspaceFiles, type WorkspaceFilesIdentity } from "../runtime/workspace-files.ts"; + +/** No race-to-timeout: await cancelled transport, then let the attempt's finally tear down. */ +export async function stageWorkspaceFiles( + container: AttemptContainer, + declaration: WorkspaceFiles, + intended: WorkspaceFilesIdentity, + setup: Record, + parentSignal?: AbortSignal, +): Promise { + const controller = new AbortController(); + const abort = () => controller.abort(); + const timer = setTimeout(abort, WORKSPACE_FILES_LIMITS.timeoutMs); + parentSignal?.addEventListener("abort", abort, { once: true }); + if (parentSignal?.aborted) abort(); + const signal = controller.signal; + try { + const setupPaths = Object.keys(setup); + for (const path of setupPaths) { + const relative = posix.relative("/workspace", posix.resolve("/workspace", path)); + if ([...intended.files.map((file) => file.path), ...intended.absent].some((target) => relative === "" || pathsOverlap(relative, target))) { + throw new Error("workspaceFiles: adapter setup collision"); + } + } + const check = async (phase: "stage" | "verify") => { + signal.throwIfAborted(); + const actual = await container.workspaceFiles({ declaration, phase, setupPaths }, signal); + signal.throwIfAborted(); + if (canonicalJson(actual) !== canonicalJson(intended)) throw new Error("workspaceFiles: read-back/absence verification mismatch"); + return actual; + }; + await check("stage"); + for (const [path, content] of Object.entries(setup)) { + signal.throwIfAborted(); + await container.writeFile(path, content, { signal }); + } + const verified = await check("verify"); + return { version: 1, verified, verifiedAt: Date.now() }; + } catch (err) { + if (signal.aborted) throw new Error("workspaceFiles: staging cancelled or timed out"); + // Only our own fixed diagnostics survive; transport/setup errors may carry payloads. + if (err instanceof Error && ["workspaceFiles: adapter setup collision", "workspaceFiles: read-back/absence verification mismatch"].includes(err.message)) throw err; + throw new Error("workspaceFiles: staging filesystem/setup operation failed"); + } finally { + clearTimeout(timer); + parentSignal?.removeEventListener("abort", abort); + } +} diff --git a/src/runtime/workspace-files.ts b/src/runtime/workspace-files.ts new file mode 100644 index 0000000..0f177a4 --- /dev/null +++ b/src/runtime/workspace-files.ts @@ -0,0 +1,91 @@ +import { hashString, SECRET_FILE } from "../hash.ts"; +import { EXCLUDED_WORKSPACE_DIRS } from "../store/artifacts.ts"; + +/** Non-secret, regular, non-executable files. Source labels are declarations, not attestations. */ +export interface WorkspaceFile { + path: string; + content: string; + encoding: "utf8" | "base64"; + source: { id: string; version: string }; +} + +/** v1: create-only, no symlinks/hardlinks, 0644 files, final read-back, no deletion. */ +export interface WorkspaceFiles { + version: 1; + files: readonly WorkspaceFile[]; + absent: readonly string[]; +} + +export interface WorkspaceFilesIdentity { + version: 1; + files: readonly { path: string; size: number; sha256: string; source: { id: string; version: string } }[]; + absent: readonly string[]; +} + +export const WORKSPACE_FILES_LIMITS = Object.freeze({ files: 128, absent: 128, fileBytes: 262144, totalBytes: 1048576, pathBytes: 240, depth: 16, timeoutMs: 30000 }); + +function invalid(reason: string): never { + // Never interpolate consumer data: even malformed metadata may contain secrets. + throw new Error(`workspaceFiles: ${reason}`); +} + +function pathCheck(path: unknown): asserts path is string { + if (typeof path !== "string" || path.length > WORKSPACE_FILES_LIMITS.pathBytes) invalid("invalid relative path"); + const parts = path.split("/"); + if (parts.length > WORKSPACE_FILES_LIMITS.depth || parts.some((part) => + !/^[A-Za-z0-9_.-]+$/.test(part) || part === "." || part === ".." || part.endsWith(".") || + /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(?:\.|$)/i.test(part) || + EXCLUDED_WORKSPACE_DIRS.includes(part.toLowerCase()) || SECRET_FILE.test(part) || + /^\.(?:git|hg|svn)/i.test(part) || /^\.quickstudy/i.test(part) + )) invalid("unsafe or reserved relative path"); +} + +export function pathsOverlap(a: string, b: string): boolean { + a = a.toLowerCase(); b = b.toLowerCase(); + return a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`); +} + +/** Validate, copy, canonicalize to base64, and freeze without retaining mutable caller state. */ +export function defineWorkspaceFiles(input: WorkspaceFiles): WorkspaceFiles { + if (!input || input.version !== 1) invalid("unsupported staging version"); + if (!Array.isArray(input.files) || !Array.isArray(input.absent) || + input.files.length > WORKSPACE_FILES_LIMITS.files || input.absent.length > WORKSPACE_FILES_LIMITS.absent) invalid("invalid inventory or count limit exceeded"); + let total = 0; + const files = input.files.map((file) => { + if (!file || typeof file.content !== "string" || file.content.length > WORKSPACE_FILES_LIMITS.fileBytes * 4) invalid("invalid or oversized payload"); + pathCheck(file.path); + if (!file.source || ![file.source.id, file.source.version].every((v) => typeof v === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:@+-]{0,159}$/.test(v))) invalid("invalid source id/version (use opaque labels, not paths or URLs)"); + if (file.encoding !== "utf8" && file.encoding !== "base64") invalid("unsupported encoding"); + const bytes = Buffer.from(file.content, file.encoding === "utf8" ? "utf8" : "base64"); + if (file.encoding === "base64" && bytes.toString("base64") !== file.content) invalid("base64 must be canonical padded RFC 4648"); + if (file.encoding === "utf8" && bytes.toString("utf8") !== file.content) invalid("utf8 must not contain unpaired surrogates"); + total += bytes.length; + if (bytes.length > WORKSPACE_FILES_LIMITS.fileBytes || total > WORKSPACE_FILES_LIMITS.totalBytes) invalid("payload size limit exceeded"); + return Object.freeze({ path: file.path, content: bytes.toString("base64"), encoding: "base64" as const, source: Object.freeze({ id: file.source.id, version: file.source.version }) }); + }).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); + const absent = [...input.absent]; + for (const path of absent) pathCheck(path); + const paths = [...files.map((file) => file.path), ...absent]; + for (let i = 0; i < paths.length; i++) for (let j = i + 1; j < paths.length; j++) { + if (pathsOverlap(paths[i]!, paths[j]!)) invalid("duplicate or overlapping destinations/absence assertions"); + } + return Object.freeze({ version: 1, files: Object.freeze(files), absent: Object.freeze(absent.sort()) }); +} + +export function workspaceFilesIdentity(input: WorkspaceFiles): WorkspaceFilesIdentity { + const declaration = defineWorkspaceFiles(input); + return { + version: declaration.version, + files: declaration.files.map((file) => { + const bytes = Buffer.from(file.content, "base64"); + return { path: file.path, size: bytes.length, sha256: hashString(bytes), source: file.source }; + }), + absent: declaration.absent, + }; +} + +export function validateWorkspaceFilesRuntime(runtime: { kind: string; command?: string[]; workspaceFiles?: WorkspaceFiles }): void { + if (runtime.workspaceFiles === undefined) return; + defineWorkspaceFiles(runtime.workspaceFiles); + if (runtime.kind !== "container" || (runtime.command?.length ?? 0) > 0) invalid("requires a container adapter; host and scripted commands are unsupported"); +} diff --git a/src/store/artifacts.ts b/src/store/artifacts.ts index 0ac24c2..084ae0a 100644 --- a/src/store/artifacts.ts +++ b/src/store/artifacts.ts @@ -22,6 +22,7 @@ import { cp, mkdir, writeFile, lstat, realpath, readlink } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; +import type { WorkspaceFilesEvidence } from "../isolation/workspace-files.ts"; import type { TranscriptEvent } from "../adapters/types.ts"; /** Dependency directories never copied into an exported workspace. */ @@ -73,6 +74,12 @@ export class ArtifactsStore { return paths; } + /** Placement evidence, distinct from intended identity; proves no model use. */ + async writeWorkspaceFilesEvidence(runId: string, attemptId: string, evidence: WorkspaceFilesEvidence): Promise { + const paths = await this.prepareAttempt(runId, attemptId); + await writeFile(join(paths.attemptDir, "workspace-files.json"), `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + } + async writeTranscript(runId: string, attemptId: string, events: TranscriptEvent[]): Promise { const paths = await this.prepareAttempt(runId, attemptId); const jsonl = events.map((event) => JSON.stringify(event)).join("\n"); diff --git a/tests/mcp-proxy.test.ts b/tests/mcp-proxy.test.ts index 2c9ef12..d16f695 100644 --- a/tests/mcp-proxy.test.ts +++ b/tests/mcp-proxy.test.ts @@ -23,6 +23,7 @@ import { import { sanitizeServerName, startMcpProxy, type McpUpstream } from "../src/isolation/mcp-proxy/server.ts"; import { discoverEvals } from "../src/evals/discovery.ts"; import type { LoadedExperiment } from "../src/experiments/types.ts"; +import { buildRunManifest } from "../src/manifest.ts"; import { planRun } from "../src/plan.ts"; import { executeRunV3 } from "../src/runner/execute.ts"; import { ArtifactsStore } from "../src/store/artifacts.ts"; @@ -308,8 +309,8 @@ describe("mcp wiring helpers", () => { }); // --------------------------------------------------------------------------- -// Runner fail-fast (no docker needed: the credentials check throws before any -// container work). Successor to the deleted pack-loader/orchestrator guard: +// Runner fail-fast (inject a locally built manifest to bypass image discovery +// for offline isolation; the credentials check throws before container work). Successor to the deleted pack-loader/orchestrator guard: // the shape validation of auth blocks now lives with the runtime authors // (TypeScript types), while the credentials-file preflight lives in the v3 // runner's treatment setup. @@ -362,6 +363,7 @@ describe("mcp auth: v3 runner fail-fast", () => { executeRunV3({ evals, experiments, + manifest: buildRunManifest({ evals, experiments }), plan: planRun(evals, experiments.map((entry) => entry.experiment), 1), store, artifacts, diff --git a/tests/workspace-files.test.ts b/tests/workspace-files.test.ts new file mode 100644 index 0000000..78237cd --- /dev/null +++ b/tests/workspace-files.test.ts @@ -0,0 +1,525 @@ +import { afterAll, afterEach, describe, expect, test } from "bun:test"; +import { chmodSync, cpSync, existsSync, linkSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { defineExperiment, defineWorkspaceFiles, WORKSPACE_FILES_LIMITS, type WorkspaceFiles, type Experiment } from "../src/index.ts"; +import { CodexAdapter, CODEX_CONFIG_PATH } from "../src/adapters/codex.ts"; +import { OracleContainerAdapter } from "../src/adapters/oracle.ts"; +import type { ContainerAgentAdapter } from "../src/adapters/types.ts"; +import { discoverEvals } from "../src/evals/discovery.ts"; +import { compareGroups } from "../src/experiments/groups.ts"; +import { pairKey, resolvePairEnvironments } from "../src/execution-policy.ts"; +import { hashString, hashTree } from "../src/hash.ts"; +import { WORKSPACE_FILES_PROGRAM } from "../src/isolation/workspace-files.ts"; +import type { AttemptContainer, CreateAttemptContainerOptions } from "../src/isolation/docker.ts"; +import { buildRunManifest, type HarnessIdentity } from "../src/manifest.ts"; +import { executeRunV3 } from "../src/runner/execute.ts"; +import { ArtifactsStore } from "../src/store/artifacts.ts"; +import { ResultsStore } from "../src/store/db.ts"; + +const roots: string[] = []; +const stores: ResultsStore[] = []; +afterEach(() => { + for (const store of stores.splice(0)) store.close(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function temp() { const root = realpathSync(mkdtempSync(join(tmpdir(), "qs-files-"))); roots.push(root); return root; } +const harness: HarnessIdentity = { version: "test", revision: "test", dirty: false, source: { sha256: "a".repeat(64) } }; +const text = "# Harbor guide\r\nUnicode: 雪 🌊\n\n$(do-not-execute) `literal`\n"; +const destination = ".agents/skills/harbor/SKILL.md"; +const source = { id: "harbor-guide", version: "commit-abc123" }; +function files(content = text): WorkspaceFiles { + return { version: 1, files: [{ path: destination, encoding: "utf8", content, source }], absent: [] }; +} +function control(): WorkspaceFiles { return { version: 1, files: [], absent: [".agents/skills/harbor"] }; } +function recipe(id = "files", overlay: WorkspaceFiles | undefined = files()): Experiment { + return { id, agent: { adapter: "codex", model: "gpt-5.4-2026-03-05", reasoning: "high", cliVersion: "test-cli" }, + comparisonGroup: "files-study", treatment: id, + runtime: { kind: "container", image: "test-image", ...(overlay ? { workspaceFiles: overlay } : {}), + treatmentPolicyFields: ["workspaceFiles"], mcpServers: () => ({ guide: { url: "https://docs.example.test/mcp" } }), + } }; +} + +/** Retain the actual Codex login/command, config and parser; only provider I/O is removed. */ +function offlineCodex(): ContainerAgentAdapter { + const adapter = new CodexAdapter("codex", { model: "gpt-5.4-2026-03-05", reasoning: "high" }); + return { kind: "container", name: adapter.name, identity: adapter.identity, + requiredHostEnv: ["QUICKSTUDY_TEST_FILE_KEY"], setupFiles: adapter.setupFiles.bind(adapter), + command: adapter.command.bind(adapter), parseStream: adapter.parseStream.bind(adapter) }; +} +const stream = [ + { type: "thread.started", thread_id: "offline" }, + { type: "item.completed", item: { type: "agent_message", text: "Done" } }, + { type: "turn.completed", usage: { input_tokens: 12, output_tokens: 7 } }, +].map((event) => JSON.stringify(event)).join("\n"); + +type Faults = { + agentStream?: string; + contaminate?: (workspace: string) => void; + corrupt?: boolean; + setupMutation?: boolean; + afterSetup?: (workspace: string) => void; + writeFailure?: boolean; + teardownFailure?: boolean; + waitForAbort?: boolean; + onStage?: () => void; + noGit?: boolean; +}; +function rig(experiments = [recipe()], faults: Faults = {}) { + const root = temp(); + const dir = join(root, "evals", "task"); + mkdirSync(join(dir, "local"), { recursive: true }); + writeFileSync(join(dir, "local", "seed.txt"), "untouched\n"); + writeFileSync(join(dir, "PROMPT.md"), "---\nid: task\nsuite: benchmark\n---\n\nDo the task.\n"); + writeFileSync(join(dir, "EVAL.ts"), "export default async (ctx) => ({ passed: await ctx.fileExists('seed.txt'), checks: [] });\n"); + const evals = discoverEvals(join(root, "evals")); + const loaded = experiments.map((experiment) => { + const path = join(root, experiment.id + ".ts"); + writeFileSync(path, "export default {};\n"); // Dynamic payload deliberately absent from module source. + return { experiment, path }; + }); + const store = new ResultsStore(join(root, "results.db")); stores.push(store); + const artifacts = new ArtifactsStore(join(root, "results")); + const events: string[] = []; + const commands: string[][] = []; + const envs: Record[] = []; + const setupContents: Record = {}; + const staged: Buffer[] = []; + const baselineHadFile: boolean[] = []; + const createContainer = async (options: CreateAttemptContainerOptions): Promise => { + events.push("create"); envs.push(options.env); + const containerRoot = join(root, options.attemptId); + const workspace = join(containerRoot, "workspace"); + mkdirSync(workspace, { recursive: true }); + cpSync(options.fixtureDir, workspace, { recursive: true }); + faults.contaminate?.(workspace); + const map = (path: string) => path.startsWith("/workspace/") ? join(workspace, path.slice(11)) : path.startsWith("/") ? join(containerRoot, path.slice(1)) : join(workspace, path); + return { + id: options.attemptId, + execInWorkspace: async (cmd) => { + if (cmd[0] === "git") { + if (faults.noGit) return { exitCode: 1, stdout: "", stderr: "", timedOut: false }; + if (cmd.includes("commit")) { + events.push("baseline"); + baselineHadFile.push(existsSync(join(workspace, destination))); + } + const result = Bun.spawnSync(cmd, { cwd: workspace, stdout: "pipe", stderr: "pipe" }); + return { exitCode: result.exitCode, stdout: result.stdout.toString(), stderr: result.stderr.toString(), timedOut: false }; + } + events.push("agent"); commands.push(cmd); + return { exitCode: 0, stdout: faults.agentStream ?? stream, stderr: "", timedOut: false }; + }, + workspaceFiles: async (request, signal) => { + events.push(request.phase); + if (request.phase === "stage") faults.onStage?.(); + if (faults.waitForAbort) { + if (!signal.aborted) await new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true })); + events.push("staging-stopped"); + throw new Error("stopped"); + } + const proc = Bun.spawn([process.execPath, "-e", WORKSPACE_FILES_PROGRAM, workspace], { + stdin: Buffer.from(JSON.stringify({ ...request, setupPaths: request.setupPaths.map(map) })), stdout: "pipe", stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]); + if (exitCode !== 0) throw new Error(stderr); + const actual = JSON.parse(stdout); + if (request.phase === "stage" && request.declaration.files.length) staged.push(readFileSync(join(workspace, request.declaration.files[0]!.path))); + if (faults.corrupt && actual.files.length) actual.files[0].sha256 = "0".repeat(64); + return actual; + }, + writeFile: async (path, content) => { + events.push("setup"); setupContents[path] = content; + if (faults.writeFailure) throw new Error("DO-NOT-LOG-PAYLOAD"); + const target = map(path); mkdirSync(join(target, ".."), { recursive: true }); writeFileSync(target, content); + if (faults.setupMutation) writeFileSync(join(workspace, destination), "altered by setup"); + faults.afterSetup?.(workspace); + }, + exportWorkspace: async (to) => { events.push("export"); cpSync(workspace, to, { recursive: true }); }, + teardown: async () => { + events.push("teardown"); rmSync(containerRoot, { recursive: true, force: true }); + if (faults.teardownFailure) throw new Error("offline teardown failure"); + }, + }; + }; + const manifest = () => buildRunManifest({ evals, experiments: loaded, harness }); + const plan = loaded.map(({ experiment }) => ({ evalId: "task", experimentId: experiment.id, trialIndex: 0 })); + const run = (extra: Partial[0]> = {}) => executeRunV3({ + evals, experiments: loaded, plan, store, artifacts, manifest: extra.manifest ?? manifest(), createContainer, + containerAdapters: { codex: offlineCodex, oracle: () => new OracleContainerAdapter() }, log: () => {}, ...extra, + }); + return { root, dir, evals, loaded, store, artifacts, events, commands, envs, setupContents, staged, baselineHadFile, manifest, run }; +} + +// A made-up key, not a real provider secret. No default adapter/probe is invoked. +process.env.QUICKSTUDY_TEST_FILE_KEY = "fake-overlay-key-not-a-credential"; +afterAll(() => { delete process.env.QUICKSTUDY_TEST_FILE_KEY; }); + +describe("workspaceFiles public declaration", () => { + test("public helper snapshots, canonicalizes and deeply freezes exact bytes", () => { + const input = files(); + const resolved = defineWorkspaceFiles(input); + input.files[0]!.content = "changed"; + expect(Buffer.from(resolved.files[0]!.content, "base64").toString()).toBe(text); + expect(Object.isFrozen(resolved.files[0]!.source)).toBe(true); + expect(Object.isFrozen(defineExperiment(recipe()).runtime.workspaceFiles)).toBe(true); + }); + for (const path of ["", "/absolute", "../escape", "a/../b", "a/./b", "a//b", "a/", "C:/drive", "C:drive", "\\\\server\\file", "a\\b", "a:b", "a\u0000b", "a\nb", ".git/config", "x/.GIT/config", ".gitmodules", ".env", "private-key", "node_modules/file", ".quickstudy/file", "NUL.md", "a.", "x".repeat(241), Array(18).fill("a").join("/")]) { + test(`rejects unsafe path ${JSON.stringify(path)}`, () => { + const input = files(); input.files[0]!.path = path; + expect(() => defineWorkspaceFiles(input)).toThrow("workspaceFiles:"); + }); + } + test("rejects overlaps, invalid metadata, encodings, versions, and all size/count limits", () => { + const invalid: WorkspaceFiles[] = [ + { ...files(), files: [files().files[0]!, files().files[0]!] }, + { ...files(), absent: [".agents"] }, + { ...control(), absent: ["tree", "tree/child"] }, + { ...files(), version: 2 as 1 }, + files("\ud800"), + files("x".repeat(WORKSPACE_FILES_LIMITS.fileBytes + 1)), + { ...files(), files: Array(129).fill(files().files[0]) }, + { ...files(), files: Array.from({ length: 5 }, (_, i) => ({ ...files("x".repeat(WORKSPACE_FILES_LIMITS.fileBytes)).files[0]!, path: `file-${i}` })) }, + { ...control(), absent: Array.from({ length: 129 }, (_, i) => `file-${i}`) }, + { ...files(), files: [{ ...files().files[0]!, encoding: "base64", content: "YQ" }] }, + { ...files(), files: [{ ...files().files[0]!, encoding: "hex" as "utf8" }] }, + { ...files(), files: [{ ...files().files[0]!, source: { id: "/host/source", version: "v1" } }] }, + ]; + for (const input of invalid) expect(() => defineWorkspaceFiles(input)).toThrow("workspaceFiles:"); + }); + test("unsupported host/scripted overlays fail before container, provision or provider activity", async () => { + for (const runtime of [{ kind: "host" as const }, { kind: "container" as const, command: ["echo", "not-a-hook"] }]) { + const experiment = recipe(); Object.assign(experiment.runtime, runtime); + expect(() => defineExperiment(experiment)).toThrow("requires a container adapter"); + const r = rig([experiment]); + expect(() => r.run({ manifest: undefined })).toThrow("requires a container adapter"); + expect(r.events).toEqual([]); + } + }); +}); + +describe("production runner with offline containers", () => { + test("baseline -> stage -> normal Codex setup -> verification -> normal login/command, metrics, export and cleanup", async () => { + const r = rig(); const before = hashTree(join(r.dir, "local")); + const summary = await r.run(); + expect(summary).toMatchObject({ passed: 1, errors: 0 }); + expect(r.events).toEqual(["create", "baseline", "stage", "setup", "verify", "agent", "export", "teardown"]); + expect(r.staged[0]).toEqual(Buffer.from(text)); + expect(r.baselineHadFile).toEqual([false]); + expect(r.commands[0]).toEqual(offlineCodex().command({ runId: "", attemptId: "", prompt: r.evals[0]!.promptBody, trialIndex: 0, webPolicy: "native-web-blocked" })); + expect(r.commands[0]!.join(" ")).toContain("codex login --with-api-key"); + expect(r.setupContents[CODEX_CONFIG_PATH]).toContain('[mcp_servers.guide]'); + expect(r.envs[0]!.QUICKSTUDY_TEST_FILE_KEY).toBe(process.env.QUICKSTUDY_TEST_FILE_KEY!); + const attempt = r.store.listAttempts(summary.runId)[0]!; + expect(attempt).toMatchObject({ tokensIn: 12, tokensOut: 7, turns: 1, status: "completed" }); + expect(attempt.agentStartedAt).not.toBeNull(); + expect(hashTree(join(r.dir, "local"))).toEqual(before); + const evidence = JSON.parse(readFileSync(join(r.artifacts.paths(summary.runId, attempt.id).attemptDir, "workspace-files.json"), "utf8")); + expect(evidence.verified).toEqual(r.manifest().pairs![pairKey("task", "files")]!.workspaceFiles); + expect(evidence.verified.files[0].sha256).toBe(hashString(Buffer.from(text))); + expect(JSON.stringify(r.store.getRun(summary.runId))).not.toContain(text); + expect(JSON.stringify(evidence)).not.toContain(process.env.QUICKSTUDY_TEST_FILE_KEY!); + expect(readFileSync(attempt.transcriptRef!, "utf8")).not.toContain(process.env.QUICKSTUDY_TEST_FILE_KEY!); + }); + + test("overlay staging preserves the recorded preamble, check kinds and fetch-only docs evidence", async () => { + const url = "https://docs.example.test/guide"; + const experiment = recipe("files", files(`${text}\n${url}\n`)); + let preamble = " Read the local Harbor guide.\n"; + let resolutions = 0; + experiment.runtime.promptPreamble = () => { resolutions++; return preamble; }; + experiment.runtime.observationTargets = { docsHosts: ["docs.example.test"] }; + const agentStream = [stream, + JSON.stringify({ type: "item.completed", item: { id: "write", type: "command_execution", command: `printf '%s' '${url}' > README.md` } }), + JSON.stringify({ type: "item.completed", item: { id: "fetch", type: "command_execution", command: `curl '${url}'` } }), + ].join("\n"); + const r = rig([experiment], { agentStream }); + const checks = [ + { name: "sdk_seen", passed: true, graded: false, kind: "sdk" }, + { name: "wiring_seen", passed: false, graded: false, kind: "wiring" }, + ]; + writeFileSync(join(r.dir, "EVAL.ts"), `export default async () => (${JSON.stringify({ passed: true, checks })});\n`); + const manifest = r.manifest(); + const intended = manifest.pairs![pairKey("task", "files")]!; + expect(intended.promptPreamble).toBe("Read the local Harbor guide."); + preamble = "Changed after manifest resolution; must not be delivered."; + const prompt = `${intended.promptPreamble}\n\n${r.evals[0]!.promptBody}`; + const adapter = offlineCodex(); + const setupFiles = adapter.setupFiles; + adapter.setupFiles = (ctx) => { expect(ctx.prompt).toBe(prompt); return setupFiles(ctx); }; + const summary = await r.run({ manifest, containerAdapters: { codex: () => adapter } }); + expect(summary).toMatchObject({ passed: 1, errors: 0 }); + expect(resolutions).toBe(1); + expect(r.commands[0]).toContain(prompt); + expect(r.events).toEqual(["create", "baseline", "stage", "setup", "verify", "agent", "export", "teardown"]); + const attempt = r.store.listAttempts(summary.runId)[0]!; + expect(attempt.result?.checks).toEqual(checks); + expect(attempt.surfaceUsage?.docs).toEqual({ count: 1, urls: [url] }); + const evidence = JSON.parse(readFileSync(join(r.artifacts.paths(summary.runId, attempt.id).attemptDir, "workspace-files.json"), "utf8")); + expect(evidence.verified).toEqual(intended.workspaceFiles); + }); + + test("base64 preserves arbitrary bytes and empty files; sorted inventory agrees", async () => { + const bytes = Buffer.from([0, 255, 128, 10, 13, 0]); + const overlay: WorkspaceFiles = { version: 1, files: [ + { path: "z.bin", encoding: "base64", content: bytes.toString("base64"), source }, + { path: "a.md", encoding: "utf8", content: "", source }, + ], absent: ["unused"] }; + const r = rig([recipe("binary", overlay)]); + const summary = await r.run(); expect(summary.passed).toBe(1); + const attempt = r.store.listAttempts(summary.runId)[0]!; + const workspace = r.artifacts.paths(summary.runId, attempt.id).workspace; + expect(readFileSync(join(workspace, "z.bin"))).toEqual(bytes); + expect(readFileSync(join(workspace, "a.md"))).toEqual(Buffer.alloc(0)); + expect(r.manifest().pairs![pairKey("task", "binary")]!.workspaceFiles!.files.map((f) => f.path)).toEqual(["a.md", "z.bin"]); + }); + + for (const fault of ["fixture file", "fixture directory", "symlink ancestor", "symlink leaf", "file ancestor", "corrupt readback", "setup invalidation", "write failure", "missing git"] as const) { + test(`${fault} fails closed without agent execution`, async () => { + const faults: Faults = { + corrupt: fault === "corrupt readback", setupMutation: fault === "setup invalidation", writeFailure: fault === "write failure", noGit: fault === "missing git", + contaminate: (workspace) => { + if (fault === "symlink ancestor") symlinkSync("/nonexistent-outside", join(workspace, ".agents")); + if (fault === "file ancestor") writeFileSync(join(workspace, ".agents"), "occupied"); + if (["fixture file", "fixture directory", "symlink leaf"].includes(fault)) { + mkdirSync(join(workspace, ".agents/skills/harbor"), { recursive: true }); + if (fault === "fixture file") writeFileSync(join(workspace, destination), "occupied"); + if (fault === "fixture directory") mkdirSync(join(workspace, destination)); + if (fault === "symlink leaf") symlinkSync("/nonexistent-outside", join(workspace, destination)); + } + }, + }; + const r = rig([recipe()], faults); + const summary = await r.run(); + expect(summary.errors).toBe(1); expect(r.commands).toEqual([]); expect(r.events.at(-1)).toBe("teardown"); + const attempt = r.store.listAttempts(summary.runId)[0]!; + expect(attempt).toMatchObject({ status: "error", errorKind: "environment", agentStartedAt: null, result: null }); + expect(attempt.error).not.toContain("DO-NOT-LOG-PAYLOAD"); + expect(existsSync(join(r.artifacts.paths(summary.runId, attempt.id).attemptDir, "workspace-files.json"))).toBe(false); + }); + } + + test("runner preflight rejects unsafe/oversized/unknown-semantics declarations even with an injected manifest", async () => { + for (const invalid of [ + { ...files(), files: [{ ...files().files[0]!, path: "../escape" }] }, + files("x".repeat(WORKSPACE_FILES_LIMITS.fileBytes + 1)), + { ...files(), version: 2 as 1 }, + ]) { + const r = rig(); const manifest = r.manifest(); + r.loaded[0]!.experiment.runtime.workspaceFiles = invalid; + await expect(r.run({ manifest })).rejects.toThrow("workspaceFiles:"); + expect(r.events).toEqual([]); + } + }); + + test("final verification rejects symlink/hardlink/type/mode changes after adapter setup", async () => { + for (const change of ["symlink", "hardlink", "directory", "executable"]) { + const r = rig([recipe()], { afterSetup: (workspace) => { + const path = join(workspace, destination); + if (change === "hardlink") linkSync(path, join(workspace, "alias.md")); + else if (change === "executable") chmodSync(path, 0o755); + else { + rmSync(path); + if (change === "symlink") symlinkSync(join(workspace, "seed.txt"), path); + else mkdirSync(path); + } + } }); + expect((await r.run()).errors).toBe(1); + expect(r.events).toContain("verify"); expect(r.commands).toEqual([]); + } + }); + + test("adapter-owned destination/tree conflicts are rejected before staging", async () => { + for (const path of ["/workspace/" + destination, "/workspace/.agents", "/workspace/.agents/skills/harbor/SKILL.md/child"]) { + const r = rig(); const adapter = offlineCodex(); adapter.setupFiles = () => ({ [path]: "owned" }); + const summary = await r.run({ containerAdapters: { codex: () => adapter } }); + expect(summary.errors).toBe(1); expect(r.events).not.toContain("stage"); expect(r.commands).toEqual([]); + expect(r.store.listAttempts(summary.runId)[0]!.error).toContain("adapter setup collision"); + } + }); + + test("files then no-files control has no leakage; absence is checked twice and never deletes contamination", async () => { + const r = rig([recipe("files"), recipe("control", control())]); + // Separate runs make the order explicit, rather than relying on scheduler hashing. + await r.run({ plan: [{ evalId: "task", experimentId: "files", trialIndex: 0 }] }); + const summary = await r.run({ plan: [{ evalId: "task", experimentId: "control", trialIndex: 0 }] }); + expect(summary.passed).toBe(1); + const attempt = r.store.listAttempts(summary.runId)[0]!; + expect(existsSync(join(r.artifacts.paths(summary.runId, attempt.id).workspace, ".agents"))).toBe(false); + const evidence = JSON.parse(readFileSync(join(r.artifacts.paths(summary.runId, attempt.id).attemptDir, "workspace-files.json"), "utf8")); + expect(evidence.verified.absent).toEqual([".agents/skills/harbor"]); + for (const kind of ["file", "directory", "symlink"]) { + const contaminated = rig([recipe("control", control())], { contaminate: (workspace) => { + mkdirSync(join(workspace, ".agents/skills"), { recursive: true }); + const target = join(workspace, ".agents/skills/harbor"); + if (kind === "directory") mkdirSync(target); + else if (kind === "file") writeFileSync(target, "contaminated"); + else symlinkSync("/does-not-exist", target); + } }); + expect((await contaminated.run()).errors).toBe(1); expect(contaminated.commands).toEqual([]); + } + }); + + test("cancellation joins staging before teardown, preserves primary error alongside cleanup failure", async () => { + const controller = new AbortController(); + const r = rig([recipe()], { waitForAbort: true, onStage: () => setTimeout(() => controller.abort(), 10), teardownFailure: true }); + const summary = await r.run({ signal: controller.signal }); + expect(summary.errors).toBe(1); + expect(r.events.slice(-2)).toEqual(["staging-stopped", "teardown"]); expect(r.commands).toEqual([]); + const attempt = r.store.listAttempts(summary.runId)[0]!; + expect(attempt.errorKind).toBe("environment"); expect(attempt.agentStartedAt).toBeNull(); + expect(attempt.error).toContain("staging cancelled or timed out"); expect(attempt.error).toContain("container teardown failed"); + }); + + test("fixed staging deadline aborts and joins before cleanup, never starts the agent", async () => { + const r = rig([recipe()], { waitForAbort: true }); + const start = Date.now(); const summary = await r.run(); + expect(Date.now() - start).toBeGreaterThanOrEqual(WORKSPACE_FILES_LIMITS.timeoutMs); + expect(summary.errors).toBe(1); expect(r.commands).toEqual([]); + expect(r.events.slice(-2)).toEqual(["staging-stopped", "teardown"]); + expect(r.store.listAttempts(summary.runId)[0]!.agentStartedAt).toBeNull(); + }, 40000); + + test("class runtime capabilities keep their original receiver with workspace files", async () => { + const calls: string[] = []; + class GuideRuntime { + readonly kind = "container" as const; + readonly workspaceFiles = files(); + #image = "test-image"; + containerImage() { calls.push("image"); return this.#image; } + promptPreamble() { return `Guide for ${this.#image}.`; } + mcpServers() { return { guide: { url: `https://${this.#image}.example.test/mcp` } }; } + async provision() { + expect(runtime).toBe(this); + expect(this.#image).toBe("test-image"); + calls.push("provision"); + return {}; + } + scorerCapabilities() { + expect(runtime).toBe(this); + calls.push("scorer"); + return { query: async () => this.#image }; + } + async teardown() { + expect(runtime).toBe(this); + expect(this.#image).toBe("test-image"); + calls.push("teardown"); + } + } + const runtime = new GuideRuntime(); + const experiment = defineExperiment({ ...recipe(), runtime }); + const r = rig([experiment]); + writeFileSync(join(r.dir, "EVAL.ts"), `export default async (ctx) => ({ passed: await ctx.query({}) === 'test-image', checks: [] });\n`); + expect((await r.run()).passed).toBe(1); + expect(calls).toContain("image"); + expect(calls.slice(-3)).toEqual(["provision", "scorer", "teardown"]); + expect(r.commands[0]).toContain(`Guide for test-image.\n\n${r.evals[0]!.promptBody}`); + expect(r.setupContents[CODEX_CONFIG_PATH]).toContain("https://test-image.example.test/mcp"); + expect(r.staged[0]).toEqual(Buffer.from(text)); + expect(experiment.runtime).toBe(runtime); + expect(Object.isFrozen(runtime)).toBe(true); + expect(runtime.workspaceFiles.files[0]!.encoding).toBe("utf8"); + }); + + test("snapshot is bound before awaits; caller mutation cannot substitute payload bytes", async () => { + const input = files(); const experiment = recipe("files", input); + const r = rig([experiment], { onStage: () => { input.files[0]!.content = "mutated after identity"; } }); + expect((await r.run()).passed).toBe(1); expect(r.staged[0]).toEqual(Buffer.from(text)); + }); + + test("no-overlay normal, scripted, host and oracle paths remain supported", async () => { + const normal = recipe(); delete normal.runtime.workspaceFiles; + const scripted = recipe("scripted"); delete scripted.runtime.workspaceFiles; scripted.runtime.command = ["echo", "scripted"]; + const host: Experiment = { id: "host", agent: { adapter: "echo" }, runtime: { kind: "host" } }; + const r = rig([normal, scripted, host]); + expect((await r.run()).passed).toBe(3); expect(r.events).not.toContain("stage"); + const oracle = recipe("oracle"); oracle.agent = { adapter: "oracle" }; + const o = rig([oracle]); + mkdirSync(join(o.dir, "solution")); writeFileSync(join(o.dir, "solution", "answer.md"), "reference"); + o.evals[0]!.solutionDir = join(o.dir, "solution"); + expect((await o.run()).passed).toBe(1); + expect(o.events.indexOf("baseline")).toBeLessThan(o.events.indexOf("stage")); + }); +}); + +describe("generated identity, resume and actual group comparison", () => { + test("dynamic bytes, path, provenance and absence change identity; encoding aliases do not", () => { + const r = rig(); const baseline = r.manifest(); + for (const mutate of [ + (f: WorkspaceFiles) => { f.files[0]!.content += "!"; }, + (f: WorkspaceFiles) => { f.files[0]!.path += ".md"; }, + (f: WorkspaceFiles) => { f.files[0]!.source = { ...source, version: "commit-def456" }; }, + (f: WorkspaceFiles) => { (f.absent as string[]).push("unused"); }, + ]) { + const input = files(); mutate(input); r.loaded[0]!.experiment.runtime.workspaceFiles = input; + const changed = r.manifest(); expect(changed.hash).not.toBe(baseline.hash); + expect(changed.experiments.files!.hash).not.toBe(baseline.experiments.files!.hash); + } + const encoded = files(); encoded.files[0]!.encoding = "base64"; encoded.files[0]!.content = Buffer.from(text).toString("base64"); + r.loaded[0]!.experiment.runtime.workspaceFiles = encoded; + expect(r.manifest().hash).toBe(baseline.hash); + r.loaded[0]!.experiment.runtime.workspaceFiles = { ...files(), version: 2 as 1 }; + expect(() => r.manifest()).toThrow("unsupported staging version"); + expect(JSON.stringify(baseline)).not.toContain(r.root); + expect(JSON.stringify(baseline)).not.toContain(Buffer.from(text).toString("base64")); + }); + + test("manifest and pair resolution use captured files even if the runtime declaration changes", () => { + for (const offered of [files(), undefined]) { + const experiment = recipe(); + experiment.runtime.workspaceFiles = offered; + const r = rig([experiment]); + const expected = r.manifest(); + const workspaceFilesByExperiment = new Map([[experiment.id, offered ? defineWorkspaceFiles(offered) : undefined]]); + experiment.runtime.workspaceFiles = files("changed after capture"); + const options = { evals: r.evals, experiments: r.loaded, harness, workspaceFilesByExperiment }; + expect(resolvePairEnvironments(options)).toEqual(expected.pairs!); + expect(buildRunManifest(options)).toEqual(expected); + } + }); + + test("stale payload manifests and stale resume are rejected; unchanged resume executes nothing", async () => { + const r = rig(); const oldManifest = r.manifest(); const first = await r.run(); + const count = r.commands.length; + expect((await r.run({ resumeRunId: first.runId })).passed).toBe(1); expect(r.commands.length).toBe(count); + r.loaded[0]!.experiment.runtime.workspaceFiles = files("new dynamic bytes"); + await expect(r.run({ manifest: oldManifest })).rejects.toThrow("manifest payload identity mismatch"); + await expect(r.run({ resumeRunId: first.runId })).rejects.toThrow("resume refused"); + expect(r.commands.length).toBe(count); + }); + + test("only mutually declared file availability varies; other config/policy/agent/image/budget differences withhold", () => { + const r = rig([recipe("files"), recipe("control", control())]); + const experiments = r.loaded.map((x) => x.experiment); + const compare = () => compareGroups(experiments, r.manifest())[0]!; + expect(compare().comparison).toBe("allowed"); + expect(r.manifest().experiments.files!.hash).not.toBe(r.manifest().experiments.control!.hash); + const arm = experiments[1]!; + for (const exemptions of [[], ["workspaceFiles"]] as const) { + experiments[0]!.runtime.treatmentPolicyFields = [...exemptions]; arm.runtime.treatmentPolicyFields = []; + expect(compare().comparison).toBe("withheld"); + } + experiments[0]!.runtime.treatmentPolicyFields = ["workspaceFiles"]; arm.runtime.treatmentPolicyFields = ["workspaceFiles"]; + const changes: Array<(e: Experiment) => void> = [ + (e) => { e.agent.model = "different-model"; }, (e) => { e.agent.cliVersion = "different-cli"; }, + (e) => { e.runtime.image = "different-image"; }, (e) => { e.runtime.config = { unrelated: true }; }, + (e) => { e.runtime.egressHosts = () => ["different.example.test"]; }, + (e) => { e.runtime.webPolicy = () => "native-web-allowed"; }, + (e) => { e.runtime.pathTreatment = () => "/opt/different"; }, + (e) => { e.runtime.promptPreamble = () => "A different treatment hint."; }, + ]; + for (const change of changes) { + const original = { ...arm, agent: { ...arm.agent }, runtime: { ...arm.runtime } }; + change(arm); expect(compare().comparison).toBe("withheld"); Object.assign(arm, original); + } + const pairs = resolvePairEnvironments({ evals: r.evals, experiments: r.loaded, mcpHashes: { files: "a", control: "b" } }); + const manifest = buildRunManifest({ evals: r.evals, experiments: r.loaded, harness, pairs }); + expect(compareGroups(experiments, manifest)[0]!.comparison).toBe("withheld"); + const budgetPairs = Object.assign({}, ...r.loaded.map((loaded, i) => resolvePairEnvironments({ + evals: r.evals, experiments: [loaded], budgets: { agentMs: 1000 + i }, + }))); + const budgets = buildRunManifest({ evals: r.evals, experiments: r.loaded, harness, pairs: budgetPairs }); + expect(compareGroups(experiments, budgets)[0]!.comparison).toBe("withheld"); + arm.runtime.command = ["echo", "different-command"]; + expect(() => r.manifest()).toThrow("requires a container adapter"); + }); +});