feat(runtime): resolve capacity per eval

docs/extensions.md

@@ -110,8 +110,11 @@ native web tools do not prohibit explicit shell paths or package-local alternati
 
 `run --concurrency N --seed VALUE` uses a reproducible seeded dispatch order; default
 concurrency is 1 and seed is `0`. Trial coordinates never depend on completion order.
-Runtime `capacity: {key, limit}` shares a capacity bound across arms. CPU, memory,
-Docker sidecars and provider limits determine a suitable concurrency.
+Runtime `capacity: {key, limit}` shares a capacity bound across arms. A resolver,
+`capacity: (metadata) => {key, limit} | undefined`, decides per eval, so an eval
+that consumes no pooled resource answers `undefined` and never waits for a slot;
+unlike the static form, a resolver is not part of the experiment's identity. CPU,
+memory, Docker sidecars and provider limits determine a suitable concurrency.
 
 `run --resume RUN_ID` inherits the recorded selection, roots, budgets, seed and
 policy and verifies v4 identity. Completed failures are skipped exactly like passes.

src/experiments/types.ts

@@ -6,9 +6,16 @@
  * manifest), so nothing may mutate it after definition.
  */
 
+import type { EvalMetadata } from "../evals/types.ts";
 import type { SurfaceUsageConfig } from "../surface-usage.ts";
 import type { RuntimeCapabilities } from "../runtime/types.ts";
 
+/** A shared scarce resource: the scheduler lets at most `limit` attempts hold `key` at once. */
+export interface RuntimeCapacity {
+  key: string;
+  limit: number;
+}
+
 /** Which agent executes the attempt, plus the identity fields recorded in the manifest. */
 export interface AgentSpec {
   /** Adapter name, e.g. "echo" (the built-in no-op pipeline smoke agent). */
@@ -35,8 +42,13 @@ export interface Runtime extends RuntimeCapabilities {
   kind: "host" | "container";
   /** Shared observation targets, independent of which surfaces this arm offers. */
   observationTargets?: SurfaceUsageConfig;
-  /** Shared scarce resource pool; scheduler waits instead of starting doomed jobs. */
-  capacity?: { key: string; limit: number };
+  /**
+   * Shared scarce resource pool; the scheduler waits instead of starting
+   * doomed jobs. A resolver decides per eval, so an eval that consumes no
+   * pooled resource can answer `undefined` and never hold a slot. The static
+   * form is identity; a resolver, like every capability function, is not.
+   */
+  capacity?: RuntimeCapacity | ((metadata: EvalMetadata) => RuntimeCapacity | undefined);
   /** Explicit environment fields the declared treatment may vary in a comparison.
    * Images and lifecycle budgets can never be exempted. All group members must
    * declare the same fields. Source/config identity still changes per attempt.

src/index.ts

@@ -18,7 +18,7 @@ export type {
 } from "./evals/types.ts";
 export { EVAL_SUITES } from "./evals/types.ts";
 export { defineExperiment } from "./experiments/types.ts";
-export type { AgentSpec, Experiment, LoadedExperiment, Runtime } from "./experiments/types.ts";
+export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts";
 export type {
   ProvisionedEnvironment,
   RuntimeCapabilities,

src/manifest.ts

@@ -190,7 +190,8 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
       },
       runtime: contentHash({
         kind: experiment.runtime.kind,
-        capacity: experiment.runtime.capacity ?? null,
+        // A capacity resolver is a capability function: per-eval, never identity.
+        capacity: typeof experiment.runtime.capacity === "function" ? null : (experiment.runtime.capacity ?? null),
         observationTargets: experiment.runtime.observationTargets ?? null,
         treatmentPolicyFields: [...(experiment.runtime.treatmentPolicyFields ?? [])].sort(),
         image: experiment.runtime.image ?? null,

src/runner/capacity.ts

@@ -0,0 +1,17 @@
+import type { EvalMetadata } from "../evals/types.ts";
+import type { Runtime, RuntimeCapacity } from "../experiments/types.ts";
+
+/**
+ * The capacity an attempt of `metadata` holds under `runtime`: the static
+ * pool, the resolver's answer for that eval, or none. The scheduler validates
+ * the shape of whatever comes back.
+ */
+export function resolveRuntimeCapacity(
+  runtime: Runtime | undefined,
+  metadata: EvalMetadata | undefined,
+): RuntimeCapacity | undefined {
+  const capacity = runtime?.capacity;
+  if (capacity === undefined) return undefined;
+  if (typeof capacity === "function") return metadata === undefined ? undefined : capacity(metadata);
+  return capacity;
+}

src/runner/execute.ts

@@ -36,6 +36,7 @@ import { EchoAdapter } from "../adapters/echo.ts";
 import { OracleAdapter, OracleContainerAdapter } from "../adapters/oracle.ts";
 import { validEvalResult } from "../evals/result.ts";
 import { importEvalScorer } from "../evals/discovery.ts";
+import { resolveRuntimeCapacity } from "./capacity.ts";
 import { deadline, StageTimeoutError } from "./deadline.ts";
 import { scoreInWorker } from "./score-worker.ts";
 import { prepWorkspaceSource } from "./workspace-source.ts";
@@ -616,7 +617,10 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
         `attempt ${attemptId} (${entry.evalId} × ${entry.experimentId} trial ${entry.trialIndex + 1}): ` +
           `${disposition.status}${disposition.result ? ` — ${disposition.result.passed ? "passed" : "failed"}` : ""}`,
       );
-    }, { ...(options.signal ? { signal: options.signal } : {}), capacity: (entry) => experimentsById.get(entry.experimentId)?.runtime.capacity });
+    }, {
+      ...(options.signal ? { signal: options.signal } : {}),
+      capacity: (entry) => resolveRuntimeCapacity(experimentsById.get(entry.experimentId)?.runtime, evalsById.get(entry.evalId)?.metadata),
+    });
 
     const current = activeAttempts(store.listAttempts(runId));
     summary.passed = current.filter((a) => a.status === "completed" && a.result?.passed).length;

tests/capacity.test.ts

@@ -0,0 +1,102 @@
+import { expect, test } from "bun:test";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { EvalMetadata } from "../src/evals/types.ts";
+import type { Runtime } from "../src/experiments/types.ts";
+import { buildRunManifest, type HarnessIdentity } from "../src/manifest.ts";
+import { resolveRuntimeCapacity } from "../src/runner/capacity.ts";
+import { schedule } from "../src/runner/scheduler.ts";
+
+const pooled: EvalMetadata = { id: "authkit-nextjs", suite: "benchmark", topics: ["authentication"] };
+const accountless: EvalMetadata = { id: "authkit-nocreds-nextjs", suite: "benchmark", topics: ["no-credentials"] };
+
+test("a static capacity applies to every eval", () => {
+  const runtime: Runtime = { kind: "container", image: "x:1", capacity: { key: "pool", limit: 2 } };
+  expect(resolveRuntimeCapacity(runtime, pooled)).toEqual({ key: "pool", limit: 2 });
+  expect(resolveRuntimeCapacity(runtime, accountless)).toEqual({ key: "pool", limit: 2 });
+});
+
+test("a resolver decides per eval and can exempt one", () => {
+  const seen: string[] = [];
+  const runtime: Runtime = {
+    kind: "container",
+    image: "x:1",
+    capacity: (metadata) => {
+      seen.push(metadata.id);
+      return metadata.topics?.includes("no-credentials") ? undefined : { key: "pool", limit: 2 };
+    },
+  };
+  expect(resolveRuntimeCapacity(runtime, pooled)).toEqual({ key: "pool", limit: 2 });
+  expect(resolveRuntimeCapacity(runtime, accountless)).toBeUndefined();
+  expect(seen).toEqual(["authkit-nextjs", "authkit-nocreds-nextjs"]);
+});
+
+test("no runtime, no capacity, or no eval to ask about means no slot", () => {
+  expect(resolveRuntimeCapacity(undefined, pooled)).toBeUndefined();
+  expect(resolveRuntimeCapacity({ kind: "host" }, pooled)).toBeUndefined();
+  expect(resolveRuntimeCapacity({ kind: "host", capacity: () => ({ key: "pool", limit: 1 }) }, undefined)).toBeUndefined();
+});
+
+test("a static capacity enters the experiment's identity; a resolver, like other capability functions, does not", () => {
+  const dir = mkdtempSync(join(tmpdir(), "qs-capacity-identity-"));
+  const path = join(dir, "x.ts");
+  writeFileSync(path, "export default {};\n");
+  const harness: HarnessIdentity = { version: "test", revision: "test", dirty: false, source: { sha256: "0".repeat(64) } };
+  const hashOf = (runtime: Runtime): string =>
+    buildRunManifest({
+      evals: [],
+      experiments: [{ experiment: { id: "x", agent: { adapter: "echo", model: "echo-v1" }, runtime }, path }],
+      harness,
+    }).experiments["x"]!.components.runtime;
+  const none = hashOf({ kind: "host" });
+  const fixed = hashOf({ kind: "host", capacity: { key: "pool", limit: 2 } });
+  const resolved = hashOf({ kind: "host", capacity: () => ({ key: "pool", limit: 2 }) });
+  expect(fixed).not.toBe(none);
+  expect(resolved).toBe(none);
+});
+
+test("through the scheduler: an exempt eval starts while the pool is full, and pooled attempts keep the limit", async () => {
+  const runtime: Runtime = {
+    kind: "container",
+    image: "x:1",
+    capacity: (metadata) => (metadata.topics?.includes("no-credentials") ? undefined : { key: "pool", limit: 1 }),
+  };
+  const metadataById: Record<string, EvalMetadata> = { "authkit-nextjs": pooled, "authkit-nocreds-nextjs": accountless };
+  // Two pooled attempts and one exempt one; the first pooled attempt holds the only slot until released.
+  const plan = [
+    { evalId: "authkit-nextjs", experimentId: "x", trialIndex: 0 },
+    { evalId: "authkit-nextjs", experimentId: "x", trialIndex: 1 },
+    { evalId: "authkit-nocreds-nextjs", experimentId: "x", trialIndex: 0 },
+  ];
+  const started: string[] = [];
+  let release = (): void => {};
+  const held = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  let activePooled = 0;
+  let peakPooled = 0;
+  const run = schedule(
+    plan,
+    3,
+    async (entry) => {
+      started.push(`${entry.evalId}#${entry.trialIndex}`);
+      const pooledEntry = entry.evalId === "authkit-nextjs";
+      if (pooledEntry) {
+        activePooled += 1;
+        peakPooled = Math.max(peakPooled, activePooled);
+      }
+      if (entry.trialIndex === 0 && pooledEntry) await held;
+      else await Bun.sleep(5);
+      if (pooledEntry) activePooled -= 1;
+    },
+    { capacity: (entry) => resolveRuntimeCapacity(runtime, metadataById[entry.evalId]) },
+  );
+  await Bun.sleep(25);
+  // The exempt attempt is running beside the slot holder; the second pooled attempt waits.
+  expect(started).toEqual(["authkit-nextjs#0", "authkit-nocreds-nextjs#0"]);
+  release();
+  await run;
+  expect(started).toEqual(["authkit-nextjs#0", "authkit-nocreds-nextjs#0", "authkit-nextjs#1"]);
+  expect(peakPooled).toBe(1);
+});