diff --git a/docs/extensions.md b/docs/extensions.md index 97c9bd7..cf3998a 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -110,8 +110,11 @@ native web tools do not prohibit explicit shell paths or package-local alternati `run --concurrency N --seed VALUE` uses a reproducible seeded dispatch order; default concurrency is 1 and seed is `0`. Trial coordinates never depend on completion order. -Runtime `capacity: {key, limit}` shares a capacity bound across arms. CPU, memory, -Docker sidecars and provider limits determine a suitable concurrency. +Runtime `capacity: {key, limit}` shares a capacity bound across arms. A resolver, +`capacity: (metadata) => {key, limit} | undefined`, decides per eval, so an eval +that consumes no pooled resource answers `undefined` and never waits for a slot; +unlike the static form, a resolver is not part of the experiment's identity. CPU, +memory, Docker sidecars and provider limits determine a suitable concurrency. `run --resume RUN_ID` inherits the recorded selection, roots, budgets, seed and policy and verifies v4 identity. Completed failures are skipped exactly like passes. diff --git a/src/experiments/types.ts b/src/experiments/types.ts index fc02288..d8a443e 100644 --- a/src/experiments/types.ts +++ b/src/experiments/types.ts @@ -6,9 +6,16 @@ * manifest), so nothing may mutate it after definition. */ +import type { EvalMetadata } from "../evals/types.ts"; import type { SurfaceUsageConfig } from "../surface-usage.ts"; import type { RuntimeCapabilities } from "../runtime/types.ts"; +/** A shared scarce resource: the scheduler lets at most `limit` attempts hold `key` at once. */ +export interface RuntimeCapacity { + key: string; + limit: number; +} + /** Which agent executes the attempt, plus the identity fields recorded in the manifest. */ export interface AgentSpec { /** Adapter name, e.g. "echo" (the built-in no-op pipeline smoke agent). */ @@ -35,8 +42,13 @@ export interface Runtime extends RuntimeCapabilities { kind: "host" | "container"; /** Shared observation targets, independent of which surfaces this arm offers. */ observationTargets?: SurfaceUsageConfig; - /** Shared scarce resource pool; scheduler waits instead of starting doomed jobs. */ - capacity?: { key: string; limit: number }; + /** + * Shared scarce resource pool; the scheduler waits instead of starting + * doomed jobs. A resolver decides per eval, so an eval that consumes no + * pooled resource can answer `undefined` and never hold a slot. The static + * form is identity; a resolver, like every capability function, is not. + */ + capacity?: RuntimeCapacity | ((metadata: EvalMetadata) => RuntimeCapacity | undefined); /** Explicit environment fields the declared treatment may vary in a comparison. * Images and lifecycle budgets can never be exempted. All group members must * declare the same fields. Source/config identity still changes per attempt. diff --git a/src/index.ts b/src/index.ts index 3bd28d7..eac4cdf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,7 +18,7 @@ export type { } from "./evals/types.ts"; export { EVAL_SUITES } from "./evals/types.ts"; export { defineExperiment } from "./experiments/types.ts"; -export type { AgentSpec, Experiment, LoadedExperiment, Runtime } from "./experiments/types.ts"; +export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts"; export type { ProvisionedEnvironment, RuntimeCapabilities, diff --git a/src/manifest.ts b/src/manifest.ts index 19c3ed4..f2dcc49 100644 --- a/src/manifest.ts +++ b/src/manifest.ts @@ -190,7 +190,8 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest }, runtime: contentHash({ kind: experiment.runtime.kind, - capacity: experiment.runtime.capacity ?? null, + // A capacity resolver is a capability function: per-eval, never identity. + capacity: typeof experiment.runtime.capacity === "function" ? null : (experiment.runtime.capacity ?? null), observationTargets: experiment.runtime.observationTargets ?? null, treatmentPolicyFields: [...(experiment.runtime.treatmentPolicyFields ?? [])].sort(), image: experiment.runtime.image ?? null, diff --git a/src/runner/capacity.ts b/src/runner/capacity.ts new file mode 100644 index 0000000..add9184 --- /dev/null +++ b/src/runner/capacity.ts @@ -0,0 +1,17 @@ +import type { EvalMetadata } from "../evals/types.ts"; +import type { Runtime, RuntimeCapacity } from "../experiments/types.ts"; + +/** + * The capacity an attempt of `metadata` holds under `runtime`: the static + * pool, the resolver's answer for that eval, or none. The scheduler validates + * the shape of whatever comes back. + */ +export function resolveRuntimeCapacity( + runtime: Runtime | undefined, + metadata: EvalMetadata | undefined, +): RuntimeCapacity | undefined { + const capacity = runtime?.capacity; + if (capacity === undefined) return undefined; + if (typeof capacity === "function") return metadata === undefined ? undefined : capacity(metadata); + return capacity; +} diff --git a/src/runner/execute.ts b/src/runner/execute.ts index c35dfe6..ea509a5 100644 --- a/src/runner/execute.ts +++ b/src/runner/execute.ts @@ -36,6 +36,7 @@ import { EchoAdapter } from "../adapters/echo.ts"; import { OracleAdapter, OracleContainerAdapter } from "../adapters/oracle.ts"; import { validEvalResult } from "../evals/result.ts"; import { importEvalScorer } from "../evals/discovery.ts"; +import { resolveRuntimeCapacity } from "./capacity.ts"; import { deadline, StageTimeoutError } from "./deadline.ts"; import { scoreInWorker } from "./score-worker.ts"; import { prepWorkspaceSource } from "./workspace-source.ts"; @@ -616,7 +617,10 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise experimentsById.get(entry.experimentId)?.runtime.capacity }); + }, { + ...(options.signal ? { signal: options.signal } : {}), + capacity: (entry) => resolveRuntimeCapacity(experimentsById.get(entry.experimentId)?.runtime, evalsById.get(entry.evalId)?.metadata), + }); const current = activeAttempts(store.listAttempts(runId)); summary.passed = current.filter((a) => a.status === "completed" && a.result?.passed).length; diff --git a/tests/capacity.test.ts b/tests/capacity.test.ts new file mode 100644 index 0000000..366b3f7 --- /dev/null +++ b/tests/capacity.test.ts @@ -0,0 +1,102 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { EvalMetadata } from "../src/evals/types.ts"; +import type { Runtime } from "../src/experiments/types.ts"; +import { buildRunManifest, type HarnessIdentity } from "../src/manifest.ts"; +import { resolveRuntimeCapacity } from "../src/runner/capacity.ts"; +import { schedule } from "../src/runner/scheduler.ts"; + +const pooled: EvalMetadata = { id: "authkit-nextjs", suite: "benchmark", topics: ["authentication"] }; +const accountless: EvalMetadata = { id: "authkit-nocreds-nextjs", suite: "benchmark", topics: ["no-credentials"] }; + +test("a static capacity applies to every eval", () => { + const runtime: Runtime = { kind: "container", image: "x:1", capacity: { key: "pool", limit: 2 } }; + expect(resolveRuntimeCapacity(runtime, pooled)).toEqual({ key: "pool", limit: 2 }); + expect(resolveRuntimeCapacity(runtime, accountless)).toEqual({ key: "pool", limit: 2 }); +}); + +test("a resolver decides per eval and can exempt one", () => { + const seen: string[] = []; + const runtime: Runtime = { + kind: "container", + image: "x:1", + capacity: (metadata) => { + seen.push(metadata.id); + return metadata.topics?.includes("no-credentials") ? undefined : { key: "pool", limit: 2 }; + }, + }; + expect(resolveRuntimeCapacity(runtime, pooled)).toEqual({ key: "pool", limit: 2 }); + expect(resolveRuntimeCapacity(runtime, accountless)).toBeUndefined(); + expect(seen).toEqual(["authkit-nextjs", "authkit-nocreds-nextjs"]); +}); + +test("no runtime, no capacity, or no eval to ask about means no slot", () => { + expect(resolveRuntimeCapacity(undefined, pooled)).toBeUndefined(); + expect(resolveRuntimeCapacity({ kind: "host" }, pooled)).toBeUndefined(); + expect(resolveRuntimeCapacity({ kind: "host", capacity: () => ({ key: "pool", limit: 1 }) }, undefined)).toBeUndefined(); +}); + +test("a static capacity enters the experiment's identity; a resolver, like other capability functions, does not", () => { + const dir = mkdtempSync(join(tmpdir(), "qs-capacity-identity-")); + const path = join(dir, "x.ts"); + writeFileSync(path, "export default {};\n"); + const harness: HarnessIdentity = { version: "test", revision: "test", dirty: false, source: { sha256: "0".repeat(64) } }; + const hashOf = (runtime: Runtime): string => + buildRunManifest({ + evals: [], + experiments: [{ experiment: { id: "x", agent: { adapter: "echo", model: "echo-v1" }, runtime }, path }], + harness, + }).experiments["x"]!.components.runtime; + const none = hashOf({ kind: "host" }); + const fixed = hashOf({ kind: "host", capacity: { key: "pool", limit: 2 } }); + const resolved = hashOf({ kind: "host", capacity: () => ({ key: "pool", limit: 2 }) }); + expect(fixed).not.toBe(none); + expect(resolved).toBe(none); +}); + +test("through the scheduler: an exempt eval starts while the pool is full, and pooled attempts keep the limit", async () => { + const runtime: Runtime = { + kind: "container", + image: "x:1", + capacity: (metadata) => (metadata.topics?.includes("no-credentials") ? undefined : { key: "pool", limit: 1 }), + }; + const metadataById: Record = { "authkit-nextjs": pooled, "authkit-nocreds-nextjs": accountless }; + // Two pooled attempts and one exempt one; the first pooled attempt holds the only slot until released. + const plan = [ + { evalId: "authkit-nextjs", experimentId: "x", trialIndex: 0 }, + { evalId: "authkit-nextjs", experimentId: "x", trialIndex: 1 }, + { evalId: "authkit-nocreds-nextjs", experimentId: "x", trialIndex: 0 }, + ]; + const started: string[] = []; + let release = (): void => {}; + const held = new Promise((resolve) => { + release = resolve; + }); + let activePooled = 0; + let peakPooled = 0; + const run = schedule( + plan, + 3, + async (entry) => { + started.push(`${entry.evalId}#${entry.trialIndex}`); + const pooledEntry = entry.evalId === "authkit-nextjs"; + if (pooledEntry) { + activePooled += 1; + peakPooled = Math.max(peakPooled, activePooled); + } + if (entry.trialIndex === 0 && pooledEntry) await held; + else await Bun.sleep(5); + if (pooledEntry) activePooled -= 1; + }, + { capacity: (entry) => resolveRuntimeCapacity(runtime, metadataById[entry.evalId]) }, + ); + await Bun.sleep(25); + // The exempt attempt is running beside the slot holder; the second pooled attempt waits. + expect(started).toEqual(["authkit-nextjs#0", "authkit-nocreds-nextjs#0"]); + release(); + await run; + expect(started).toEqual(["authkit-nextjs#0", "authkit-nocreds-nextjs#0", "authkit-nextjs#1"]); + expect(peakPooled).toBe(1); +});