diff --git a/api-surface.json b/api-surface.json index ff7a54d4..b86d3584 100644 --- a/api-surface.json +++ b/api-surface.json @@ -925,7 +925,7 @@ "ExecutorExecutionBinding": "type cbd52e7e2eb8", "ExecutorFactory": "type 0d6f475ad3d4", "ExecutorMaterialization": "type ceafe44da26b", - "ExecutorNodeContext": "type 7f86daa88edc", + "ExecutorNodeContext": "type 17a7348871c5", "ExecutorProgress": "type c91983468166", "ExecutorProgressEvent": "type 19b8d5b7224e", "ExecutorRegistry": "type 2abca0065370", @@ -1306,7 +1306,7 @@ "UntrackedCopyStats": "type 3d60a38b88e2", "UsageEvent": "type 110014e66fac", "VERIFY_TAIL_CHARS": "value 95999f4bd438", - "ValidationCtx": "type 45da43f45d39", + "ValidationCtx": "type 4905637394bb", "Validator": "type c67346b2cb69", "VerifierEnvironmentOptions": "type 8381e1b6dc2e", "Verify": "type 4e1beeeba7ee", diff --git a/docs/agent-managed-compute/reliability.md b/docs/agent-managed-compute/reliability.md index a05246ac..a4ff09a9 100644 --- a/docs/agent-managed-compute/reliability.md +++ b/docs/agent-managed-compute/reliability.md @@ -231,6 +231,14 @@ Omit `coordination.port` to allocate a separate port for each concurrent manager Runtime does not provision a proxy or tunnel. Remote public endpoints require HTTPS. +Before provider admission, Runtime checks each configured public endpoint with authenticated `initialize` and `tools/list` requests. +The returned grants must match that manager's exact tool names. +The check takes at most 10 seconds, or `coordination.requestTimeoutMs` when shorter, and stops on manager cancellation. +Responses are limited to 1 MiB independently of the incoming request limit, and redirects are refused. +Failure closes the listener and reports a credential-free cause before inference starts. +This verifies the operator's public route; it does not establish reachability from the provider's network. +Omitting `coordination.publicUrl` preserves local-only startup without this network check. + For same-host coordinator restart, configure `authentication.signingKeys` with an active key ID and a secret key map. Keep the public URL, run ID, actor ID, tool grants, and verification key stable until the retained credential expires. Stable keys support resumed coordination within the original scope deadline and any explicit credential expiry. diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 1303d036..f82074ff 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -4186,6 +4186,8 @@ OPT-IN executable score for this worker, with the SAME contract the sandbox seam has: `validate` runs while the environment is still alive, so `ValidationCtx.box` can read files and run commands in the environment it is scoring. Every other supervised hook fires after teardown and can only read the artifact. +`ValidationCtx.node` identifies the supervised node, including its recursion depth, so a +shared validator can apply a root-only contract without applying it to nested managers. The verdict becomes the settled artifact's verdict. Absent, nothing changes and the leaf falls back to its own settle verdict. @@ -17717,6 +17719,8 @@ OPT-IN executable score for this worker, with the SAME contract the sandbox seam has: `validate` runs while the environment is still alive, so `ValidationCtx.box` can read files and run commands in the environment it is scoring. Every other supervised hook fires after teardown and can only read the artifact. +`ValidationCtx.node` identifies the supervised node, including its recursion depth, so a +shared validator can apply a root-only contract without applying it to nested managers. The verdict becomes the settled artifact's verdict. Absent, nothing changes and the leaf falls back to its own settle verdict. @@ -21443,6 +21447,12 @@ Kernel-owned context for the concrete supervised node a factory is constructing. > `readonly` **nodeId**: `string` +##### depth? + +> `readonly` `optional` **depth?**: `number` + +Recursion depth supplied by Runtime scopes (root = 0). Standalone callers may omit it. + ##### attemptId > `readonly` **attemptId**: `string` @@ -24234,6 +24244,13 @@ Live sandbox for this iteration. Validators that need execution-grounded evidence can inspect files or run commands here instead of forcing callers to bypass the loop kernel with raw Sandbox SDK orchestration. +##### node? + +> `readonly` `optional` **node?**: [`ExecutorNodeContext`](#executornodecontext) + +Detached, immutable node identity supplied by supervised provider execution. +Runtime scopes include depth (root = 0); standalone execution may omit this context. + ##### signal > **signal**: `AbortSignal` diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs index 21f2e895..55f71a58 100644 --- a/scripts/check-model-execution-boundary.mjs +++ b/scripts/check-model-execution-boundary.mjs @@ -17,6 +17,7 @@ const dynamicNonModelFetchOwners = new Set([ 'src/platform/auth.ts', 'src/platform/integrations.ts', 'src/runtime/mcp-environment.ts', + 'src/runtime/supervise/coordination-preflight.ts', 'bench/src/research-shot.ts', 'bench/src/search-tool.ts', 'bench/src/benchmarks/aec-bench.ts', diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs index b2eae442..8793d8cc 100644 --- a/scripts/check-model-execution-boundary.test.mjs +++ b/scripts/check-model-execution-boundary.test.mjs @@ -48,6 +48,14 @@ describe('model execution boundary source check', () => { expect(violations[0]?.location).toBe('3:7') }) + it('permits dynamic MCP preflight routes without admitting named model endpoints', () => { + const path = 'src/runtime/supervise/coordination-preflight.ts' + expect(checkJavaScript(path, 'await fetch(input.url)')).toEqual([]) + expect( + checkJavaScript(path, "await fetch('https://router.tangle.tools/v1/chat/completions')"), + ).toHaveLength(1) + }) + it('rejects qualified and aliased global fetch calls', () => { expect( checkJavaScript( diff --git a/scripts/release-cohort.test.mjs b/scripts/release-cohort.test.mjs index 53c1cf29..dc560ae2 100644 --- a/scripts/release-cohort.test.mjs +++ b/scripts/release-cohort.test.mjs @@ -1,7 +1,9 @@ -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' +import { parse } from 'yaml' +import { rangeAdmits } from './lib/packed-package-test.mjs' import { assertReleaseCohortArtifacts, readReleaseCohort, @@ -17,6 +19,19 @@ afterEach(() => { }) describe('release cohort', () => { + it.each(Object.values(readReleaseCohort().packages))( + 'selects $name@$version within its workspace catalog range', + ({ name, version }) => { + const { catalog } = parse( + readFileSync(new URL('../pnpm-workspace.yaml', import.meta.url), 'utf8'), + ) + expect( + rangeAdmits(catalog[name], version), + `${name}@${version} must be admitted by catalog range ${catalog[name]}`, + ).toBe(true) + }, + ) + it('reads one exact source identity for every first-party dependency', () => { const cohort = readReleaseCohort() expect(Object.keys(cohort.packages)).toEqual([ diff --git a/src/mcp/tool-server.ts b/src/mcp/tool-server.ts index 19cdd94d..f4b7830d 100644 --- a/src/mcp/tool-server.ts +++ b/src/mcp/tool-server.ts @@ -18,7 +18,7 @@ import type { JsonRpcMessage, JsonRpcResponse, McpToolDescriptor, McpTransport } export type { JsonRpcMessage, JsonRpcResponse, McpToolDescriptor, McpTransport } from './protocol' -const PROTOCOL_VERSION = '2024-11-05' +export const PROTOCOL_VERSION = '2024-11-05' /** @experimental */ export interface StdioToolServerOptions { diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index 7f89f067..e0dfb8d1 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -116,6 +116,7 @@ import type { ExecutorExecutionBinding, ExecutorFactory, ExecutorMaterialization, + ExecutorNodeContext, ExecutorResult, Runtime, Spend, @@ -558,6 +559,8 @@ export interface ProviderExecutorOptions { * has: `validate` runs while the environment is still alive, so `ValidationCtx.box` can read * files and run commands in the environment it is scoring. Every other supervised hook fires * after teardown and can only read the artifact. + * `ValidationCtx.node` identifies the supervised node, including its recursion depth, so a + * shared validator can apply a root-only contract without applying it to nested managers. * * The verdict becomes the settled artifact's verdict. Absent, nothing changes and the leaf falls * back to its own settle verdict. @@ -641,6 +644,8 @@ function createProviderExecutor( placement?: { id: string; digest: string }, ): Executor { const controller = new AbortController() + const node = + ctx.node === undefined ? undefined : detachedSnapshot(ctx.node, 'provider executor node') let environment: AgentEnvironment | undefined let artifact: ExecutorResult | undefined @@ -665,8 +670,8 @@ function createProviderExecutor( 'provider placement: profileForCreate cannot change the selected profile', ) } - const executionId = retention?.executionId ?? ctx.node?.nodeId ?? `provider-run-${randomUUID()}` - const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) + const executionId = retention?.executionId ?? node?.nodeId ?? `provider-run-${randomUUID()}` + const attemptId = node?.attemptId ?? newExecutionAttemptId(executionId) const trace = createPushTraceSource({ runId: executionId }) const providerModel = concreteProfileModel(createProfile) // The provider owns the model call inside its environment and the create input carries no @@ -726,6 +731,7 @@ function createProviderExecutor( createProfile, task, signal: linked.signal, + ...(node === undefined ? {} : { node }), options, retention, executionId, @@ -907,6 +913,7 @@ interface StreamProviderExecutorArgs { createProfile: AgentProfile task: unknown signal: AbortSignal + node?: ExecutorNodeContext options: ProviderExecutorOptions retention?: RetainedExecutorContext executionId: string @@ -1085,6 +1092,7 @@ async function* streamProviderExecutor( // hook fires after teardown and can only read the artifact. const verdict = await args.options.validator?.validate(result, { iteration: 0, + ...(args.node === undefined ? {} : { node: args.node }), box: environmentAsSandboxInstance(environment, { requireTerminalEvent: args.options.requireTerminalEvent ?? true, }), diff --git a/src/runtime/supervise/coordination-mcp.ts b/src/runtime/supervise/coordination-mcp.ts index c4a28f4c..fdda8e34 100644 --- a/src/runtime/supervise/coordination-mcp.ts +++ b/src/runtime/supervise/coordination-mcp.ts @@ -51,6 +51,7 @@ import { coordinationHttpHandler, coordinationHttpLimits, } from './coordination-http' +import { preflightPublicCoordination } from './coordination-preflight' import { singleFlightTools } from './single-flight-tools' export type { CoordinationHttpAudit, CoordinationHttpOptions } from './coordination-http' @@ -567,8 +568,21 @@ export async function serveCoordinationMcp( audiences.add(publicAddress.host) if (host === '0.0.0.0' || host === '::') audiences.add(`127.0.0.1:${port}`) paths.add(publicAddress.pathname) + if (configured !== undefined) { + await preflightPublicCoordination({ + url, + headers, + signal: opts.scope.signal, + requestTimeoutMs, + toolNames: selectedNames, + }) + } } catch (error) { - await new Promise((resolve) => server.close(() => resolve())) + closed = true + await new Promise((resolve) => { + server.close(() => resolve()) + server.closeAllConnections() + }) throw error } diff --git a/src/runtime/supervise/coordination-preflight.test.ts b/src/runtime/supervise/coordination-preflight.test.ts new file mode 100644 index 00000000..64ecd222 --- /dev/null +++ b/src/runtime/supervise/coordination-preflight.test.ts @@ -0,0 +1,150 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { afterEach, describe, expect, it } from 'vitest' +import { preflightPublicCoordination } from './coordination-preflight' + +const closes: Array<() => Promise> = [] +afterEach(async () => { + await Promise.all(closes.splice(0).map((close) => close())) +}) + +async function endpoint(handler: (request: IncomingMessage, response: ServerResponse) => void) { + const server = createServer(handler) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + closes.push( + () => + new Promise((resolve) => { + server.closeAllConnections() + server.close(() => resolve()) + }), + ) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('test endpoint did not bind') + return `http://127.0.0.1:${address.port}` +} + +function inspect(url: string, signal = new AbortController().signal, requestTimeoutMs = 1000) { + return preflightPublicCoordination({ + url, + headers: { Authorization: 'Bearer private-test-credential' }, + signal, + requestTimeoutMs, + toolNames: ['stop'], + }) +} + +describe('public coordination preflight', () => { + it('performs authenticated initialization and checks the exact granted tool names', async () => { + const methods: string[] = [] + const url = await endpoint((request, response) => { + expect(request.headers.authorization).toBe('Bearer private-test-credential') + let body = '' + request.on('data', (chunk) => { + body += String(chunk) + }) + request.on('end', () => { + const rpc = JSON.parse(body) + methods.push(rpc.method) + response.end( + JSON.stringify({ + jsonrpc: '2.0', + id: rpc.id, + result: + rpc.method === 'initialize' + ? { + protocolVersion: '2024-11-05', + serverInfo: { name: 'coordination' }, + capabilities: { tools: {} }, + } + : { tools: [{ name: 'stop' }] }, + }), + ) + }) + }) + await inspect(url) + expect(methods).toEqual(['initialize', 'tools/list']) + }) + + it.each([401, 403, 404, 503])( + 'refuses HTTP %s without retaining echoed secrets', + async (status) => { + const url = await endpoint((_request, response) => { + response.writeHead(status).end('private-test-credential private-response') + }) + await expect(inspect(url)).rejects.toThrow( + `coordination public endpoint preflight failed: HTTP ${status}`, + ) + try { + await inspect(url) + } catch (error) { + expect(String(error)).not.toContain('private-') + expect(String(error)).not.toContain(url) + expect(error).not.toHaveProperty('cause') + } + }, + ) + + it('refuses redirects instead of forwarding actor credentials to another endpoint', async () => { + let redirected = 0 + const target = await endpoint((_request, response) => { + redirected++ + response.end('{}') + }) + const url = await endpoint((_request, response) => + response.writeHead(307, { location: target }).end(), + ) + await expect(inspect(url)).rejects.toThrow('HTTP 307') + expect(redirected).toBe(0) + }) + + it.each(['not-json', '{}', '{"jsonrpc":"2.0","id":"other","result":{}}'])( + 'refuses malformed MCP response %s', + async (body) => { + const url = await endpoint((_request, response) => response.end(body)) + await expect(inspect(url)).rejects.toThrow('invalid MCP response') + }, + ) + + it('bounds a hanging response body and distinguishes timeout from cancellation', async () => { + const url = await endpoint((_request, response) => { + response.writeHead(200) + response.write('{') + }) + await expect(inspect(url, undefined, 40)).rejects.toThrow('timed out') + const controller = new AbortController() + const pending = inspect(url, controller.signal) + controller.abort('private-test-credential in a caller reason') + await expect(pending).rejects.toThrow('preflight failed: cancelled') + }) + + it('bounds response bytes before decoding or retaining an upstream body', async () => { + const url = await endpoint((_request, response) => response.end('x'.repeat(1024 * 1024 + 1))) + await expect(inspect(url)).rejects.toThrow('response too large') + }) + + it('rejects incomplete or extra tool grants', async () => { + const url = await endpoint((request, response) => { + let body = '' + request.on('data', (chunk) => { + body += String(chunk) + }) + request.on('end', () => { + const rpc = JSON.parse(body) + response.end( + JSON.stringify({ + jsonrpc: '2.0', + id: rpc.id, + result: + rpc.method === 'initialize' + ? { + protocolVersion: '2024-11-05', + serverInfo: { name: 'coordination' }, + capabilities: { tools: {} }, + } + : { tools: [{ name: 'spawn_worker' }] }, + }), + ) + }) + }) + await expect(inspect(url)).rejects.toThrow('coordination tool grants differ') + }) +}) diff --git a/src/runtime/supervise/coordination-preflight.ts b/src/runtime/supervise/coordination-preflight.ts new file mode 100644 index 00000000..1260abda --- /dev/null +++ b/src/runtime/supervise/coordination-preflight.ts @@ -0,0 +1,117 @@ +import { ConfigError } from '../../errors' +import { PROTOCOL_VERSION } from '../../mcp/tool-server' +import { linkAbort, runAbortable } from './abortable' + +const MAX_RESPONSE_BYTES = 1024 * 1024 + +class CoordinationPreflightError extends ConfigError { + constructor(reason: string) { + super(`coordination public endpoint preflight failed: ${reason}`) + } +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** Check the operator's route before it becomes a provider attachment. This does not prove cloud egress. */ +export async function preflightPublicCoordination(input: { + url: string + headers: Readonly> + signal: AbortSignal + requestTimeoutMs: number + toolNames: readonly string[] +}): Promise { + const deadline = new AbortController() + const linked = linkAbort(input.signal, deadline.signal) + const timer = setTimeout(() => deadline.abort(), Math.min(input.requestTimeoutMs, 10_000)) + timer.unref() + const rpc = async (method: string, params?: unknown): Promise> => { + const response = await fetch(input.url, { + method: 'POST', + headers: { ...input.headers, 'content-type': 'application/json', accept: 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: method, method, ...(params ? { params } : {}) }), + redirect: 'manual', + signal: linked.signal, + }) + if (response.status !== 200) { + void response.body?.cancel().catch(() => undefined) + throw new CoordinationPreflightError(`HTTP ${response.status}`) + } + const reader = response.body?.getReader() + if (!reader) throw new CoordinationPreflightError('invalid MCP response') + let size = 0 + const chunks: Uint8Array[] = [] + try { + for (;;) { + const part = await reader.read() + if (part.done) break + size += part.value.byteLength + if (size > MAX_RESPONSE_BYTES) { + void reader.cancel().catch(() => undefined) + throw new CoordinationPreflightError('response too large') + } + chunks.push(part.value) + } + } finally { + reader.releaseLock() + } + let body: unknown + try { + body = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks))) + } catch { + throw new CoordinationPreflightError('invalid MCP response') + } + if ( + !record(body) || + body.jsonrpc !== '2.0' || + body.id !== method || + body.error || + !record(body.result) + ) { + throw new CoordinationPreflightError('invalid MCP response') + } + return body.result + } + try { + await runAbortable( + async () => { + const initialized = await rpc('initialize', { + protocolVersion: PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: 'agent-runtime-coordination-preflight', version: '1' }, + }) + if ( + initialized.protocolVersion !== PROTOCOL_VERSION || + !record(initialized.serverInfo) || + initialized.serverInfo.name !== 'coordination' || + !record(initialized.capabilities) || + !record(initialized.capabilities.tools) + ) + throw new CoordinationPreflightError('invalid MCP initialization') + const listing = await rpc('tools/list') + if ( + !Array.isArray(listing.tools) || + listing.tools.some((tool) => !record(tool) || typeof tool.name !== 'string') + ) { + throw new CoordinationPreflightError('invalid MCP tool list') + } + const names = listing.tools.map((tool) => tool.name).sort() + if (JSON.stringify(names) !== JSON.stringify([...input.toolNames].sort())) { + throw new CoordinationPreflightError('coordination tool grants differ') + } + }, + linked.signal, + 'coordination public endpoint preflight cancelled', + ) + } catch (error) { + // Neither a response body nor an upstream exception is safe to retain: both can echo credentials. + if (input.signal.aborted) throw new CoordinationPreflightError('cancelled') + if (deadline.signal.aborted) throw new CoordinationPreflightError('timed out') + if (error instanceof CoordinationPreflightError) throw error + throw new CoordinationPreflightError('transport unavailable') + } finally { + clearTimeout(timer) + linked.release() + } +} diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index b8247449..b38ed883 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -882,6 +882,7 @@ export function createScope(args: ScopeArgs): Scope { rootId: args.root, parentId: args.parentId, nodeId: id, + depth: args.depth + 1, attemptId, ...(identity ? { identity } : {}), }, @@ -1875,6 +1876,7 @@ export function createScope(args: ScopeArgs): Scope { journal: args.journal, root: args.ownerMaterialization.journalRoot ?? args.root, nodeId: args.ownerMaterialization.nodeId ?? args.parentId, + depth: args.depth, runtime: args.ownerMaterialization.runtime, attemptId: args.ownerMaterialization.attemptId, ...(authoredProfile === undefined ? {} : { authoredProfile }), @@ -2062,6 +2064,7 @@ interface OwnerMaterializationState { readonly journal: SpawnJournal readonly root: NodeId readonly nodeId: NodeId + readonly depth: number readonly runtime: NodeSnapshot['runtime'] /** Rotated by `beginScopeOwnerAttempt` on every driver attempt after the first. */ attemptId: string @@ -2202,6 +2205,7 @@ export function scopeOwnerExecutorNodeContext(scope: Scope): ExecutorNo rootId: state.root, parentId: state.nodeId, nodeId: state.nodeId, + depth: state.depth, attemptId: state.attemptId, }) } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 297e7fc7..c93f024f 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -701,6 +701,8 @@ export interface ExecutorNodeContext { readonly rootId: NodeId readonly parentId: NodeId readonly nodeId: NodeId + /** Recursion depth supplied by Runtime scopes (root = 0). Standalone callers may omit it. */ + readonly depth?: number /** Kernel-minted identity for this concrete execution attempt. */ readonly attemptId: string readonly identity?: NodeExecutionIdentity diff --git a/src/runtime/types.ts b/src/runtime/types.ts index 7b872887..28d724a8 100644 --- a/src/runtime/types.ts +++ b/src/runtime/types.ts @@ -24,6 +24,7 @@ import type { import type { AgentRunOutcome } from '@tangle-network/sandbox/runtime' import type { RuntimeHooks } from '../runtime-hooks' import type { RuntimeRunHandle } from '../runtime-run' +import type { ExecutorNodeContext } from './supervise/types' // DefaultVerdict is a substrate primitive — it lives in @tangle-network/agent-eval. // agent-runtime re-exports it here so existing consumers keep working without @@ -42,6 +43,11 @@ export interface ValidationCtx { * to bypass the loop kernel with raw Sandbox SDK orchestration. */ box?: SandboxInstance + /** + * Detached, immutable node identity supplied by supervised provider execution. + * Runtime scopes include depth (root = 0); standalone execution may omit this context. + */ + readonly node?: ExecutorNodeContext /** Cooperative cancellation channel. */ signal: AbortSignal /** diff --git a/tests/helpers/coordination-proxy.ts b/tests/helpers/coordination-proxy.ts new file mode 100644 index 00000000..d819b25d --- /dev/null +++ b/tests/helpers/coordination-proxy.ts @@ -0,0 +1,42 @@ +import { createServer, request } from 'node:http' + +/** A stable public address whose upstream changes when a retained coordinator restarts. */ +export async function coordinationProxy() { + let upstream: { host: string; port: number } | undefined + const server = createServer((incoming, outgoing) => { + if (!upstream) { + outgoing.writeHead(503).end() + return + } + const target = request( + { + hostname: upstream.host, + port: upstream.port, + path: incoming.url, + method: incoming.method, + headers: { ...incoming.headers, host: `${upstream.host}:${upstream.port}` }, + }, + (response) => { + outgoing.writeHead(response.statusCode ?? 502, response.headers) + response.pipe(outgoing) + }, + ) + target.on('error', () => outgoing.writeHead(502).end()) + outgoing.on('close', () => target.destroy()) + incoming.pipe(target) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('proxy did not bind') + return { + url: `http://127.0.0.1:${address.port}`, + forwardTo(port: number, host = '127.0.0.1') { + upstream = { host, port } + }, + close: () => + new Promise((resolve) => { + server.closeAllConnections() + server.close(() => resolve()) + }), + } +} diff --git a/tests/kernel/coordination-mcp.test.ts b/tests/kernel/coordination-mcp.test.ts index f3bfcd1b..6645e46a 100644 --- a/tests/kernel/coordination-mcp.test.ts +++ b/tests/kernel/coordination-mcp.test.ts @@ -1,6 +1,10 @@ +import { mkdtemp, rm } from 'node:fs/promises' import { createServer, request } from 'node:http' -import { networkInterfaces } from 'node:os' -import { describe, expect, it, vi } from 'vitest' +import { connect, type Socket } from 'node:net' +import { networkInterfaces, tmpdir } from 'node:os' +import { join } from 'node:path' +import { createKnowledgeTools, createRunScopedStores } from '@tangle-network/agent-knowledge' +import { afterEach, describe, expect, it, vi } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { DEFAULT_AWAIT_EVENT_TIMEOUT_MS } from '../../src/mcp/tools/coordination' import { coordinationHttpHandler } from '../../src/runtime/supervise/coordination-http' @@ -21,9 +25,20 @@ import type { Scope, UsageEvent, } from '../../src/runtime/supervise/types' +import { coordinationProxy } from '../helpers/coordination-proxy' import { supervisorAgent } from '../helpers/runtime-with-test-brain' import { runtimeToolDeclarations, testAgentProfile } from './test-agent-profile' +const proxies: Awaited>[] = [] +afterEach(async () => { + await Promise.all(proxies.splice(0).map((proxy) => proxy.close())) +}) +async function publicProxy() { + const proxy = await coordinationProxy() + proxies.push(proxy) + return proxy +} + // A real (simple) delivering leaf — NOT a mock of the MCP path; the HTTP→MCP→Scope.spawn is real. function deliveringLeaf(name: string, out: unknown): Agent { const ex: Executor = { @@ -526,6 +541,55 @@ function postHttp( } describe('authenticated and bounded coordination HTTP', () => { + it('preflights the coordination and Knowledge tools independently of the incoming request limit', async () => { + const root = await mkdtemp(join(tmpdir(), 'coordination-knowledge-')) + try { + const nodeTools = createKnowledgeTools({ + stores: createRunScopedStores({ root }), + runId: 'preflight', + retrieverVersion: 'test', + }).map((tool) => ({ + name: tool.name, + description: tool.description, + inputSchema: tool.inputSchemaJson!, + handler: tool.handler, + })) + const toolNames = [ + 'spawn_worker', + 'observe_agent', + 'steer_agent', + 'await_event', + 'list_questions', + 'answer_question', + 'ask_parent', + 'stop', + 'read_journal', + 'list_analysts', + 'run_analyst', + ...nodeTools.map((tool) => tool.name), + ] + await withBoundHttp( + { + maxRequestBytes: 1024, + publicUrl: ({ port }) => `http://127.0.0.1:${port}/mcp`, + analysts: { kinds: [], run: async () => [] }, + nodeTools, + toolNames, + }, + async (mcp) => { + const listing = await jsonRpc(mcp.url, 'tools/list', {}, mcp.headers) + expect(Buffer.byteLength(JSON.stringify(listing))).toBeGreaterThan(1024) + expect(Buffer.byteLength(JSON.stringify(listing))).toBeLessThan(1024 * 1024) + expect(listing.result).toMatchObject({ + tools: toolNames.map((name) => expect.objectContaining({ name })), + }) + }, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it.each(['0.0.0.0', '::'])( 'accepts only the actual bound address behind a proxy with wildcard %s', async (host) => { @@ -534,6 +598,7 @@ describe('authenticated and bounded coordination HTTP', () => { .find((entry) => entry?.family === 'IPv4' && !entry.internal)?.address if (!address) throw new Error('This HTTP regression needs an assigned non-loopback IPv4 address') + const proxy = await publicProxy() const post = (port: number, headers: Record, path = '/mcp') => new Promise((resolve, reject) => { const req = request( @@ -553,12 +618,13 @@ describe('authenticated and bounded coordination HTTP', () => { publicUrl: async ({ port }) => { // The bound socket exists before public routing and credentials are ready. expect(await post(port, { Host: `${address}:${port}` })).toBe(403) - return 'https://coordination.example/mcp' + proxy.forwardTo(port, address) + return `${proxy.url}/mcp` }, }, async (mcp) => { expect(await post(mcp.port, { ...mcp.headers, Host: `${address}:${mcp.port}` })).toBe(200) - expect(await post(mcp.port, { ...mcp.headers, Host: 'coordination.example' })).toBe(200) + expect(await post(mcp.port, { ...mcp.headers, Host: new URL(proxy.url).host })).toBe(200) for (const authority of [ `${address}:${mcp.port + 1}`, `192.0.2.1:${mcp.port}`, @@ -850,21 +916,23 @@ describe('authenticated and bounded coordination HTTP', () => { it.each([false, true])( 'binds the caller-owned endpoint and credential audience with async resolution=%s', async (asynchronous) => { + const proxy = await publicProxy() await withBoundHttp( { - publicUrl: ({ actorId }) => { - const url = `https://coordination.example/${actorId}` + publicUrl: ({ actorId, port }) => { + proxy.forwardTo(port) + const url = `${proxy.url}/${actorId}` return asynchronous ? Promise.resolve(url) : url }, }, async (mcp) => { - expect(mcp.url).toBe('https://coordination.example/actor-a') + expect(mcp.url).toBe(`${proxy.url}/actor-a`) expect(mcp.url).not.toContain(mcp.headers.Authorization!) const response = await fetch(`http://127.0.0.1:${mcp.port}/actor-a`, { method: 'POST', headers: { ...mcp.headers, - Host: 'coordination.example', + Host: new URL(proxy.url).host, 'content-type': 'application/json', }, body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }), @@ -981,6 +1049,51 @@ describe('authenticated and bounded coordination HTTP', () => { }) describe('coordination credential continuity', () => { + it('closes partial startup connections when public preflight fails', async () => { + let socket: Socket | undefined + try { + await withBoundHttp({}, async (parent) => { + await expect( + withBoundHttp( + { + publicUrl: async ({ port }) => { + socket = connect(port, '127.0.0.1') + await new Promise((resolve, reject) => { + socket!.once('connect', resolve) + socket!.once('error', reject) + }) + socket.write('POST /mcp HTTP/1.1\r\n') + return parent.url + }, + identity: { runId: 'run-a', actorId: 'actor-b' }, + }, + async () => { + throw new Error('must not admit a failed endpoint') + }, + ), + ).rejects.toThrow('coordination public endpoint preflight failed: HTTP 401') + }) + } finally { + socket?.destroy() + } + }, 2_000) + + it('refuses a public route to another actor before exposing a manager handle', async () => { + await withBoundHttp({}, async (parent) => { + await expect( + withBoundHttp( + { + publicUrl: parent.url, + identity: { runId: 'run-a', actorId: 'actor-b' }, + }, + async () => { + throw new Error('must not expose the wrong actor') + }, + ), + ).rejects.toThrow('coordination public endpoint preflight failed: HTTP 401') + }) + }) + it('does not revive a revoked signed credential through signature padding', async () => { await withBoundHttp( { @@ -1006,7 +1119,11 @@ describe('coordination credential continuity', () => { it('resumes scope-bound signed credentials only when the receiver permits that lifetime', async () => { const signingKeys = { activeKeyId: 'run', keys: { run: 'k'.repeat(48) } } - const publicUrl = 'https://coordination.example/scope-bound' + const proxy = await publicProxy() + const publicUrl = ({ port }: { port: number }) => { + proxy.forwardTo(port) + return `${proxy.url}/scope-bound` + } let original: Readonly> = {} await withBoundHttp({ authentication: { signingKeys }, publicUrl }, async (mcp) => { expect(mcp.credentialExpiresAt).toBeUndefined() @@ -1032,7 +1149,11 @@ describe('coordination credential continuity', () => { it('accepts an original credential only for its exact restarted authority and retained verification key', async () => { const signingKeys = { activeKeyId: 'original', keys: { original: 'a'.repeat(48) } } - const publicUrl = 'https://coordination.example/manager' + const proxy = await publicProxy() + const publicUrl = ({ port }: { port: number }) => { + proxy.forwardTo(port) + return `${proxy.url}/manager` + } let original: Readonly> = {} await withBoundHttp( { authentication: { signingKeys, ttlMs: 900_000 }, publicUrl }, @@ -1053,7 +1174,12 @@ describe('coordination credential continuity', () => { for (const mismatch of [ { identity: { runId: 'other-run', actorId: 'actor-a' } }, { identity: { runId: 'run-a', actorId: 'other-actor' } }, - { publicUrl: 'https://coordination.example/other' }, + { + publicUrl: ({ port }: { port: number }) => { + proxy.forwardTo(port) + return `${proxy.url}/other` + }, + }, { toolNames: ['probe', 'stop'] }, { authentication: { signingKeys: { activeKeyId: 'next', keys: { next: 'b'.repeat(48) } } } }, ]) { diff --git a/tests/kernel/nested-retained-owner-journal.test.ts b/tests/kernel/nested-retained-owner-journal.test.ts index 05c047e7..24d2711a 100644 --- a/tests/kernel/nested-retained-owner-journal.test.ts +++ b/tests/kernel/nested-retained-owner-journal.test.ts @@ -8,13 +8,16 @@ import type { import { afterEach, describe, expect, it } from 'vitest' import { createFileRunContext } from '../../src/runtime/supervise/run-context' import type { SpawnEvent, SpawnJournal } from '../../src/runtime/supervise/types' +import { coordinationProxy } from '../helpers/coordination-proxy' import { durableRetainedProvider } from '../helpers/durable-retained-provider' import { supervise } from '../helpers/runtime-with-test-brain' import { scriptedBrain } from './scripted-brain' import { runtimeToolDeclarations, testAgentProfile } from './test-agent-profile' const directories: string[] = [] +const proxies: Awaited>[] = [] afterEach(async () => { + await Promise.all(proxies.splice(0).map((proxy) => proxy.close())) await Promise.all( directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), ) @@ -103,11 +106,16 @@ describe('nested retained owner journal isolation', () => { } }, } + const proxy = await coordinationProxy() + proxies.push(proxy) const coordination = { authentication: { signingKeys: { activeKeyId: 'test', keys: { test: 'nested-test-secret-'.repeat(3) } }, }, - publicUrl: () => 'https://coordination.example/nested', + publicUrl: ({ port }: { port: number }) => { + proxy.forwardTo(port) + return `${proxy.url}/nested` + }, } const finalizer = () => ({ finalizedBy: 'manager' }) const manager = testAgentProfile('manager', { diff --git a/tests/kernel/provider-executor-composition.test.ts b/tests/kernel/provider-executor-composition.test.ts index 0e498f66..52d27374 100644 --- a/tests/kernel/provider-executor-composition.test.ts +++ b/tests/kernel/provider-executor-composition.test.ts @@ -211,6 +211,7 @@ describe("createExecutor({ backend: 'provider' })", () => { provider, validator: { async validate(out, validationCtx) { + expect(validationCtx.node).toBeUndefined() // The environment is still alive here: the check runs a command inside the box it // is scoring, which no post-teardown hook can do. const proof = await validationCtx.box?.exec('cat answer.txt') @@ -227,6 +228,42 @@ describe("createExecutor({ backend: 'provider' })", () => { expect(lifecycle).toEqual(['exec:cat answer.txt', 'destroy']) }) + it('detaches and freezes the validator node before execution begins', async () => { + const { provider } = recordingProvider() + const node = { + rootId: 'root', + parentId: 'root', + nodeId: 'root:s0', + depth: 1, + attemptId: 'root:s0:attempt:1', + identity: { correlation: { project: 'original' } }, + } + const executor = createExecutor({ + backend: 'provider', + provider, + validator: { + async validate(_out, validationCtx) { + expect(validationCtx.node).not.toBe(node) + expect(validationCtx.node?.depth).toBe(1) + expect(validationCtx.node?.identity?.correlation?.project).toBe('original') + expect(Object.isFrozen(validationCtx.node)).toBe(true) + expect(Object.isFrozen(validationCtx.node?.identity?.correlation)).toBe(true) + expect(() => Object.assign(validationCtx.node ?? {}, { depth: 0 })).toThrow() + return { valid: true, score: 1 } + }, + }, + })(spec, { ...ctx(), node }) + node.depth = 9 + node.identity.correlation.project = 'changed' + + for await (const _event of executor.execute('work', new AbortController().signal)) { + // Drain before reading the checked artifact. + } + + expect((await executor.resultArtifact()).verdict).toEqual({ valid: true, score: 1 }) + expect(node.depth).toBe(9) + }) + it('carries the declared prompt options through the steerable session', async () => { const turns: AgentTurnInput[] = [] const provider: AgentEnvironmentProvider = { diff --git a/tests/kernel/provider-validator-context.test.ts b/tests/kernel/provider-validator-context.test.ts new file mode 100644 index 00000000..2da869c2 --- /dev/null +++ b/tests/kernel/provider-validator-context.test.ts @@ -0,0 +1,126 @@ +import type { + AgentEnvironmentProvider, + CreateAgentEnvironmentInput, +} from '@tangle-network/agent-interface/environment-provider' +import { expect, it } from 'vitest' +import { supervise } from '../../src/runtime/supervise/supervise' +import type { ExecutorNodeContext } from '../../src/runtime/supervise/types' +import { runtimeToolDeclarations, testAgentProfile } from './test-agent-profile' + +async function callTool( + input: CreateAgentEnvironmentInput, + name: string, + args: Record, +): Promise> { + const server = input.runtimeAttachments?.mcp['agent-runtime-coordination'] + if (server?.transport !== 'http') throw new Error('missing coordination attachment') + const headers = Object.fromEntries( + Object.entries(server.headers ?? {}).map(([key, value]) => { + if (value.kind !== 'secret-ref' || value.format !== 'bearer') { + throw new Error('expected private bearer reference') + } + return [key, `Bearer ${input.env?.[value.key]}`] + }), + ) + const response = await fetch(server.url, { + method: 'POST', + headers: { ...headers, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: name, + method: 'tools/call', + params: { name, arguments: args }, + }), + }) + const reply = (await response.json()) as { + result?: { structuredContent?: Record; isError?: boolean } + } + if (!response.ok || reply.result?.isError || !reply.result?.structuredContent) { + throw new Error(`coordination ${name} failed: ${JSON.stringify(reply)}`) + } + return reply.result.structuredContent +} + +it('scopes a live product check to the root across a provider director and its leaf', async () => { + const tools = runtimeToolDeclarations('spawn_worker', 'await_event') + const root = testAgentProfile('root', { harness: 'codex', tools }) + const director = testAgentProfile('director', { harness: 'codex', tools }) + const leaf = testAgentProfile('leaf', { harness: 'codex' }) + const checked: Array<{ name: string; node: ExecutorNodeContext }> = [] + const commands: string[] = [] + const destroyed = new Set() + const provider: AgentEnvironmentProvider = { + name: 'recursive-validation-fixture', + capabilities: () => ({ create: { runtimeAttachments: { mcp: true } } }), + async create(input) { + const name = input.profile?.name ?? 'missing' + return { + id: `env-${name}`, + provider: 'recursive-validation-fixture', + status: async () => 'running', + destroy: async () => { + destroyed.add(name) + }, + exec: async (command) => { + expect(destroyed.has(name)).toBe(false) + commands.push(`${name}:${command}`) + return { exitCode: name === 'root' ? 0 : 1, stdout: '', stderr: '' } + }, + async *stream() { + if (name !== 'leaf') { + const spawned = await callTool(input, 'spawn_worker', { + profile: name === 'root' ? director : leaf, + task: 'Complete this assignment.', + budget: { + maxIterations: name === 'root' ? 8 : 2, + maxTokens: name === 'root' ? 1000 : 100, + }, + }) + expect(typeof spawned.workerId, JSON.stringify(spawned)).toBe('string') + await callTool(input, 'await_event', { timeoutMs: 5000 }) + } + yield { type: 'text', data: { text: name } } + yield { type: 'done', data: { outcome: { type: 'completed' } } } + }, + } + }, + } + + const result = await supervise(root, 'Build the product.', { + runId: 'validator-context', + backend: { + backend: 'provider', + provider, + validator: { + async validate(out, ctx) { + if (ctx.node?.depth === undefined) throw new Error('missing supervised depth') + checked.push({ name: out.content, node: ctx.node }) + if (ctx.node.depth !== 0) return { valid: out.content.length > 0, score: 1 } + const check = await ctx.box?.exec('verify-product') + return { valid: check?.exitCode === 0, score: 1 } + }, + }, + }, + budget: { maxIterations: 32, maxTokens: 4000 }, + perWorker: { maxIterations: 8, maxTokens: 1000 }, + driverRetry: { enabled: false }, + coordination: { + authentication: true, + publicUrl: ({ port }) => `http://127.0.0.1:${port}/mcp`, + }, + }) + + expect( + checked.map(({ name, node }) => ({ name, depth: node.depth })), + JSON.stringify(result), + ).toEqual([ + { name: 'leaf', depth: 2 }, + { name: 'director', depth: 1 }, + { name: 'root', depth: 0 }, + ]) + expect(new Set(checked.map(({ node }) => node.nodeId)).size).toBe(3) + expect(new Set(checked.map(({ node }) => node.attemptId)).size).toBe(3) + expect(checked.every(({ node }) => Object.isFrozen(node))).toBe(true) + expect(commands).toEqual(['root:verify-product']) + expect([...destroyed]).toEqual(['leaf', 'director', 'root']) +}) diff --git a/tests/kernel/supervise-coordination-channel-preflight.test.ts b/tests/kernel/supervise-coordination-channel-preflight.test.ts index 9706e0ec..065b46b7 100644 --- a/tests/kernel/supervise-coordination-channel-preflight.test.ts +++ b/tests/kernel/supervise-coordination-channel-preflight.test.ts @@ -8,6 +8,7 @@ * refusal points at, so the same child is admitted once one is present. */ +import { createServer } from 'node:http' import type { AgentProfile } from '@tangle-network/agent-interface' import type { AgentEnvironmentCapabilities, @@ -229,3 +230,41 @@ it('never admits a provider manager when asynchronous endpoint provisioning fail expect(events.some((event) => event.kind === 'settled' && event.status === 'down')).toBe(true) await expect(fetch(localUrl)).rejects.toThrow() }) + +it.each([401, 403, 404, 503, 200, 'timeout'] as const)( + 'refuses a public coordinator failure %s before provider creation', + async (status) => { + const server = createServer((_request, response) => { + if (status !== 'timeout') response.writeHead(status).end('private-upstream-response') + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('endpoint did not bind') + const { provider: base, creates } = neverCreatingProvider() + const provider: AgentEnvironmentProvider = { + ...base, + capabilities: () => ({ create: { runtimeAttachments: { mcp: true } } }), + } + try { + const { events } = await spawnLeadFromRoot( + { + backend: { backend: 'provider', provider }, + coordination: { + authentication: true, + publicUrl: `http://127.0.0.1:${address.port}/manager`, + requestTimeoutMs: 100, + }, + driverRetry: { enabled: false }, + }, + true, + ) + expect(creates()).toBe(0) + expect(events.some((event) => event.kind === 'settled' && event.status === 'down')).toBe(true) + expect(JSON.stringify(events)).toContain('coordination public endpoint preflight failed') + expect(JSON.stringify(events)).not.toContain('private-upstream-response') + } finally { + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + } + }, +) diff --git a/tests/kernel/supervise-retained-owner-recovery.test.ts b/tests/kernel/supervise-retained-owner-recovery.test.ts index eec1e9ac..6c8b1a3a 100644 --- a/tests/kernel/supervise-retained-owner-recovery.test.ts +++ b/tests/kernel/supervise-retained-owner-recovery.test.ts @@ -4,16 +4,20 @@ import { join } from 'node:path' import type { AgentEnvironment, AgentEnvironmentProvider, + AgentTurnInput, } from '@tangle-network/agent-interface/environment-provider' import { afterEach, describe, expect, it } from 'vitest' import { createFileRunContext } from '../../src/runtime/supervise/run-context' import { supervise } from '../../src/runtime/supervise/supervise' import type { SpawnEvent, SpawnJournal } from '../../src/runtime/supervise/types' +import { coordinationProxy } from '../helpers/coordination-proxy' import { durableRetainedProvider } from '../helpers/durable-retained-provider' import { runtimeToolDeclarations, testAgentProfile } from './test-agent-profile' const directories: string[] = [] +const proxies: Awaited>[] = [] afterEach(async () => { + await Promise.all(proxies.splice(0).map((proxy) => proxy.close())) await Promise.all( directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), ) @@ -23,6 +27,8 @@ describe('retained external supervisor recovery', () => { it('reconstructs a reprompt interrupted after environment admission without a third dispatch', async () => { const directory = await mkdtemp(join(tmpdir(), 'retained-owner-reprompt-crash-')) directories.push(directory) + const proxy = await coordinationProxy() + proxies.push(proxy) const stateFile = join(directory, 'provider.json') const runDirectory = join(directory, 'run') const context = createFileRunContext(runDirectory) @@ -113,7 +119,8 @@ describe('retained external supervisor recovery', () => { }, publicUrl: (address: { port: number }) => { port = address.port - return 'https://coordination.example/manager' + proxy.forwardTo(port) + return `${proxy.url}/manager` }, }, journal: { @@ -184,6 +191,8 @@ describe('retained external supervisor recovery', () => { async (cleanup) => { const directory = await mkdtemp(join(tmpdir(), 'retained-owner-reprompt-')) directories.push(directory) + const proxy = await coordinationProxy() + proxies.push(proxy) const stateFile = join(directory, 'provider.json') const runDirectory = join(directory, 'run') const context = createFileRunContext(runDirectory) @@ -285,7 +294,8 @@ describe('retained external supervisor recovery', () => { }, publicUrl: (address) => { coordinationPort = address.port - return `https://coordination.example:${address.port}/manager` + proxy.forwardTo(coordinationPort) + return `${proxy.url}/manager` }, }, }, @@ -445,6 +455,8 @@ async function setup( ) { const directory = await mkdtemp(join(tmpdir(), 'retained-owner-')) directories.push(directory) + const proxy = await coordinationProxy() + proxies.push(proxy) const stateFile = join(directory, 'provider.json') const runDirectory = join(directory, 'run') const context = createFileRunContext(runDirectory) @@ -464,7 +476,7 @@ async function setup( method: 'POST', headers: { Authorization: `Bearer ${originalToken}`, - Host: 'coordination.example', + Host: new URL(proxy.url).host, 'content-type': 'application/json', }, body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }), @@ -564,7 +576,8 @@ async function setup( }, publicUrl: (address) => { port = address.port - return 'https://coordination.example/manager' + proxy.forwardTo(port) + return `${proxy.url}/manager` }, }, }) diff --git a/tests/runtime/cli-executor-shutdown.test.ts b/tests/runtime/cli-executor-shutdown.test.ts index 478f4ed9..6c08769b 100644 --- a/tests/runtime/cli-executor-shutdown.test.ts +++ b/tests/runtime/cli-executor-shutdown.test.ts @@ -104,7 +104,8 @@ describe('CLI shutdown acknowledgement', () => { it('escalates an ignored SIGTERM and waits for actual process exit', async () => { const child = await startChild('') try { - expect(await child.executor.teardown(40)).toEqual({ destroyed: true }) + // The child must be scheduled to record SIGTERM before escalation; full-suite load can exceed 40 ms. + expect(await child.executor.teardown(1_000)).toEqual({ destroyed: true }) expect(await readFile(child.terminated, 'utf8')).toBe('SIGTERM') const pid = Number(await readFile(child.ready, 'utf8')) expect(() => process.kill(pid, 0)).toThrow()