diff --git a/services/cloud-agent-next/.dev.vars.example b/services/cloud-agent-next/.dev.vars.example index 9dd2719655..90edf1dba9 100644 --- a/services/cloud-agent-next/.dev.vars.example +++ b/services/cloud-agent-next/.dev.vars.example @@ -40,8 +40,8 @@ WORKER_URL=http://host.docker.internal:8794 CLI_TIMEOUT_SECONDS=900 REAPER_INTERVAL_MS=300000 -# Per-session sandbox org IDs (optional) -# Comma-separated list of org IDs that use per-session sandboxes, or `*` for all orgs +# Per-session Cloudflare sandbox org IDs (optional) +# Comma-separated list of org IDs that use per-session Cloudflare sandboxes, or `*` for all orgs PER_SESSION_SANDBOX_ORG_IDS= # Credential containment org IDs (optional) diff --git a/services/cloud-agent-next/AGENTS.md b/services/cloud-agent-next/AGENTS.md index 562b016820..ff3c2d5463 100644 --- a/services/cloud-agent-next/AGENTS.md +++ b/services/cloud-agent-next/AGENTS.md @@ -32,6 +32,7 @@ Git tokens (GitHub App installation tokens, managed GitLab tokens) are resolved - `services/cloud-agent-next/test/e2e/README.md` is the source of truth for setup, fake-LLM routing, lifecycle directives, and troubleshooting. - Prefer focused scenario debugging first: `pnpm exec tsx services/cloud-agent-next/test/e2e/run.ts `. - Run the aggregate local regression matrix with `pnpm exec tsx services/cloud-agent-next/test/e2e/smoke.ts` when validating the full real Worker + DO + sandbox + wrapper path. +- Non-zero `portOffset` sessions (the common case) require `WORKER_URL`/`FAKE_LLM_URL` env overrides on every driver invocation — see README "Running". Discover the offset with `pnpm dev:status --json`. - This harness is local/manual rather than part of normal `pnpm test` or CI; use `dev/logs/cloud-agent-next.log` and `dev/logs/fake-llm.log` when debugging it. - Read `services/cloud-agent-next/DEBUG.md` when correlating local Wrangler logs with Docker sandbox containers, wrapper log files, Kilo CLI logs, uploaded archives, or stuck session flows. diff --git a/services/cloud-agent-next/DEBUG.md b/services/cloud-agent-next/DEBUG.md index 7994895e3d..57fdb2de00 100644 --- a/services/cloud-agent-next/DEBUG.md +++ b/services/cloud-agent-next/DEBUG.md @@ -169,6 +169,28 @@ The internal `getWrapperLogs` path also discovers these sandbox-side files direc - Worker queueing succeeds, but no wrapper logs appear: - inspect pending flush scheduling, sandbox creation, and wrapper startup logs in `dev/logs/cloud-agent-next.log`. +- Worker log shows `Failed to issue Kilo session capability` / `Worker "git-token-service-dev" not found`: + - the `GIT_TOKEN_SERVICE` service binding did not resolve. This fails before + any wrapper or sandbox work, so no wrapper log or fake-LLM traffic exists. + Check `.wrangler/dev-registry/` for a missing `git-token-service-dev` + entry; if absent, `pnpm dev:restart cloudflare-git-token-service` and + confirm it reappears before retrying the turn. +- Turn stalls in `preparing` (repeats several times, no kilo events, no + fake-LLM traffic): + - the wrapper inside the sandbox is failing to start. Look for + `Reconciling physical wrapper stop … reason: 'startup-failed'`, + `ContainerControlConnection upgrade returned retryable status` (503), + or `Container is not listening to port …` in + `dev/logs/cloud-agent-next.log`, then read the wrapper log inside the + sandbox (`/tmp/kilocode-wrapper-*.log`) for bootstrap/import detail. The + 503 from the container control connection means the Docker container's + control plane isn't ready — this is an environmental issue under Docker + Desktop load, not a code regression. If the failure is transient, suspect + Docker contention: leftover stopped containers, a competing dev session + also running Cloud Agent sandboxes, or stale DO alarm timers from + previous sessions. Confirm the fake LLM was never reached with + `curl -s $FAKE_LLM_URL/test/requests` — a flat `chatCompletions` count + proves the stall is upstream of kilo inference. - Wrapper log reaches bootstrap/import, then repeats: - inspect import metadata, import exit code, and post-import `getSession()` lookup. - Wrapper ingest connects, but UI stays stale: diff --git a/services/cloud-agent-next/src/agent-sandbox/agent-sandbox-contract.test.ts b/services/cloud-agent-next/src/agent-sandbox/agent-sandbox-contract.test.ts new file mode 100644 index 0000000000..d497be6d53 --- /dev/null +++ b/services/cloud-agent-next/src/agent-sandbox/agent-sandbox-contract.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it, vi } from 'vitest'; + +vi.mock('@cloudflare/sandbox', () => ({ getSandbox: vi.fn() })); + +import type { SessionMetadata } from '../persistence/session-metadata.js'; +import type { Env, SandboxInstance } from '../types.js'; +import { CloudflareAgentSandbox } from './cloudflare/cloudflare-agent-sandbox.js'; +import type { AgentSandbox } from './protocol.js'; + +/** + * Shared behavioral contract every AgentSandbox adapter must satisfy for a + * session with no running wrapper. Provider-specific flows (ensureWrapper, + * workspace preparation, reconciliation) stay in the per-provider suites. + */ + +function sessionMetadata(provider: 'cloudflare'): SessionMetadata { + return { + metadataSchemaVersion: 2, + identity: { sessionId: 'agent_contract', userId: 'user_contract' }, + auth: {}, + workspace: { sandboxId: 'ses-abcdef', sandboxProvider: provider }, + lifecycle: { version: 1, timestamp: 1 }, + } satisfies SessionMetadata; +} + +function idleCloudflareSandbox(): AgentSandbox { + const sandbox = { + listProcesses: vi.fn().mockResolvedValue([]), + destroy: vi.fn().mockResolvedValue(undefined), + renewActivityTimeout: vi.fn(), + } as unknown as SandboxInstance; + return new CloudflareAgentSandbox({} as Env, sessionMetadata('cloudflare'), { + resolveSandbox: () => sandbox, + sleep: () => Promise.resolve(), + stopObservationDelaysMs: [0], + }); +} + +describe.each([['cloudflare', idleCloudflareSandbox]] as const)( + 'AgentSandbox contract (%s)', + (_provider, createIdleSandbox) => { + it('reports absent wrappers when none are running', async () => { + await expect(createIdleSandbox().discoverSessionWrappers()).resolves.toEqual({ + status: 'absent', + }); + }); + + it('treats stopping wrappers as settled when none are running', async () => { + const result = await createIdleSandbox().stopWrappers({ + target: { kind: 'session' }, + attemptId: 'attempt-contract', + reason: 'session-delete', + }); + + expect(result.status).toBe('absent'); + }); + + it('scopes instance-targeted stops to the leased instance', async () => { + const result = await createIdleSandbox().stopWrappers({ + target: { + kind: 'instance', + instance: { instanceId: 'instance-other', instanceGeneration: 7 }, + }, + attemptId: 'attempt-contract', + reason: 'startup-failed', + }); + + expect(result.status).toBe('absent'); + }); + + it('returns no running wrapper when none exists', async () => { + await expect(createIdleSandbox().getRunningWrapper()).resolves.toBeNull(); + }); + + it('resolves recovery deletion without throwing', async () => { + await expect(createIdleSandbox().delete('recovery')).resolves.toBeUndefined(); + }); + + it('keeps the sandbox alive without a running wrapper', async () => { + await expect(createIdleSandbox().keepAlive()).resolves.toBeUndefined(); + }); + + it('reports terminal availability as a declared capability outcome', async () => { + const result = await createIdleSandbox().getRunningTerminalClient(); + + expect(['ready', 'not-running', 'unhealthy', 'capability-unavailable']).toContain( + result.status + ); + }); + } +); diff --git a/services/cloud-agent-next/src/agent-sandbox/capabilities.ts b/services/cloud-agent-next/src/agent-sandbox/capabilities.ts new file mode 100644 index 0000000000..ea948d6b46 --- /dev/null +++ b/services/cloud-agent-next/src/agent-sandbox/capabilities.ts @@ -0,0 +1,14 @@ +import type { AgentSandboxProvider } from '../types.js'; + +export type ProviderCapabilities = { + terminal: boolean; + devcontainer: boolean; +}; + +/** + * Static capability matrix per sandbox provider. Metadata validation and + * feature gates read this table instead of hard-coding provider names. + */ +export const PROVIDER_CAPABILITIES: Record = { + cloudflare: { terminal: true, devcontainer: true }, +}; diff --git a/services/cloud-agent-next/src/agent-sandbox/factory.test.ts b/services/cloud-agent-next/src/agent-sandbox/factory.test.ts index 2dde48278b..fc880e1824 100644 --- a/services/cloud-agent-next/src/agent-sandbox/factory.test.ts +++ b/services/cloud-agent-next/src/agent-sandbox/factory.test.ts @@ -6,18 +6,23 @@ import { CloudflareAgentSandbox } from './cloudflare/cloudflare-agent-sandbox.js vi.mock('@cloudflare/sandbox', () => ({ getSandbox: vi.fn() })); -function metadata(): SessionMetadata { +function metadata(provider?: 'cloudflare'): SessionMetadata { return { metadataSchemaVersion: 2, identity: { sessionId: 'agent_sandbox', userId: 'user_sandbox' }, auth: {}, - workspace: { sandboxId: 'ses-abcdef' }, + workspace: { sandboxId: 'ses-abcdef', ...(provider ? { sandboxProvider: provider } : {}) }, lifecycle: { version: 1, timestamp: 1 }, }; } -describe('AgentSandbox factory', () => { - it('constructs the Cloudflare runtime adapter', () => { - expect(createAgentSandbox({} as Env, metadata())).toBeInstanceOf(CloudflareAgentSandbox); - }); +describe('AgentSandbox provider factory', () => { + it.each([undefined, 'cloudflare'] as const)( + 'resolves %s metadata to the Cloudflare runtime adapter', + provider => { + expect(createAgentSandbox({} as Env, metadata(provider))).toBeInstanceOf( + CloudflareAgentSandbox + ); + } + ); }); diff --git a/services/cloud-agent-next/src/agent-sandbox/factory.ts b/services/cloud-agent-next/src/agent-sandbox/factory.ts index b542f10b45..a076f276df 100644 --- a/services/cloud-agent-next/src/agent-sandbox/factory.ts +++ b/services/cloud-agent-next/src/agent-sandbox/factory.ts @@ -1,8 +1,25 @@ import type { SessionMetadata } from '../persistence/session-metadata.js'; import type { Env } from '../types.js'; -import type { AgentSandbox } from './protocol.js'; +import type { AgentSandbox, AgentSandboxLifecycle, AgentSandboxLifecycleHost } from './protocol.js'; import { CloudflareAgentSandbox } from './cloudflare/cloudflare-agent-sandbox.js'; export function createAgentSandbox(env: Env, metadata: SessionMetadata): AgentSandbox { return new CloudflareAgentSandbox(env, metadata); } + +/** + * Provider-side lifecycle reconciliation seam. Cloudflare runtimes are + * locally addressed, so creation never needs reconciliation and deletion is + * driven synchronously by the session; providers whose runtimes are created + * or deleted through remote APIs dispatch to their own lifecycle here. + */ +export function createAgentSandboxLifecycle( + _env: Env, + _host: AgentSandboxLifecycleHost +): AgentSandboxLifecycle { + return { + reconcileCreateIntent: async () => undefined, + planDeletion: async () => ({ kind: 'not-applicable' }), + reconcilePendingDeletion: async () => 'none', + }; +} diff --git a/services/cloud-agent-next/src/agent-sandbox/protocol.ts b/services/cloud-agent-next/src/agent-sandbox/protocol.ts index 978d697957..b60552d08a 100644 --- a/services/cloud-agent-next/src/agent-sandbox/protocol.ts +++ b/services/cloud-agent-next/src/agent-sandbox/protocol.ts @@ -6,9 +6,36 @@ import type { FencedWrapperDispatchRequest, WorkspaceReady, } from '../execution/types.js'; +import type { SessionMetadata } from '../persistence/session-metadata.js'; export type SandboxDeleteReason = 'explicit' | 'retention-expired' | 'recovery'; +export type SessionDeletionIntent = { + reason: Extract; + startedAt: number; +}; + +export type AgentSandboxFailure = + | 'provider_not_configured' + | 'provider_auth_failed' + | 'runtime_not_running' + | 'runtime_creation_failed' + | 'runtime_configuration_drift' + | 'runtime_deleted_during_active_work' + | 'runtime_max_duration_reached' + | 'runtime_infrastructure_failed' + | 'capability_unavailable'; + +export class AgentSandboxUnavailableError extends Error { + constructor( + message: string, + public readonly failure: AgentSandboxFailure = 'capability_unavailable' + ) { + super(message); + this.name = 'AgentSandboxUnavailableError'; + } +} + export const WRAPPER_DISCOVERY_LIST_PROCESSES_TIMEOUT_REASON = 'wrapper_discovery_list_processes_timeout'; export type WrapperInspectionFailureReason = typeof WRAPPER_DISCOVERY_LIST_PROCESSES_TIMEOUT_REASON; @@ -66,7 +93,8 @@ export type StopWrappersResult = export type TerminalClientResult = | { status: 'ready'; client: TerminalWrapperClient } | { status: 'not-running' } - | { status: 'unhealthy' }; + | { status: 'unhealthy' } + | { status: 'capability-unavailable'; message: string }; export type WrapperLogs = { files: Record; @@ -115,3 +143,40 @@ export type AgentSandbox = { keepAlive(): Promise; delete(reason: SandboxDeleteReason): Promise; }; + +export type ProviderDeletionPlan = + | { kind: 'not-applicable' } + | { kind: 'complete' } + | { kind: 'deferred'; entries: Record }; + +/** + * Session-DO capabilities a provider lifecycle needs: durable storage for its + * intents/tombstones, alarm scheduling for retries, and terminal-state + * transitions owned by the session. + */ +export type AgentSandboxLifecycleHost = { + storage: DurableObjectStorage; + scheduleAlarmAtOrBefore(deadline: number): Promise; + eraseDurableObjectState(): Promise; + purgeDeletedSessionPayload(): Promise; + getSessionIdForLogs(): string | undefined; +}; + +/** + * Provider-side lifecycle reconciliation for one Cloud Agent session: + * settling interrupted runtime creation and driving deletion to a terminal + * state. Methods self-guard on stored provider state, so the session can + * invoke them on alarm paths without knowing its provider. + * + * `planDeletion` only returns what to persist; the session commits the + * returned entries atomically with its own deletion-intent fence. + */ +export type AgentSandboxLifecycle = { + reconcileCreateIntent(now: number): Promise; + planDeletion(input: { + metadata: SessionMetadata; + intent: SessionDeletionIntent; + now: number; + }): Promise; + reconcilePendingDeletion(now: number): Promise<'handled' | 'none'>; +}; diff --git a/services/cloud-agent-next/src/execution/errors.ts b/services/cloud-agent-next/src/execution/errors.ts index 2ba3d2aae4..8cfb38f80a 100644 --- a/services/cloud-agent-next/src/execution/errors.ts +++ b/services/cloud-agent-next/src/execution/errors.ts @@ -25,7 +25,8 @@ export type RetryableErrorCode = export type PermanentErrorCode = | 'INVALID_REQUEST' // Bad input (missing fields, invalid format) | 'SESSION_NOT_FOUND' // Session doesn't exist - | 'WRAPPER_JOB_CONFLICT'; // Wrapper busy (internal error - shouldn't happen) + | 'WRAPPER_JOB_CONFLICT' // Wrapper busy (internal error - shouldn't happen) + | 'SANDBOX_CAPABILITY_UNAVAILABLE'; // This session targets a deliberately disabled runtime capability /** * All possible execution error codes. @@ -126,6 +127,13 @@ export class ExecutionError extends Error { static wrapperJobConflict(message: string): ExecutionError { return new ExecutionError('WRAPPER_JOB_CONFLICT', message, { retryable: false }); } + + static sandboxCapabilityUnavailable(message: string, cause?: unknown): ExecutionError { + return new ExecutionError('SANDBOX_CAPABILITY_UNAVAILABLE', message, { + retryable: false, + cause, + }); + } } /** diff --git a/services/cloud-agent-next/src/execution/orchestrator.test.ts b/services/cloud-agent-next/src/execution/orchestrator.test.ts index c6d79a922d..9ff34702a1 100644 --- a/services/cloud-agent-next/src/execution/orchestrator.test.ts +++ b/services/cloud-agent-next/src/execution/orchestrator.test.ts @@ -1,5 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { AgentSandbox, WrapperInstanceLease } from '../agent-sandbox/protocol.js'; +import { + AgentSandboxUnavailableError, + type AgentSandbox, + type WrapperInstanceLease, +} from '../agent-sandbox/protocol.js'; import type { Env } from '../types.js'; import { WrapperError } from '../kilo/wrapper-client.js'; import type { ExecutionError } from './errors.js'; @@ -220,17 +224,10 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => { command: 'compact', arguments: '--aggressive', }, - finalization: { - autoCommit: true, - condenseOnComplete: false, - }, } satisfies FencedWrapperDispatchRequest; const commandRequest = { command: 'compact', - args: '--aggressive', messageId: commandPlan.turn.messageId, - autoCommit: true, - condenseOnComplete: false, session: prepared.readyRequest.session, }; buildWrapperSessionReadyAndPromptRequestsMock.mockResolvedValueOnce({ @@ -264,6 +261,18 @@ describe('ExecutionOrchestrator AgentSandbox delivery', () => { } satisfies Partial); }); + it('classifies intentionally unavailable wrapper delivery as permanent', async () => { + const { orchestrator, ensureWrapper } = createOrchestrator(); + ensureWrapper.mockRejectedValueOnce( + new AgentSandboxUnavailableError('Sandbox delivery capability is not enabled') + ); + + await expect(orchestrator.execute(basePlan)).rejects.toMatchObject({ + code: 'SANDBOX_CAPABILITY_UNAVAILABLE', + retryable: false, + } satisfies Partial); + }); + it('keeps ordinary wrapper bootstrap failure retryable', async () => { const { orchestrator, ensureWrapper } = createOrchestrator(); ensureWrapper.mockRejectedValueOnce(new Error('wrapper unavailable')); diff --git a/services/cloud-agent-next/src/execution/orchestrator.ts b/services/cloud-agent-next/src/execution/orchestrator.ts index 3b3241c775..5d98deee86 100644 --- a/services/cloud-agent-next/src/execution/orchestrator.ts +++ b/services/cloud-agent-next/src/execution/orchestrator.ts @@ -25,7 +25,11 @@ import { getPreparationInfrastructureFailure, withPreparationInfrastructureRecovery, } from '../sandbox-recovery.js'; -import type { AgentSandbox, WrapperInstanceLease } from '../agent-sandbox/protocol.js'; +import { + AgentSandboxUnavailableError, + type AgentSandbox, + type WrapperInstanceLease, +} from '../agent-sandbox/protocol.js'; import { isCodeReviewEphemeralSandboxId } from '../code-review-ephemeral-sandbox.js'; /** Maximum time allowed for complete wrapper readiness, including Kilo startup. */ @@ -147,6 +151,12 @@ export class ExecutionOrchestrator { await this.destroyEphemeralSandboxAfterPreAcceptanceFailure(sandbox, plan, error); if (error instanceof ExecutionError) throw error; if (error instanceof WrapperError && error.code === 'WRAPPER_FINALIZING') throw error; + if (error instanceof AgentSandboxUnavailableError) { + throw ExecutionError.sandboxCapabilityUnavailable( + 'Sandbox runtime delivery is unavailable for this session', + error + ); + } throw ExecutionError.wrapperStartFailed( `Failed to start wrapper: ${error instanceof Error ? error.message : String(error)}`, error diff --git a/services/cloud-agent-next/src/execution/types.ts b/services/cloud-agent-next/src/execution/types.ts index de490346b5..65c93794be 100644 --- a/services/cloud-agent-next/src/execution/types.ts +++ b/services/cloud-agent-next/src/execution/types.ts @@ -228,6 +228,8 @@ export type RetryableResultCode = | 'WRAPPER_START_FAILED' | 'WRAPPER_FINALIZING'; +export type PermanentDeliveryResultCode = 'SANDBOX_CAPABILITY_UNAVAILABLE'; + export type AdmissionFailure = { success: false; code: 'NOT_FOUND' | 'BAD_REQUEST' | 'INTERNAL' | 'PENDING_QUEUE_FULL' | RetryableResultCode; @@ -235,6 +237,12 @@ export type AdmissionFailure = { failureBoundary?: 'registration' | 'admission'; }; +export type PermanentDeliveryFailure = { + success: false; + code: PermanentDeliveryResultCode; + error: string; +}; + /** Durable acknowledgement that a message intent is stored for asynchronous delivery. */ export type DurableAdmissionAck = { success: true; @@ -252,7 +260,10 @@ export type RuntimeAcceptanceResult = { }; export type SessionMessageAdmissionResult = DurableAdmissionAck | AdmissionFailure; -export type MessageDeliveryResult = RuntimeAcceptanceResult | AdmissionFailure; +export type MessageDeliveryResult = + | RuntimeAcceptanceResult + | AdmissionFailure + | PermanentDeliveryFailure; /** Compatibility request retained only for external schema/type imports. */ export type QueueSessionMessageRequest = SubmittedSessionMessageRequest; diff --git a/services/cloud-agent-next/src/kilo/wrapper-client.test.ts b/services/cloud-agent-next/src/kilo/wrapper-client.test.ts index 07066c3d75..77f03d082d 100644 --- a/services/cloud-agent-next/src/kilo/wrapper-client.test.ts +++ b/services/cloud-agent-next/src/kilo/wrapper-client.test.ts @@ -177,6 +177,102 @@ describe('WrapperClient', () => { expect(client).toBeDefined(); }); + + it('supports every protocol endpoint through transport-only construction', async () => { + const responses = [ + { healthy: true, state: 'idle', version: WRAPPER_VERSION, sessionId: 'kilo_sess' }, + { state: 'active', sessionId: 'kilo_sess' }, + { status: 'ready', kiloSessionId: 'kilo_sess', workspaceReady: {} }, + { status: 'sent', messageId: 'msg_result' }, + { status: 'sent', result: { output: 'done' } }, + { status: 'answered', success: true }, + { status: 'answered', success: true }, + { status: 'rejected', success: true }, + { status: 'aborted' }, + ]; + const transport: WrapperTransport = { + request: vi.fn().mockImplementation(async () => Response.json(responses.shift())), + }; + const client = new WrapperClient({ transport }); + const readyRequest = { + agentSessionId: 'agent_test', + userId: 'user_test', + sandboxId: 'ses-test', + kiloSessionId: 'kilo_sess', + workspace: { + workspacePath: '/workspace/test', + sessionHome: '/home/test', + branchName: 'main', + }, + materialized: { env: {} }, + session: defaultPromptSession, + }; + + await expect(client.health()).resolves.toMatchObject({ healthy: true }); + await expect(client.status()).resolves.toMatchObject({ state: 'active' }); + await expect(client.ensureSessionReady(readyRequest)).resolves.toMatchObject({ + kiloSessionId: 'kilo_sess', + }); + await expect(client.prompt(createPromptOptions())).resolves.toEqual({ + messageId: 'msg_result', + }); + await expect(client.command({ command: 'status' })).resolves.toEqual({ output: 'done' }); + await expect(client.answerPermission('perm_1', 'once')).resolves.toEqual({ success: true }); + await expect(client.answerQuestion('question_1', [['yes']])).resolves.toEqual({ + success: true, + }); + await expect(client.rejectQuestion('question_2')).resolves.toEqual({ success: true }); + await expect(client.abort()).resolves.toBeUndefined(); + expect(transport.request).toHaveBeenCalledTimes(9); + }); + + it('preserves finalizing errors through transport-only construction', async () => { + const transport: WrapperTransport = { + request: vi.fn().mockResolvedValue( + Response.json( + { + error: 'WRAPPER_FINALIZING', + message: 'Wrapper batch is finalizing', + wrapperRunId: 'wr_transport', + }, + { status: 409 } + ) + ), + }; + const client = new WrapperClient({ transport }); + + await expect(client.prompt(createPromptOptions())).rejects.toEqual( + expect.objectContaining({ + name: 'WrapperFinalizingError', + code: 'WRAPPER_FINALIZING', + wrapperRunId: 'wr_transport', + }) + ); + }); + + it('preserves workspace setup errors through transport-only construction', async () => { + const transport: WrapperTransport = { + request: vi.fn().mockResolvedValue( + Response.json( + { + error: 'WORKSPACE_SETUP_FAILED', + message: 'Workspace checkout failed', + }, + { status: 503 } + ) + ), + }; + const client = new WrapperClient({ transport }); + + await expect(client.health()).rejects.toEqual( + expect.objectContaining({ + name: 'WrapperError', + code: 'WORKSPACE_SETUP_FAILED', + message: 'Workspace checkout failed', + statusCode: 503, + }) + ); + }); }); describe('ensureSessionReady', () => { diff --git a/services/cloud-agent-next/src/kilo/wrapper-client.ts b/services/cloud-agent-next/src/kilo/wrapper-client.ts index 8b4eccf68b..ad8356abf9 100644 --- a/services/cloud-agent-next/src/kilo/wrapper-client.ts +++ b/services/cloud-agent-next/src/kilo/wrapper-client.ts @@ -39,14 +39,21 @@ import { parseWrapperSessionReadyErrorResponse } from './wrapper-ready-error.js' // Types // --------------------------------------------------------------------------- -export type WrapperClientOptions = { - /** Sandbox session for exec/writeFile operations */ - session: ExecutionSession; - /** Wrapper HTTP port (typically 5xxx) */ - port: number; - /** Transport for wrapper HTTP requests. Defaults to curl through session.exec. */ - transport?: WrapperTransport; -}; +export type WrapperClientOptions = + | { + /** Sandbox session for exec/writeFile operations */ + session: ExecutionSession; + /** Wrapper HTTP port (typically 5xxx) */ + port: number; + /** Transport for wrapper HTTP requests. Defaults to curl through session.exec. */ + transport?: WrapperTransport; + } + | { + /** Protocol-only transport for providers without a public wrapper port. */ + transport: WrapperTransport; + session?: never; + port?: never; + }; export type EnsureRunningOptions = { agentSessionId: string; @@ -365,11 +372,17 @@ export class ContainerFetchWrapperTransport implements WrapperTransport { // --------------------------------------------------------------------------- export class WrapperClient { - private readonly session: ExecutionSession; - private readonly port: number; - private readonly baseUrl: string; + private readonly cloudflareRuntime?: { session: ExecutionSession; port: number }; private readonly transport: WrapperTransport; + private get session(): ExecutionSession { + return this.requireCloudflareRuntime().session; + } + + private get port(): number { + return this.requireCloudflareRuntime().port; + } + /** * Wrap a wrapper-start command line so it runs inside the dev container via * `devcontainer exec --workspace-folder ... --id-label kilo.agentSession=...`. @@ -479,16 +492,30 @@ export class WrapperClient { } constructor(options: WrapperClientOptions) { - this.session = options.session; - this.port = options.port; - this.baseUrl = `http://127.0.0.1:${this.port}`; - this.transport = - options.transport ?? - new ExecCurlWrapperTransport({ - session: options.session, - baseUrl: this.baseUrl, - shellQuote: value => this.shellQuote(value), - }); + if (options.session !== undefined && options.port !== undefined) { + this.cloudflareRuntime = { session: options.session, port: options.port }; + } + if (options.transport) { + this.transport = options.transport; + return; + } + const runtime = this.requireCloudflareRuntime(); + this.transport = new ExecCurlWrapperTransport({ + session: runtime.session, + baseUrl: `http://127.0.0.1:${runtime.port}`, + shellQuote: value => this.shellQuote(value), + }); + } + + private requireCloudflareRuntime(): { session: ExecutionSession; port: number } { + if (!this.cloudflareRuntime) { + throw new WrapperNotReadyError('Wrapper lifecycle requires a Cloudflare execution session'); + } + return this.cloudflareRuntime; + } + + private protocolDiagnosticFields(): { port?: number } { + return this.cloudflareRuntime ? { port: this.cloudflareRuntime.port } : {}; } /** @@ -520,7 +547,13 @@ export class WrapperClient { const errorCode = readyError?.error ?? parsed.error ?? `HTTP_${response.status}`; const statusCode = ERROR_STATUS_CODES[errorCode] ?? response.status ?? 500; logger - .withFields({ method, path, port: this.port, errorCode, statusCode }) + .withFields({ + method, + path, + ...this.protocolDiagnosticFields(), + errorCode, + statusCode, + }) .warn('Wrapper HTTP request returned an application error'); if (errorCode === 'NO_JOB') { @@ -555,12 +588,12 @@ export class WrapperClient { .withFields({ method, path, - port: this.port, + ...this.protocolDiagnosticFields(), responseBytes: responseText.length, statusCode: response.status, }) .error('Failed to parse wrapper HTTP response'); - throw new WrapperError(`Failed to parse response: ${responseText}`, 'PARSE_ERROR', 500); + throw new WrapperError('Failed to parse wrapper response', 'PARSE_ERROR', 500); } } @@ -576,6 +609,7 @@ export class WrapperClient { * Port retry on EADDRINUSE is handled by the static ensureWrapper() method. */ async ensureRunning(options: EnsureRunningOptions): Promise<{ started: boolean }> { + this.requireCloudflareRuntime(); const { agentSessionId, userId, diff --git a/services/cloud-agent-next/src/persistence/CloudAgentSession.ts b/services/cloud-agent-next/src/persistence/CloudAgentSession.ts index 001c1eefb2..5176561533 100644 --- a/services/cloud-agent-next/src/persistence/CloudAgentSession.ts +++ b/services/cloud-agent-next/src/persistence/CloudAgentSession.ts @@ -8,6 +8,7 @@ import { DurableObject } from 'cloudflare:workers'; import type { CloudAgentQueueReport } from '@kilocode/worker-utils/cloud-agent-queue-report'; import type { OperationResult } from './types.js'; import { + getSandboxProvider, parseSessionMetadata, serializeSessionMetadata, type SessionMetadata, @@ -16,7 +17,9 @@ import { readProfileBundle, type SessionProfileBundle } from '../session-profile import { fitCallbackJobToQueueLimit } from '../callbacks/queue-payload.js'; import type { CallbackJob, CallbackTarget } from '../callbacks/index.js'; import { projectTerminalClientError } from '../session/terminal-error-projector.js'; +import { sql } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/durable-sqlite'; +import { commandQueue, events, executionLeases } from '../db/sqlite-schema.js'; import { logger } from '../logger.js'; import { BUILTIN_AGENT_MODES, Limits } from '../schema.js'; import { migrate } from 'drizzle-orm/durable-sqlite/migrator'; @@ -141,8 +144,11 @@ import { type WrapperTerminalEvent, } from '../session/wrapper-supervisor.js'; import { emitRunStateReport } from '../telemetry/queue-reports.js'; -import { createAgentSandbox } from '../agent-sandbox/factory.js'; +import { createAgentSandbox, createAgentSandboxLifecycle } from '../agent-sandbox/factory.js'; import type { + AgentSandboxLifecycle, + SandboxDeleteReason, + SessionDeletionIntent, StopWrappersResult, WrapperObservation, WrapperStopReason, @@ -167,7 +173,7 @@ const EVENT_RETENTION_MS = Limits.SESSION_TTL_MS; /** Storage key for tracking last activity timestamp */ const LAST_ACTIVITY_KEY = 'last_activity'; -const EXPLICIT_DELETION_PENDING_KEY = 'explicit_deletion_pending'; +const DELETION_INTENT_KEY = 'session_deletion_intent'; const EPHEMERAL_SANDBOX_DESTROY_AFTER_KEY = 'ephemeral_sandbox_destroy_after'; const EPHEMERAL_SANDBOX_DESTROYED_AT_KEY = 'ephemeral_sandbox_destroyed_at'; @@ -252,7 +258,12 @@ type GroupedRegisterSessionInput = { callback?: SessionMetadata['callback']; workspace?: Pick< NonNullable, - 'sandboxId' | 'sandboxRoute' | 'shallow' | 'credentialContainment' | 'devcontainerRequested' + | 'sandboxId' + | 'sandboxRoute' + | 'sandboxProvider' + | 'shallow' + | 'credentialContainment' + | 'devcontainerRequested' >; }; @@ -424,6 +435,7 @@ export class CloudAgentSession extends DurableObject { private ephemeralSandboxDestroyer?: () => Promise; private sharedSandboxFailoverRecorder?: (routeKey: SandboxId) => Promise; private agentRuntime?: AgentRuntime; + private sandboxLifecycle?: AgentSandboxLifecycle; private messageSettlementOutbox?: MessageSettlementOutbox; private sessionMessageQueue?: SessionMessageQueue; private wrapperSupervisor?: WrapperSupervisor; @@ -660,6 +672,7 @@ export class CloudAgentSession extends DurableObject { storage: this.ctx.storage, env: this.env, getMetadata: () => this.getMetadata(), + canUseSandboxRuntime: () => this.canUseSandboxRuntime(), getSessionIdForLogs: () => this.sessionId, sendToWrapper: (ingestTagId, command, fence) => this.sendToWrapper(ingestTagId, command, fence), @@ -677,6 +690,27 @@ export class CloudAgentSession extends DurableObject { return this.agentRuntime; } + private async hasDeletionIntent(): Promise { + return (await this.ctx.storage.get(DELETION_INTENT_KEY)) !== undefined; + } + + private async canUseSandboxRuntime(): Promise { + return !(await this.hasDeletionIntent()); + } + + private getSandboxLifecycle(): AgentSandboxLifecycle { + if (!this.sandboxLifecycle) { + this.sandboxLifecycle = createAgentSandboxLifecycle(this.env, { + storage: this.ctx.storage, + scheduleAlarmAtOrBefore: deadline => this.scheduleAlarmAtOrBefore(deadline), + eraseDurableObjectState: () => this.eraseDurableObjectState(), + purgeDeletedSessionPayload: () => this.purgeDeletedSessionPayload(), + getSessionIdForLogs: () => this.sessionId, + }); + } + return this.sandboxLifecycle; + } + private getWrapperSupervisor(): WrapperSupervisor { if (!this.wrapperSupervisor) { this.wrapperSupervisor = createWrapperSupervisor({ @@ -686,7 +720,10 @@ export class CloudAgentSession extends DurableObject { }, messageSettlementOutbox: this.getMessageSettlementOutbox(), sessionMessageQueue: this.getSessionMessageQueue(), - getMetadata: () => this.getMetadata(), + // Unguarded: the supervisor performs the physical wrapper stop that + // deletion itself depends on, so it must still see metadata once + // deletion intent has been persisted. + getMetadata: () => this.getStoredMetadata(), getAssistantMessageForUserMessage: (sessionId, kiloSessionId, parentMessageId) => this.eventQueries.getAssistantMessageForUserMessage( sessionId, @@ -701,13 +738,26 @@ export class CloudAgentSession extends DurableObject { this.ensureAcceptedMessageBeforeTerminal(messageId, wrapperRunId), stopWrappers: async request => { if (this.physicalWrapperStopper) return this.physicalWrapperStopper(request); - if (this.orchestrator || (!this.env.Sandbox && !this.env.SandboxSmall)) { + if (this.orchestrator) return { status: 'absent' }; + const metadata = await this.getStoredMetadata(); + if (!metadata) { + return { status: 'inspection-failed', error: 'Session metadata unavailable' }; + } + if ( + getSandboxProvider(metadata) === 'cloudflare' && + !this.env.Sandbox && + !this.env.SandboxSmall + ) { return { status: 'absent' }; } - const metadata = await this.getMetadata(); - if (!metadata) - return { status: 'inspection-failed', error: 'Session metadata unavailable' }; - return createAgentSandbox(this.env, metadata).stopWrappers(request); + try { + return await createAgentSandbox(this.env, metadata).stopWrappers(request); + } catch (error) { + return { + status: 'inspection-failed', + error: error instanceof Error ? error.message : String(error), + }; + } }, recordSharedSandboxFailover: routeKey => this.sharedSandboxFailoverRecorder @@ -889,6 +939,9 @@ export class CloudAgentSession extends DurableObject { if (!ticketSessionId || ticketSessionId !== sessionIdParam) { return new Response('Invalid ticket session', { status: 401 }); } + if (await this.hasDeletionIntent()) { + return new Response('Session not found', { status: 404 }); + } const streamHandler = await this.getStreamHandler(sessionIdParam ?? undefined); const response = await streamHandler.handleStreamRequest(request); @@ -903,6 +956,9 @@ export class CloudAgentSession extends DurableObject { // Route ingest WebSocket (internal only - from queue consumer) if (url.pathname === '/ingest') { + if (await this.hasDeletionIntent()) { + return new Response('Session not found', { status: 404 }); + } const ingestHandler = await this.getIngestHandler(); return ingestHandler.handleIngestRequest(request); } @@ -924,6 +980,7 @@ export class CloudAgentSession extends DurableObject { // Check if this is an ingest connection if (tags.some(tag => tag.startsWith('ingest:'))) { + if (await this.hasDeletionIntent()) return; const ingestHandler = await this.getIngestHandler(); await ingestHandler.handleIngestMessage(ws, message); return; @@ -947,6 +1004,7 @@ export class CloudAgentSession extends DurableObject { // Clean up ingest connection tracking if (tags.some(tag => tag.startsWith('ingest:'))) { + if (await this.hasDeletionIntent()) return; const ingestHandler = await this.getIngestHandler(); const disconnected = await ingestHandler.handleIngestClose(ws); @@ -1196,11 +1254,16 @@ export class CloudAgentSession extends DurableObject { * Get session metadata. * Returns null if no metadata has been written yet (e.g., before first CLI execution). */ - async getMetadata(): Promise { + private async getStoredMetadata(): Promise { const metadata = await this.ctx.storage.get('metadata'); return metadata ? parseSessionMetadata(metadata) : null; } + async getMetadata(): Promise { + if (await this.hasDeletionIntent()) return null; + return this.getStoredMetadata(); + } + async validateKiloGlobalFeedProducer(params: { kiloSessionId: string; wrapperRunId: string; @@ -1282,7 +1345,19 @@ export class CloudAgentSession extends DurableObject { * Throws an error if validation fails. */ async updateMetadata(data: unknown): Promise { + if (await this.hasDeletionIntent()) { + throw new Error('Cannot update deleted session metadata'); + } const newMetadata = serializeSessionMetadata(parseSessionMetadata(data)); + const existingMetadata = await this.getMetadata(); + if (existingMetadata) { + if (getSandboxProvider(existingMetadata) !== getSandboxProvider(newMetadata)) { + throw new Error('Registered sandbox provider cannot be changed'); + } + if (existingMetadata.workspace?.sandboxId !== newMetadata.workspace?.sandboxId) { + throw new Error('Registered sandbox name cannot be changed'); + } + } await this.ctx.storage.put('metadata', newMetadata); // Track activity for session TTL @@ -1708,58 +1783,49 @@ export class CloudAgentSession extends DurableObject { } } - private async finalizeSessionDeletion( - reason: 'explicit' | 'retention-expired' - ): Promise { - const metadata = await this.getMetadata(); - if (!metadata) { - const lease = await getWrapperLease(this.ctx.storage); - if (lease.state !== 'none' && !isWrapperCleanupExhausted(lease)) { - await this.schedulePhysicalWrapperCleanupRetry(); - return false; - } - } else { - const supervisor = this.getWrapperSupervisor(); - await supervisor.requestPhysicalWrapperStop('session-delete', { kind: 'session' }); - await supervisor.runMaintenance(Date.now()); - const lease = await getWrapperLease(this.ctx.storage); - if (lease.state !== 'none' && !isWrapperCleanupExhausted(lease)) { - if (reason === 'explicit') { - await this.schedulePhysicalWrapperCleanupRetry(); - } - return false; - } - if (isWrapperCleanupExhausted(lease)) { - try { - await this.deleteSandboxSessionResources(metadata, reason); - } catch (error) { - logger - .withFields({ - sessionId: this.sessionId, - attempts: lease.attempts, - reason, - error: error instanceof Error ? error.message : String(error), - }) - .warn('Best-effort quarantined sandbox session deletion failed'); - } - } else { - await this.deleteSandboxSessionResources(metadata, reason); - } - } + /** Erase all Durable Object storage once a session's deletion is fully resolved. */ + private async eraseDurableObjectState(): Promise { + await this.ctx.storage.deleteAlarm(); + await this.ctx.storage.deleteAll(); + } + /** + * Finish the Cloudflare deletion path: defer while a shared-sandbox failover + * publication is still pending, otherwise erase Durable Object state. + */ + private async finalizeCloudflareDeletion(): Promise { const recovery = await getSandboxRecoveryState(this.ctx.storage); if (recovery?.failoverPublication?.status === 'pending') { - if (reason === 'explicit') await this.schedulePhysicalWrapperCleanupRetry(); + await this.schedulePhysicalWrapperCleanupRetry(); return false; } - - await this.ctx.storage.deleteAlarm(); - await this.ctx.storage.deleteAll(); + await this.eraseDurableObjectState(); return true; } - private async isExplicitDeletionPending(): Promise { - return (await this.ctx.storage.get(EXPLICIT_DELETION_PENDING_KEY)) === true; + private async purgeDeletedSessionPayload( + additionalRetainedKeys: readonly string[] = [] + ): Promise { + // The deletion fence survives the purge, along with any durable keys a + // provider's deferred deletion plan asked the session to persist. + const retainedKeys = new Set([DELETION_INTENT_KEY, ...additionalRetainedKeys]); + while (true) { + const persistedKeys = await this.ctx.storage.list({ limit: 128 }); + const sensitiveKeys = [...persistedKeys.keys()].filter(key => !retainedKeys.has(key)); + if (sensitiveKeys.length === 0) break; + await this.ctx.storage.delete(sensitiveKeys); + } + + const db = drizzle(this.ctx.storage, { logger: false }); + db.delete(events) + .where(sql`true`) + .run(); + db.delete(executionLeases) + .where(sql`true`) + .run(); + db.delete(commandQueue) + .where(sql`true`) + .run(); } async isSandboxCleanupScheduled(): Promise { @@ -1846,8 +1912,8 @@ export class CloudAgentSession extends DurableObject { } private async deletionPendingAdmissionFailure(): Promise { - if (await this.isExplicitDeletionPending()) { - return { success: false, code: 'NOT_FOUND', error: 'Session deletion is pending' }; + if (await this.hasDeletionIntent()) { + return { success: false, code: 'NOT_FOUND', error: 'Session not found' }; } if (await this.isSandboxCleanupScheduled()) { return { success: false, code: 'BAD_REQUEST', error: 'Session sandbox cleanup is scheduled' }; @@ -1856,12 +1922,87 @@ export class CloudAgentSession extends DurableObject { } /** - * Delete session after physical wrapper absence is verified or cleanup is quarantined. + * Delete session after physical wrapper absence is verified, its provider's + * lifecycle cleanup is confirmed or quarantined, or provider deletion is + * dispatched for asynchronous reconciliation via alarms. */ + private async initiateSessionDeletion( + reason: Extract, + persistedIntent?: SessionDeletionIntent + ): Promise<'complete' | 'deferred' | 'physical-cleanup-pending'> { + const metadata = await this.getStoredMetadata(); + if (!metadata) { + // Metadata already purged but the deletion fence is still present: + // provider-side deletion is reconciling asynchronously via alarms. + if (await this.ctx.storage.get(DELETION_INTENT_KEY)) { + return 'deferred'; + } + const lease = await getWrapperLease(this.ctx.storage); + if (lease.state !== 'none' && !isWrapperCleanupExhausted(lease)) { + await this.schedulePhysicalWrapperCleanupRetry(); + return 'physical-cleanup-pending'; + } + return (await this.finalizeCloudflareDeletion()) ? 'complete' : 'physical-cleanup-pending'; + } + + const now = Date.now(); + const existingIntent = await this.ctx.storage.get(DELETION_INTENT_KEY); + const intent = + persistedIntent ?? + existingIntent ?? + ({ reason, startedAt: now } satisfies SessionDeletionIntent); + const deletionPlan = await this.getSandboxLifecycle().planDeletion({ metadata, intent, now }); + if (deletionPlan.kind === 'complete') { + await this.eraseDurableObjectState(); + return 'complete'; + } + if (deletionPlan.kind === 'deferred') { + await this.ctx.storage.put({ + [DELETION_INTENT_KEY]: intent, + ...deletionPlan.entries, + }); + await this.purgeDeletedSessionPayload(Object.keys(deletionPlan.entries)); + await this.getSandboxLifecycle().reconcilePendingDeletion(now); + return 'deferred'; + } + + if (!existingIntent) { + await this.ctx.storage.put(DELETION_INTENT_KEY, intent); + } + + const supervisor = this.getWrapperSupervisor(); + await supervisor.requestPhysicalWrapperStop('session-delete', { kind: 'session' }); + // Use a fresh timestamp: requestPhysicalWrapperStop stamps nextAttemptAt + // with its own Date.now(), which can be a millisecond after `now` above. + await supervisor.runMaintenance(Date.now()); + const lease = await getWrapperLease(this.ctx.storage); + if (lease.state !== 'none' && !isWrapperCleanupExhausted(lease)) { + await this.schedulePhysicalWrapperCleanupRetry(); + return 'physical-cleanup-pending'; + } + if (isWrapperCleanupExhausted(lease)) { + try { + await this.deleteSandboxSessionResources(metadata, reason); + } catch (error) { + logger + .withFields({ + sessionId: this.sessionId, + attempts: lease.attempts, + reason, + error: error instanceof Error ? error.message : String(error), + }) + .warn('Best-effort quarantined sandbox session deletion failed'); + } + } else { + await this.deleteSandboxSessionResources(metadata, reason); + } + + return (await this.finalizeCloudflareDeletion()) ? 'complete' : 'physical-cleanup-pending'; + } + async deleteSession(): Promise { logger.info('Explicit DELETE requested for Durable Object'); - await this.ctx.storage.put(EXPLICIT_DELETION_PENDING_KEY, true); - if (!(await this.finalizeSessionDeletion('explicit'))) { + if ((await this.initiateSessionDeletion('explicit')) === 'physical-cleanup-pending') { throw new Error('Session deletion pending physical wrapper cleanup'); } } @@ -1872,10 +2013,10 @@ export class CloudAgentSession extends DurableObject { * delivers the first message. */ async registerSession(input: GroupedRegisterSessionInput): Promise { - if (await this.isExplicitDeletionPending()) { - return { success: false, error: 'Session deletion is pending' }; - } await this.requireSessionId(input.identity.sessionId as SessionId); + if (await this.hasDeletionIntent()) { + return { success: false, error: 'Session is deleted' }; + } const existing = await this.ctx.storage.get('metadata'); if (existing) { return { success: false, error: 'Session already registered' }; @@ -1933,7 +2074,10 @@ export class CloudAgentSession extends DurableObject { finalization: input.finalization, profile: input.profile, callback: input.callback, - workspace: input.workspace, + workspace: { + ...input.workspace, + sandboxProvider: input.workspace?.sandboxProvider ?? 'cloudflare', + }, lifecycle: { version: now, timestamp: now, @@ -2226,17 +2370,22 @@ export class CloudAgentSession extends DurableObject { let alarmWorkFailed = false; try { - if (await this.isExplicitDeletionPending()) { - logger.withFields({ sessionId: this.sessionId }).info('Resuming explicit session deletion'); - if (await this.finalizeSessionDeletion('explicit')) { + if (await this.hasDeletionIntent()) { + if ((await this.getSandboxLifecycle().reconcilePendingDeletion(now)) === 'handled') { return; } - logger - .withFields({ sessionId: this.sessionId }) - .info('Postponing explicit session deletion until wrapper cleanup confirms absence'); + const intent = await this.ctx.storage.get(DELETION_INTENT_KEY); + const metadata = await this.getStoredMetadata(); + if (intent && metadata) { + await this.initiateSessionDeletion(intent.reason, intent); + return; + } + await this.scheduleAlarmAtOrBefore(now + REAPER_INTERVAL_MS_DEFAULT); return; } + await this.getSandboxLifecycle().reconcileCreateIntent(now); + // Check if session should be deleted due to inactivity (90 days) const lastActivity = await this.ctx.storage.get(LAST_ACTIVITY_KEY); if (lastActivity && now - lastActivity > Limits.SESSION_TTL_MS) { @@ -2244,12 +2393,8 @@ export class CloudAgentSession extends DurableObject { .withFields({ sessionId: this.sessionId, lastActivity }) .info('Deleting session due to inactivity'); - if (await this.finalizeSessionDeletion('retention-expired')) { - return; - } - logger - .withFields({ sessionId: this.sessionId }) - .info('Postponing inactive session deletion until wrapper cleanup confirms absence'); + await this.initiateSessionDeletion('retention-expired'); + return; } await this.getWrapperSupervisor().runMaintenance(now); @@ -2503,6 +2648,7 @@ export class CloudAgentSession extends DurableObject { * Called from the ingest heartbeat adapter; AgentRuntime owns the renewal transport. */ private async keepContainerAlive(): Promise { + if (await this.hasDeletionIntent()) return; await this.getAgentRuntime().keepSandboxAlive(); } @@ -3162,6 +3308,14 @@ export class CloudAgentSession extends DurableObject { const sessionId = plan.scope.sessionId; const eventSourceId = '' as EventSourceId; + if (await this.hasDeletionIntent()) { + return { + success: false, + code: 'INTERNAL', + error: 'Session deletion is in progress', + }; + } + await this.scheduleAlarmAtOrBefore(Date.now() + PENDING_FLUSH_DEBOUNCE_MS); const result = await this.getAgentRuntime().send(plan, { @@ -3193,21 +3347,23 @@ export class CloudAgentSession extends DurableObject { onAccepted: delivery => this.recordRuntimeAcceptedMessage(plan, delivery), }); - this.broadcastVolatileEvent({ - executionId: eventSourceId, - sessionId, - streamEventType: 'cloud.status', - payload: JSON.stringify({ cloudStatus: { type: 'ready' } }), - timestamp: Date.now(), - }); - - logger - .withFields({ + if (result.success) { + this.broadcastVolatileEvent({ + executionId: eventSourceId, sessionId, - messageId: plan.turn.messageId, - wrapperRunId: result.success ? result.wrapperRunId : undefined, - }) - .info('Wrapper accepted delivered session message'); + streamEventType: 'cloud.status', + payload: JSON.stringify({ cloudStatus: { type: 'ready' } }), + timestamp: Date.now(), + }); + + logger + .withFields({ + sessionId, + messageId: plan.turn.messageId, + wrapperRunId: result.wrapperRunId, + }) + .info('Wrapper accepted delivered session message'); + } return result; } diff --git a/services/cloud-agent-next/src/persistence/session-metadata.test.ts b/services/cloud-agent-next/src/persistence/session-metadata.test.ts index 333b3281a8..546b500ac5 100644 --- a/services/cloud-agent-next/src/persistence/session-metadata.test.ts +++ b/services/cloud-agent-next/src/persistence/session-metadata.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest'; import { CurrentSessionMetadataSchema, getEffectiveCredentialContainment, + getSandboxProvider, parseSessionMetadata, requiresContainmentSandbox, serializeSessionMetadata, @@ -129,6 +130,7 @@ describe('session metadata boundary', () => { routeKey: 'usr-000000000000000000000000000000000000000000000000' as const, suffix: 'shared-slot-v1' as const, }, + sandboxProvider: 'cloudflare' as const, workspacePath: '/workspace', sessionHome: '/home/kilo', branchName: 'session/agent_123', @@ -174,6 +176,27 @@ describe('session metadata boundary', () => { ).toThrow(); }); + it('accepts legacy current metadata without an explicit sandbox provider as Cloudflare', () => { + const current = { + metadataSchemaVersion: 2 as const, + identity: { + sessionId: 'agent_legacy_provider', + userId: 'user_legacy_provider', + }, + auth: {}, + workspace: { + sandboxId: 'ses-abcdef' as const, + }, + lifecycle: { + version: 1, + timestamp: 1, + }, + }; + + expect(parseSessionMetadata(current)).toEqual(current); + expect(getSandboxProvider(parseSessionMetadata(current))).toBe('cloudflare'); + }); + it('parses and serializes current grouped DIND workspace metadata', () => { const current = { metadataSchemaVersion: 2 as const, @@ -369,6 +392,7 @@ describe('session metadata boundary', () => { }); expect(metadata.workspace?.sandboxId).toBe('dind-abcdef'); + expect(getSandboxProvider(metadata)).toBe('cloudflare'); expect(metadata.devcontainer).toEqual({ workspacePath: '/workspace/user/sessions/agent_legacy_dind', innerWorkspaceFolder: '/workspaces/repo', diff --git a/services/cloud-agent-next/src/persistence/session-metadata.ts b/services/cloud-agent-next/src/persistence/session-metadata.ts index 6d8d371b23..1e2c073204 100644 --- a/services/cloud-agent-next/src/persistence/session-metadata.ts +++ b/services/cloud-agent-next/src/persistence/session-metadata.ts @@ -1,9 +1,10 @@ import * as z from 'zod'; +import { PROVIDER_CAPABILITIES } from '../agent-sandbox/capabilities.js'; import { isGeneratedSharedSandboxId } from '../sandbox-id.js'; import { SHARED_SANDBOX_FAILOVER_SUFFIX } from '../shared-sandbox-route.js'; import { MESSAGE_ID_FORMAT_DESCRIPTION, MESSAGE_ID_PATTERN } from '../session/message-id.js'; -import type { SandboxId } from '../types.js'; +import type { AgentSandboxProvider, SandboxId } from '../types.js'; import { AttachmentsSchema, branchNameSchema, @@ -26,6 +27,7 @@ const SharedSandboxIdSchema = z .transform(s => s as SandboxId); const MessageIdSchema = z.string().regex(MESSAGE_ID_PATTERN, MESSAGE_ID_FORMAT_DESCRIPTION); +const SandboxProviderSchema = z.enum(['cloudflare']); const MetadataIdentitySchema = z .object({ @@ -207,6 +209,7 @@ const MetadataWorkspaceSchema = z .object({ sandboxId: SandboxIdSchema.optional(), sandboxRoute: MetadataSharedSandboxRouteSchema.optional(), + sandboxProvider: SandboxProviderSchema.optional(), workspacePath: z.string().optional(), sessionHome: z.string().optional(), branchName: z.string().optional(), @@ -245,7 +248,13 @@ const MetadataWorkspaceSchema = z path: ['sandboxId'], }); } - }); + }) + .refine( + workspace => + PROVIDER_CAPABILITIES[workspace.sandboxProvider ?? 'cloudflare'].devcontainer || + workspace.devcontainerRequested !== true, + 'Sandbox provider does not support devcontainers' + ); const MetadataDevContainerSchema = z .object({ @@ -281,7 +290,13 @@ export const CurrentSessionMetadataSchema = z devcontainer: MetadataDevContainerSchema.optional(), lifecycle: MetadataLifecycleSchema, }) - .strip(); + .strip() + .refine( + metadata => + PROVIDER_CAPABILITIES[metadata.workspace?.sandboxProvider ?? 'cloudflare'].devcontainer || + !metadata.devcontainer, + 'Sandbox provider metadata cannot contain a devcontainer runtime' + ); export type SessionMetadata = z.infer; export type CredentialContainment = z.infer; @@ -301,6 +316,10 @@ export function requiresContainmentSandbox(metadata: SessionMetadata): boolean { return containment.github || containment.gitlab || containment.kilocode; } +export function getSandboxProvider(metadata: SessionMetadata): AgentSandboxProvider { + return metadata.workspace?.sandboxProvider ?? 'cloudflare'; +} + type LegacySessionMetadata = z.output; function omitUndefined>(value: T): Partial { diff --git a/services/cloud-agent-next/src/persistence/types.ts b/services/cloud-agent-next/src/persistence/types.ts index 5647fab54d..37255a291d 100644 --- a/services/cloud-agent-next/src/persistence/types.ts +++ b/services/cloud-agent-next/src/persistence/types.ts @@ -168,7 +168,7 @@ export type PersistenceEnv = { R2_ATTACHMENTS_READONLY_ACCESS_KEY_ID?: string; R2_ATTACHMENTS_READONLY_SECRET_ACCESS_KEY?: string; R2_ATTACHMENTS_BUCKET?: string; - /** Comma-separated org IDs that use per-session sandbox containers */ + /** Comma-separated org IDs that use per-session Cloudflare sandbox containers */ PER_SESSION_SANDBOX_ORG_IDS?: string; /** Comma-separated org IDs whose GitHub token uses credential containment, or `*` for all orgs */ GITHUB_TOKEN_CONTAINMENT_ORG_IDS?: string; diff --git a/services/cloud-agent-next/src/router.test.ts b/services/cloud-agent-next/src/router.test.ts index 365ac4be54..b36a4af01b 100644 --- a/services/cloud-agent-next/src/router.test.ts +++ b/services/cloud-agent-next/src/router.test.ts @@ -90,11 +90,13 @@ import { appRouter } from './router.js'; import type { Env, TRPCContext, SessionId } from './types.js'; import type { CloudAgentSessionState } from './persistence/types.js'; import { parseSessionMetadata } from './persistence/session-metadata.js'; +import { WrapperClient } from './kilo/wrapper-client.js'; type MockSessionStub = { deleteSession?: ReturnType; markAsInterrupted?: ReturnType; interruptExecution?: ReturnType; + failExecutionRpc?: ReturnType; getCurrentRuntimeExecution?: ReturnType; getCurrentMessageWork?: ReturnType; getMetadata?: ReturnType; @@ -770,6 +772,7 @@ describe('router sessionId validation', () => { success: true, executionId: 'exc_interrupt_runtime', }), + failExecutionRpc: vi.fn().mockResolvedValue(undefined), getCurrentRuntimeExecution: vi.fn().mockResolvedValue(null), getMetadata: vi.fn().mockResolvedValue(null), }; @@ -811,6 +814,10 @@ describe('router sessionId validation', () => { }; cloudAgentSession = mockContext.env.CLOUD_AGENT_SESSION as unknown as MockCAS; + vi.stubGlobal('scheduler', { + wait: vi.fn().mockResolvedValue(undefined), + }); + caller = appRouter.createCaller(mockContext); }); @@ -1240,7 +1247,7 @@ describe('router sessionId validation', () => { caller = appRouter.createCaller(mockContext); }); - it('returns healthy sandbox and none execution health when no execution is active', async () => { + it('returns healthy sandbox after idle cleanup leaves no running wrapper', async () => { const sessionId: SessionId = 'agent_88888888-8888-8888-8888-888888888888'; const sandboxId = 'ses-a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6'; mockGetMetadata.mockResolvedValue( @@ -1734,6 +1741,88 @@ describe('getMessageResult procedure', () => { }); }); +describe('router question and permission controls', () => { + function createControlCaller(metadata: CloudAgentSessionState) { + const sessionId = metadata.identity.sessionId as SessionId; + const executionSession = {}; + const getSession = vi.fn().mockResolvedValue(executionSession); + const createSession = vi.fn(); + const listProcesses = vi.fn().mockResolvedValue([ + { + id: 'wrapper-control', + command: `kilocode-wrapper WRAPPER_PORT=5050 --agent-session ${sessionId}`, + status: 'running', + }, + ]); + vi.mocked(getSandbox).mockReturnValue({ + listProcesses, + getSession, + createSession, + } as unknown as ReturnType); + vi.mocked(fetchSessionMetadata).mockResolvedValue(metadata); + const context = { + userId: 'test-user-123', + authToken: 'token', + botId: undefined, + request: new Request('http://test.local'), + env: { + Sandbox: {}, + SandboxSmall: {}, + SandboxDIND: {}, + }, + } as unknown as TRPCContext; + return { caller: appRouter.createCaller(context), sessionId, getSession, createSession }; + } + + it('forwards question and permission responses through an existing Cloudflare wrapper', async () => { + vi.clearAllMocks(); + const metadata = parseSessionMetadata({ + metadataSchemaVersion: 2, + identity: { + sessionId: 'agent_12345678-1234-1234-1234-123456789abc', + userId: 'test-user-123', + }, + auth: {}, + workspace: { sandboxId: `ses-${'2'.repeat(48)}`, sandboxProvider: 'cloudflare' }, + lifecycle: { version: 1, timestamp: 1 }, + }); + const answerQuestion = vi + .spyOn(WrapperClient.prototype, 'answerQuestion') + .mockResolvedValueOnce({ + success: true, + }); + const rejectQuestion = vi + .spyOn(WrapperClient.prototype, 'rejectQuestion') + .mockResolvedValueOnce({ + success: true, + }); + const answerPermission = vi + .spyOn(WrapperClient.prototype, 'answerPermission') + .mockResolvedValueOnce({ success: true }); + const { caller, sessionId, getSession, createSession } = createControlCaller(metadata); + + await expect( + caller.answerQuestion({ sessionId, questionId: 'q_1', answers: [['yes']] }) + ).resolves.toEqual({ success: true }); + await expect(caller.rejectQuestion({ sessionId, questionId: 'q_2' })).resolves.toEqual({ + success: true, + }); + await expect( + caller.answerPermission({ sessionId, permissionId: 'p_1', response: 'once' }) + ).resolves.toEqual({ success: true }); + + expect(answerQuestion).toHaveBeenCalledWith('q_1', [['yes']]); + expect(rejectQuestion).toHaveBeenCalledWith('q_2'); + expect(answerPermission).toHaveBeenCalledWith('p_1', 'once'); + expect(getSession).toHaveBeenCalledTimes(3); + expect(getSession).toHaveBeenCalledWith(`${sessionId}-bootstrap`); + expect(createSession).not.toHaveBeenCalled(); + answerQuestion.mockRestore(); + rejectQuestion.mockRestore(); + answerPermission.mockRestore(); + }); +}); + describe('router terminal procedures', () => { it('creates a terminal through the session Durable Object', async () => { const createTerminal = vi.fn().mockResolvedValue({ @@ -1814,6 +1903,48 @@ describe('router terminal procedures', () => { expect(cloudAgentSession.idFromName).not.toHaveBeenCalled(); expect(cloudAgentSession.get).not.toHaveBeenCalled(); }); + + it('resizes and closes terminals through the session Durable Object', async () => { + const pty = { + id: 'pty_123', + title: 'Workspace terminal', + command: '/bin/sh', + args: [], + cwd: '/workspace/repo', + status: 'running' as const, + pid: 123, + }; + const resizeTerminal = vi.fn().mockResolvedValue({ success: true, data: { pty } }); + const closeTerminal = vi.fn().mockResolvedValue({ success: true, data: { success: true } }); + const context = { + userId: 'test-user-123', + authToken: 'token', + botId: undefined, + request: new Request('http://test.local'), + env: { + CLOUD_AGENT_SESSION: { + idFromName: vi.fn().mockReturnValue('do-id'), + get: vi.fn().mockReturnValue({ resizeTerminal, closeTerminal }), + }, + }, + } as unknown as TRPCContext; + const caller = appRouter.createCaller(context); + const sessionId = 'agent_12345678-1234-1234-1234-123456789abc' as SessionId; + + await expect( + caller.resizeTerminal({ + cloudAgentSessionId: sessionId, + ptyId: 'pty_123', + cols: 80, + rows: 24, + }) + ).resolves.toEqual({ pty }); + await expect( + caller.closeTerminal({ cloudAgentSessionId: sessionId, ptyId: 'pty_123' }) + ).resolves.toEqual({ success: true }); + expect(resizeTerminal).toHaveBeenCalledWith({ ptyId: 'pty_123', cols: 80, rows: 24 }); + expect(closeTerminal).toHaveBeenCalledWith({ ptyId: 'pty_123' }); + }); }); describe('legacy V2 execution response compatibility', () => { diff --git a/services/cloud-agent-next/src/router/handlers/session-management.ts b/services/cloud-agent-next/src/router/handlers/session-management.ts index 428f95e605..ae666927aa 100644 --- a/services/cloud-agent-next/src/router/handlers/session-management.ts +++ b/services/cloud-agent-next/src/router/handlers/session-management.ts @@ -1,5 +1,6 @@ import { TRPCError } from '@trpc/server'; import * as z from 'zod'; +import { AgentSandboxUnavailableError } from '../../agent-sandbox/protocol.js'; import { createAgentSandbox } from '../../agent-sandbox/factory.js'; import { logger, withLogTags } from '../../logger.js'; import { generateSandboxId } from '../../sandbox-id.js'; @@ -417,17 +418,20 @@ export function createSessionManagementHandlers() { s => s.isSandboxCleanupScheduled(), 'isSandboxCleanupScheduled' ); - let sandboxStatus: 'healthy' | 'destroyed' | 'unreachable' = cleanupScheduled + let sandboxStatus: 'healthy' | 'destroyed' | 'unreachable' | 'unknown' = cleanupScheduled ? 'destroyed' - : 'healthy'; + : 'unknown'; if (!cleanupScheduled) { try { await createAgentSandbox(env, metadata).probeHealth(); + sandboxStatus = 'healthy'; } catch (error) { - sandboxStatus = 'unreachable'; - logger - .withFields({ error: error instanceof Error ? error.message : String(error) }) - .warn('Sandbox health probe failed'); + if (!(error instanceof AgentSandboxUnavailableError)) { + sandboxStatus = 'unreachable'; + logger + .withFields({ error: error instanceof Error ? error.message : String(error) }) + .warn('Sandbox health probe failed'); + } } } @@ -591,7 +595,15 @@ export function createSessionManagementHandlers() { }); } - const logs = await createAgentSandbox(env, metadata).readWrapperLogs(); + let logs: Awaited['readWrapperLogs']>>; + try { + logs = await createAgentSandbox(env, metadata).readWrapperLogs(); + } catch (error) { + if (error instanceof AgentSandboxUnavailableError) { + throw new TRPCError({ code: 'PRECONDITION_FAILED', message: error.message }); + } + throw error; + } if (!logs) { throw new TRPCError({ code: 'PRECONDITION_FAILED', diff --git a/services/cloud-agent-next/src/router/handlers/session-questions.ts b/services/cloud-agent-next/src/router/handlers/session-questions.ts index 7cb1fb096e..31c2ca924d 100644 --- a/services/cloud-agent-next/src/router/handlers/session-questions.ts +++ b/services/cloud-agent-next/src/router/handlers/session-questions.ts @@ -1,6 +1,7 @@ import { TRPCError } from '@trpc/server'; import * as z from 'zod'; import { createAgentSandbox } from '../../agent-sandbox/factory.js'; +import { AgentSandboxUnavailableError } from '../../agent-sandbox/protocol.js'; import { logger, withLogTags } from '../../logger.js'; import type { SessionId, Env } from '../../types.js'; import { fetchSessionMetadata } from '../../session-service.js'; @@ -25,7 +26,15 @@ async function resolveWrapperClient(opts: { throw new TRPCError({ code: 'NOT_FOUND', message: 'Session not found' }); } - const wrapperClient = await createAgentSandbox(env, metadata).getRunningWrapper(); + let wrapperClient: WrapperClient | null; + try { + wrapperClient = await createAgentSandbox(env, metadata).getRunningWrapper(); + } catch (error) { + if (error instanceof AgentSandboxUnavailableError) { + throw new TRPCError({ code: 'PRECONDITION_FAILED', message: error.message }); + } + throw error; + } if (!wrapperClient) { throw new TRPCError({ code: 'NOT_FOUND', message: 'No wrapper found for session' }); } diff --git a/services/cloud-agent-next/src/sandbox-id.test.ts b/services/cloud-agent-next/src/sandbox-id.test.ts index 8f24f35a6d..85c99c8ce4 100644 --- a/services/cloud-agent-next/src/sandbox-id.test.ts +++ b/services/cloud-agent-next/src/sandbox-id.test.ts @@ -7,6 +7,7 @@ import { getOutboundContainerId, getSandboxNamespace, isOrgInList, + selectSandboxForNewSession, } from './sandbox-id.js'; import type { Env, SandboxId } from './types.js'; @@ -424,6 +425,45 @@ describe('generateSandboxId', () => { }); }); +describe('selectSandboxForNewSession', () => { + it('selects Cloudflare by default while preserving shared sandbox allocation', async () => { + const selection = await selectSandboxForNewSession({ + env: {}, + orgId: 'org-id', + userId: 'user-id', + sessionId: 'session-id', + }); + + expect(selection.provider).toBe('cloudflare'); + expect(selection.sandboxId).toMatch(/^org-/); + }); + + it('selects Cloudflare for isolated per-session organizations', async () => { + const selection = await selectSandboxForNewSession({ + env: { PER_SESSION_SANDBOX_ORG_IDS: 'org-id' }, + orgId: 'org-id', + userId: 'user-id', + sessionId: 'session-id', + }); + + expect(selection.provider).toBe('cloudflare'); + expect(selection.sandboxId).toMatch(/^ses-/); + }); + + it('keeps devcontainer sessions on Cloudflare DIND allocation', async () => { + const selection = await selectSandboxForNewSession({ + env: { PER_SESSION_SANDBOX_ORG_IDS: 'org-id' }, + orgId: 'org-id', + userId: 'user-id', + sessionId: 'session-id', + devcontainer: true, + }); + + expect(selection.provider).toBe('cloudflare'); + expect(selection.sandboxId).toMatch(/^dind-/); + }); +}); + describe('getSandboxNamespace', () => { const mockSandbox = {} as DurableObjectNamespace; const mockSandboxContainment = {} as DurableObjectNamespace; diff --git a/services/cloud-agent-next/src/sandbox-id.ts b/services/cloud-agent-next/src/sandbox-id.ts index 324db5199b..d69e69ae23 100644 --- a/services/cloud-agent-next/src/sandbox-id.ts +++ b/services/cloud-agent-next/src/sandbox-id.ts @@ -1,4 +1,4 @@ -import type { SandboxId, Env } from './types.js'; +import type { AgentSandboxProvider, SandboxId, Env } from './types.js'; import type { Sandbox } from '@cloudflare/sandbox'; export const MANAGED_SCM_OUTBOUND_HANDLER = 'managedScm'; @@ -130,6 +130,38 @@ export async function deriveSharedSandboxId( return hashToSandboxId(`${suffix}:${routeKey}`, getSharedSandboxPrefix(routeKey)); } +export type SandboxSelection = { + sandboxId: SandboxId; + provider: AgentSandboxProvider; +}; + +export type SandboxSelectionEnv = { + PER_SESSION_SANDBOX_ORG_IDS?: string; +}; + +type SelectSandboxForNewSessionInput = { + env: SandboxSelectionEnv; + orgId?: string; + userId: string; + sessionId: string; + botId?: string; + devcontainer?: boolean; +}; + +export async function selectSandboxForNewSession( + input: SelectSandboxForNewSessionInput +): Promise { + const sandboxId = await generateSandboxId( + input.env.PER_SESSION_SANDBOX_ORG_IDS, + input.orgId, + input.userId, + input.sessionId, + input.botId, + input.devcontainer + ); + return { sandboxId, provider: 'cloudflare' }; +} + /** * Generate a deterministic, Cloudflare-compatible sandboxId (≤63 chars). * diff --git a/services/cloud-agent-next/src/server.test.ts b/services/cloud-agent-next/src/server.test.ts index 3380cdba1b..d3001c7d7c 100644 --- a/services/cloud-agent-next/src/server.test.ts +++ b/services/cloud-agent-next/src/server.test.ts @@ -285,7 +285,6 @@ describe('server /terminal', () => { { algorithm: 'HS256', expiresIn: 60 } ); const env = createEnv(); - const sandboxId = `usr-${'a'.repeat(48)}`; const metadata = { metadataSchemaVersion: 2, identity: { @@ -295,7 +294,7 @@ describe('server /terminal', () => { }, auth: {}, workspace: { - sandboxId, + sandboxId: `usr-${'a'.repeat(48)}`, workspacePath: '/workspace/user/repo', }, lifecycle: { @@ -329,6 +328,10 @@ describe('server /terminal', () => { expect(getMetadata).toHaveBeenCalledTimes(1); expect(fetch).not.toHaveBeenCalled(); expect(getRunningTerminalClientMock).toHaveBeenCalledOnce(); + expect(connectTerminal).toHaveBeenCalledTimes(1); + const connectRequest = connectTerminal.mock.calls[0]?.[1]; + expect(connectRequest).toBeInstanceOf(Request); + expect(new URL((connectRequest as Request).url).pathname).toBe('/terminal'); expect(connectTerminal).toHaveBeenCalledWith('pty_123', request); }); diff --git a/services/cloud-agent-next/src/session-prepare.test.ts b/services/cloud-agent-next/src/session-prepare.test.ts index 799fea4e51..b91521575a 100644 --- a/services/cloud-agent-next/src/session-prepare.test.ts +++ b/services/cloud-agent-next/src/session-prepare.test.ts @@ -7,6 +7,7 @@ import * as schemas from './router/schemas.js'; const { generateSessionIdMock, generateSandboxRoutingTargetMock, + selectSandboxForNewSessionMock, createCliSessionMock, deleteCliSessionMock, createSessionReportMock, @@ -24,6 +25,10 @@ const { generateSandboxRoutingTargetMock: vi .fn() .mockResolvedValue({ kind: 'isolated', sandboxId: 'sb-test-123' }), + selectSandboxForNewSessionMock: vi.fn().mockResolvedValue({ + sandboxId: 'sb-test-123', + provider: 'cloudflare', + }), createCliSessionMock: vi.fn().mockResolvedValue({ created: true }), deleteCliSessionMock: vi.fn().mockResolvedValue({ deleted: true }), createSessionReportMock: vi.fn().mockResolvedValue(undefined), @@ -66,6 +71,7 @@ vi.mock('./sandbox-id.js', async importOriginal => { return { ...actual, generateSandboxRoutingTarget: generateSandboxRoutingTargetMock, + selectSandboxForNewSession: selectSandboxForNewSessionMock, getSandboxNamespace: vi.fn(), }; }); @@ -265,6 +271,10 @@ describe('prepareSession endpoint', () => { kind: 'isolated', sandboxId: 'sb-test-123', }); + selectSandboxForNewSessionMock.mockResolvedValue({ + sandboxId: 'sb-test-123', + provider: 'cloudflare', + }); createCliSessionMock.mockResolvedValue({ created: true }); deleteCliSessionMock.mockResolvedValue({ deleted: true }); createSessionReportMock.mockResolvedValue(undefined); @@ -533,6 +543,7 @@ describe('prepareSession endpoint', () => { }, workspace: { sandboxId: 'sb-test-123', + sandboxProvider: 'cloudflare', shallow: true, credentialContainment: { github: true, gitlab: false, kilocode: false }, }, @@ -765,6 +776,7 @@ describe('prepareSession endpoint', () => { expect.objectContaining({ workspace: { sandboxId: failoverSandboxId, + sandboxProvider: 'cloudflare', shallow: undefined, credentialContainment: { github: true, gitlab: false, kilocode: false }, sandboxRoute: { @@ -786,6 +798,10 @@ describe('prepareSession endpoint', () => { kind: 'isolated', sandboxId: 'dind-abcdef', }); + selectSandboxForNewSessionMock.mockResolvedValueOnce({ + sandboxId: 'dind-abcdef', + provider: 'cloudflare', + }); const doStub = createMockDOStub(); const caller = appRouter.createCaller(createInternalApiContext({ doStub })); @@ -809,10 +825,21 @@ describe('prepareSession endpoint', () => { createdOnPlatform: undefined, } ); + expect(selectSandboxForNewSessionMock).toHaveBeenCalledWith( + expect.objectContaining({ + env: expect.any(Object), + orgId: undefined, + userId: 'test-user-123', + sessionId: 'agent_12345678-1234-1234-1234-123456789abc', + botId: undefined, + devcontainer: true, + }) + ); expect(doStub.createSessionWithInitialAdmission).toHaveBeenCalledWith( expect.objectContaining({ workspace: { sandboxId: 'dind-abcdef', + sandboxProvider: 'cloudflare', shallow: false, credentialContainment: { github: false, gitlab: false, kilocode: false }, devcontainerRequested: true, @@ -1216,6 +1243,10 @@ describe('start endpoint', () => { kind: 'isolated', sandboxId: 'sb-test-123', }); + selectSandboxForNewSessionMock.mockResolvedValue({ + sandboxId: 'sb-test-123', + provider: 'cloudflare', + }); createCliSessionMock.mockResolvedValue({ created: true }); deleteCliSessionMock.mockResolvedValue({ deleted: true }); createSessionReportMock.mockResolvedValue(undefined); @@ -1496,6 +1527,7 @@ describe('start endpoint', () => { ).rejects.toThrow('session report unavailable'); expect(generateSandboxRoutingTargetMock).not.toHaveBeenCalled(); + expect(selectSandboxForNewSessionMock).not.toHaveBeenCalled(); expect(createCliSessionMock).not.toHaveBeenCalled(); expect(recordVisibleSessionOutcomeMock).not.toHaveBeenCalled(); expect(recordInitialAdmissionMock).not.toHaveBeenCalled(); diff --git a/services/cloud-agent-next/src/session-service.test.ts b/services/cloud-agent-next/src/session-service.test.ts index 6a46a5215f..003bc55e39 100644 --- a/services/cloud-agent-next/src/session-service.test.ts +++ b/services/cloud-agent-next/src/session-service.test.ts @@ -966,6 +966,9 @@ describe('SessionService.prepareWorkspace', () => { 'agent_test', { inspectContainers: true } ); + expect(workspaceMocks.setupWorkspace).not.toHaveBeenCalled(); + expect(sandbox.createSessionMock).not.toHaveBeenCalled(); + expect(devcontainerMocks.bringUpDevContainer).not.toHaveBeenCalled(); }); it('hydrates requested devcontainer metadata while preparing a cold DIND workspace', async () => { diff --git a/services/cloud-agent-next/src/session/agent-runtime.test.ts b/services/cloud-agent-next/src/session/agent-runtime.test.ts index 30e7daddf0..d8678b93d9 100644 --- a/services/cloud-agent-next/src/session/agent-runtime.test.ts +++ b/services/cloud-agent-next/src/session/agent-runtime.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { AgentSandbox } from '../agent-sandbox/protocol.js'; +import { AgentSandboxUnavailableError, type AgentSandbox } from '../agent-sandbox/protocol.js'; import type { Env } from '../types.js'; import type { FencedWrapperDispatchRequest, @@ -44,7 +44,7 @@ function createMemoryStorage( } as MemoryRuntimeStorage & DurableObjectStorage; } -function createMetadata(): SessionMetadata { +function createMetadata(provider: 'cloudflare' = 'cloudflare'): SessionMetadata { return { metadataSchemaVersion: 2, identity: { @@ -61,6 +61,7 @@ function createMetadata(): SessionMetadata { }, workspace: { sandboxId: 'ses-runtime', + sandboxProvider: provider, workspacePath: '/workspace/runtime', sessionHome: '/home/agent_runtime', branchName: 'main', @@ -104,6 +105,32 @@ function createWorkspaceReady(): WorkspaceReady { } describe('AgentRuntime', () => { + it('returns a permanent delivery failure for an intentionally unavailable sandbox capability', async () => { + const createSandbox = vi.fn( + () => + ({ + discoverSessionWrappers: async () => { + throw new AgentSandboxUnavailableError('Sandbox runtime is unavailable'); + }, + }) as unknown as AgentSandbox + ); + const runtime = createAgentRuntime({ + storage: createMemoryStorage(), + env: { WORKER_URL: 'http://worker.test' } as Env, + getMetadata: async () => createMetadata(), + createAgentSandbox: createSandbox, + getSessionIdForLogs: () => 'agent_runtime', + sendToWrapper: () => false, + }); + + await expect(runtime.send(createPlan(createMetadata()))).resolves.toEqual({ + success: false, + code: 'SANDBOX_CAPABILITY_UNAVAILABLE', + error: 'Sandbox runtime delivery is unavailable for this session', + }); + expect(createSandbox).toHaveBeenCalledWith(createMetadata()); + }); + it('preflights cold delivery, authorizes its physical lease, and returns the existing queue result shape', async () => { const storage = createMemoryStorage(); const ready = createWorkspaceReady(); @@ -1024,6 +1051,31 @@ describe('AgentRuntime', () => { await expect(runtime.interruptWrapper()).resolves.toEqual({ commandSent: false }); }); + it('rejects delivery and suppresses keepalive when DO deletion intent fences runtime use', async () => { + const keepAlive = vi.fn().mockResolvedValue(undefined); + const execute = vi.fn().mockResolvedValue({ kiloSessionId: 'kilo_runtime' }); + const runtime = createAgentRuntime({ + storage: createMemoryStorage(), + env: {} as Env, + getMetadata: async () => createMetadata(), + canUseSandboxRuntime: async () => false, + getOrchestratorOverride: () => ({ execute }), + getSessionIdForLogs: () => 'agent_runtime', + sendToWrapper: () => false, + createAgentSandbox: () => ({ keepAlive }) as unknown as AgentSandbox, + }); + + await expect(runtime.send(createPlan())).resolves.toEqual({ + success: false, + code: 'INTERNAL', + error: 'Session deletion is in progress', + }); + await runtime.keepSandboxAlive(); + + expect(execute).not.toHaveBeenCalled(); + expect(keepAlive).not.toHaveBeenCalled(); + }); + it('keeps the runtime sandbox alive through AgentSandbox transport controls', async () => { const keepAlive = vi.fn().mockResolvedValue(undefined); const sandbox = { keepAlive } as unknown as AgentSandbox; diff --git a/services/cloud-agent-next/src/session/agent-runtime.ts b/services/cloud-agent-next/src/session/agent-runtime.ts index c346fd8651..cf183352c9 100644 --- a/services/cloud-agent-next/src/session/agent-runtime.ts +++ b/services/cloud-agent-next/src/session/agent-runtime.ts @@ -1,10 +1,11 @@ import { ExecutionError } from '../execution/errors.js'; import { ExecutionOrchestrator } from '../execution/orchestrator.js'; import { createAgentSandbox } from '../agent-sandbox/factory.js'; -import type { - AgentSandbox, - WrapperInstanceLease, - WrapperObservation, +import { + AgentSandboxUnavailableError, + type AgentSandbox, + type WrapperInstanceLease, + type WrapperObservation, } from '../agent-sandbox/protocol.js'; import type { ExecutionResult, @@ -79,6 +80,7 @@ export type AgentRuntimeDependencies = { storage: DurableObjectStorage; env: WorkerEnv; getMetadata: () => Promise; + canUseSandboxRuntime?: () => Promise; getSessionIdForLogs: () => string | undefined; sendToWrapper: ( ingestTagId: string, @@ -129,8 +131,15 @@ function buildRuntimeAcceptanceResult( } export function createAgentRuntime(dependencies: AgentRuntimeDependencies): AgentRuntime { - const { storage, env, getMetadata, getSessionIdForLogs, sendToWrapper, getOrchestratorOverride } = - dependencies; + const { + storage, + env, + getMetadata, + canUseSandboxRuntime, + getSessionIdForLogs, + sendToWrapper, + getOrchestratorOverride, + } = dependencies; const resolveAgentSandbox = dependencies.createAgentSandbox ?? ((metadata: SessionMetadata) => createAgentSandbox(env, metadata)); @@ -170,10 +179,21 @@ export function createAgentRuntime(dependencies: AgentRuntimeDependencies): Agen throw cleanupBlockedError(current); } - const observeWrappers = () => - dependencies.discoverSessionWrappers - ? dependencies.discoverSessionWrappers(plan.workspace.metadata) - : resolveAgentSandbox(plan.workspace.metadata).discoverSessionWrappers(); + const observeWrappers = async (): Promise => { + try { + return await (dependencies.discoverSessionWrappers + ? dependencies.discoverSessionWrappers(plan.workspace.metadata) + : resolveAgentSandbox(plan.workspace.metadata).discoverSessionWrappers()); + } catch (error) { + if (error instanceof AgentSandboxUnavailableError) { + throw ExecutionError.sandboxCapabilityUnavailable( + 'Sandbox runtime delivery is unavailable for this session', + error + ); + } + throw error; + } + }; let allocatable = current; if (current.state === 'owns_wrapper') { const observation = await observeWrappers(); @@ -296,6 +316,9 @@ export function createAgentRuntime(dependencies: AgentRuntimeDependencies): Agen plan: MessageDeliveryRequest, hooks: AgentRuntimeSendHooks = {} ): Promise { + if (canUseSandboxRuntime && !(await canUseSandboxRuntime())) { + return { success: false, code: 'INTERNAL', error: 'Session deletion is in progress' }; + } const { sessionId } = plan.scope; const { turn, agent } = plan; const currentRuntimeState = await getWrapperRuntimeState(storage); @@ -306,8 +329,17 @@ export function createAgentRuntime(dependencies: AgentRuntimeDependencies): Agen error: 'Wrapper batch is finalizing', }; } + let physicalAuthorization; + try { + physicalAuthorization = await authorizePhysicalWrapper(plan); + } catch (error) { + if (error instanceof ExecutionError && error.code === 'SANDBOX_CAPABILITY_UNAVAILABLE') { + return { success: false, code: error.code, error: error.message }; + } + throw error; + } const { leasedInstance, allocatedPhysicalInstance, requiresFreshRunFence } = - await authorizePhysicalWrapper(plan); + physicalAuthorization; const previousRuntimeState = await getWrapperRuntimeState(storage); if ( (allocatedPhysicalInstance || requiresFreshRunFence) && @@ -478,6 +510,9 @@ export function createAgentRuntime(dependencies: AgentRuntimeDependencies): Agen .debug('Preserved existing runtime liveness after failed hot delivery'); } } + if (error instanceof ExecutionError && error.code === 'SANDBOX_CAPABILITY_UNAVAILABLE') { + return { success: false, code: error.code, error: error.message }; + } throw error; } } @@ -520,6 +555,7 @@ export function createAgentRuntime(dependencies: AgentRuntimeDependencies): Agen async function keepSandboxAlive(): Promise { try { + if (canUseSandboxRuntime && !(await canUseSandboxRuntime())) return; const metadata = await getMetadata(); if (!metadata) return; await resolveAgentSandbox(metadata).keepAlive(); diff --git a/services/cloud-agent-next/src/session/pending-messages.ts b/services/cloud-agent-next/src/session/pending-messages.ts index d5491000cd..ad2293f36e 100644 --- a/services/cloud-agent-next/src/session/pending-messages.ts +++ b/services/cloud-agent-next/src/session/pending-messages.ts @@ -1,6 +1,7 @@ import * as z from 'zod'; import type { ExecutionMode, + PermanentDeliveryResultCode, RetryableResultCode, SessionMessageIntent, } from '../execution/types.js'; @@ -75,6 +76,7 @@ const PendingFlushFailureCodeSchema = z.enum([ 'KILO_SERVER_FAILED', 'WRAPPER_START_FAILED', 'WRAPPER_FINALIZING', + 'SANDBOX_CAPABILITY_UNAVAILABLE', 'NOT_FOUND', 'BAD_REQUEST', 'INTERNAL', @@ -492,6 +494,7 @@ export async function recordPendingFlushFailure( policy: PendingFlushPolicy; code?: | RetryableResultCode + | PermanentDeliveryResultCode | 'NOT_FOUND' | 'BAD_REQUEST' | 'INTERNAL' @@ -573,6 +576,7 @@ export async function recordPendingFlushFailure( function isRetryableFlushCode( code: | RetryableResultCode + | PermanentDeliveryResultCode | 'NOT_FOUND' | 'BAD_REQUEST' | 'INTERNAL' diff --git a/services/cloud-agent-next/src/session/session-message-queue.test.ts b/services/cloud-agent-next/src/session/session-message-queue.test.ts index f7b384d674..cf9ca2e8d6 100644 --- a/services/cloud-agent-next/src/session/session-message-queue.test.ts +++ b/services/cloud-agent-next/src/session/session-message-queue.test.ts @@ -316,7 +316,39 @@ describe('recordPendingFlushFailure backoff progression', () => { }); describe('flushNextPendingSessionMessage', () => { - it('retries a queued flush after a pre-start failure without dropping the message', async () => { + it('does not retry a delivery whose sandbox capability is intentionally unavailable', async () => { + const storage = createMemoryStorage(); + await storePendingSessionMessage( + storage, + createPendingSessionMessage({ + messageId: FIRST_MESSAGE_ID, + role: 'user', + content: 'cannot deliver here', + createdAt: 1, + }) + ); + + const result = await flushNextPendingSessionMessage({ + storage, + now: 10, + getDeliveryContext: async () => createContext(), + validateModeAgainstRuntimeAgents: () => null, + deliver: async () => ({ + success: false, + code: 'SANDBOX_CAPABILITY_UNAVAILABLE', + error: 'Runtime delivery is disabled', + }), + }); + + expect(result).toMatchObject({ + type: 'failure', + attempts: 1, + exhausted: true, + nextFlushAttemptAt: undefined, + }); + }); + + it('retries an ordinary wrapper bootstrap failure without dropping the message', async () => { const storage = createMemoryStorage(); const message = createPendingSessionMessage({ messageId: FIRST_MESSAGE_ID, @@ -339,8 +371,8 @@ describe('flushNextPendingSessionMessage', () => { .fn<(_plan: MessageDeliveryRequest) => Promise>() .mockResolvedValueOnce({ success: false, - code: 'WORKSPACE_SETUP_FAILED', - error: 'workspace restore failed', + code: 'WRAPPER_START_FAILED', + error: 'wrapper bootstrap failed', }) .mockResolvedValueOnce({ success: true, diff --git a/services/cloud-agent-next/src/session/session-message-queue.ts b/services/cloud-agent-next/src/session/session-message-queue.ts index 6e51972592..c91daa9a86 100644 --- a/services/cloud-agent-next/src/session/session-message-queue.ts +++ b/services/cloud-agent-next/src/session/session-message-queue.ts @@ -202,6 +202,7 @@ function classifyDeliveryFailure(code: PendingFlushFailureCode | undefined): { return { failureStage: 'pre_dispatch', failureCode: 'invalid_delivery_request' }; case 'MODEL_MISSING': return { failureStage: 'pre_dispatch', failureCode: 'model_missing' }; + case 'SANDBOX_CAPABILITY_UNAVAILABLE': case 'WRAPPER_FINALIZING': case 'INTERNAL': case 'UNKNOWN': diff --git a/services/cloud-agent-next/src/session/session-registration.ts b/services/cloud-agent-next/src/session/session-registration.ts index aa12d54c18..ab3062b327 100644 --- a/services/cloud-agent-next/src/session/session-registration.ts +++ b/services/cloud-agent-next/src/session/session-registration.ts @@ -27,7 +27,12 @@ import { recordCloudAgentSandboxIdentity, recordCloudAgentSessionFailure, } from '../telemetry/session-reports.js'; -import { generateSandboxRoutingTarget, isOrgInList } from '../sandbox-id.js'; +import { + generateSandboxRoutingTarget, + isOrgInList, + selectSandboxForNewSession, + type SandboxSelection, +} from '../sandbox-id.js'; import { resolveSharedSandboxAssignment } from '../shared-sandbox-route.js'; import { generateKiloSessionId } from '../utils/kilo-session-id.js'; import { createMessageId } from './message-id.js'; @@ -57,6 +62,7 @@ export type SessionRegistrationResult = { kiloSessionId: string; sandboxId: SandboxId; sandboxRoute?: SharedSandboxRouteMetadata; + sandboxProvider: SandboxSelection['provider']; /** * Canonical initial turn reserved for a later legacy initiation request. */ @@ -168,10 +174,11 @@ async function allocateNewSession( }; let sandboxId: SandboxId; let sandboxRoute: SharedSandboxRouteMetadata | undefined; + let sandboxProvider: SandboxSelection['provider'] = 'cloudflare'; try { const target = await generateSandboxRoutingTarget( ctx.env.PER_SESSION_SANDBOX_ORG_IDS, - input.options?.kilocodeOrganizationId, + orgId, ctx.userId, cloudAgentSessionId, ctx.botId, @@ -192,7 +199,16 @@ async function allocateNewSession( ...(assignment.suffix ? { suffix: assignment.suffix } : {}), }; } else { - sandboxId = target.sandboxId; + const selection = await selectSandboxForNewSession({ + env: ctx.env, + orgId, + userId: ctx.userId, + sessionId: cloudAgentSessionId, + botId: ctx.botId, + devcontainer: input.runtime?.devcontainer, + }); + sandboxId = selection.sandboxId; + sandboxProvider = selection.provider; } } catch (error) { await recordCloudAgentSessionFailure( @@ -243,6 +259,7 @@ async function allocateNewSession( kiloSessionId, sandboxId, sandboxRoute, + sandboxProvider, initialTurn, credentialContainment, sessionService, @@ -293,6 +310,7 @@ function buildSessionRegistrationCommand( callback: input.options?.callbackTarget ? { target: input.options.callbackTarget } : undefined, workspace: { sandboxId: allocation.sandboxId, + sandboxProvider: allocation.sandboxProvider, shallow: input.options?.shallow, ...(allocation.sandboxRoute ? { sandboxRoute: allocation.sandboxRoute } : {}), credentialContainment: allocation.credentialContainment, @@ -422,6 +440,7 @@ export async function startNewSession( cloudAgentSessionId: allocation.cloudAgentSessionId, kiloSessionId: allocation.kiloSessionId, sandboxId: allocation.sandboxId, + sandboxProvider: allocation.sandboxProvider, admission, }; } diff --git a/services/cloud-agent-next/src/terminal/access.test.ts b/services/cloud-agent-next/src/terminal/access.test.ts index 69bf31f443..b456e4555c 100644 --- a/services/cloud-agent-next/src/terminal/access.test.ts +++ b/services/cloud-agent-next/src/terminal/access.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import type { AgentSandbox } from '../agent-sandbox/protocol.js'; +import { AgentSandboxUnavailableError, type AgentSandbox } from '../agent-sandbox/protocol.js'; import type { CloudAgentSessionState } from '../persistence/types.js'; import type { Env } from '../types.js'; import { resolveTerminalWrapperClient, validateTerminalMetadata } from './access.js'; @@ -112,7 +112,7 @@ function sandboxWithTerminalResult( } describe('resolveTerminalWrapperClient', () => { - it('returns the ready terminal client from AgentSandbox', async () => { + it('returns the ready terminal client from the selected AgentSandbox', async () => { const client = { health: vi.fn(), createTerminal: vi.fn(), @@ -137,7 +137,7 @@ describe('resolveTerminalWrapperClient', () => { expect(getRunningTerminalClient).toHaveBeenCalledOnce(); }); - it('returns unavailable when no wrapper process is running', async () => { + it('preserves not-running as a distinct terminal-unavailable outcome', async () => { const result = await resolveTerminalWrapperClient( { env: {} as Env, @@ -180,4 +180,49 @@ describe('resolveTerminalWrapperClient', () => { error: 'Terminal is unavailable because the session wrapper is not healthy', }); }); + + it('surfaces the provider capability-unavailable outcome reported by the sandbox', async () => { + const getRunningTerminalClient = vi.fn().mockResolvedValue({ + status: 'capability-unavailable', + message: 'Terminal access is unavailable for this sandbox provider', + }); + const createSandbox = vi + .fn() + .mockReturnValue(sandboxWithTerminalResult(getRunningTerminalClient)); + const result = await resolveTerminalWrapperClient( + { + env: {} as Env, + metadata: baseMetadata, + sessionId: baseMetadata.identity.sessionId, + }, + { createSandbox } + ); + + expect(result).toEqual({ + success: false, + error: 'Terminal access is unavailable for this sandbox provider', + }); + }); + + it('maps sandbox construction failures to a terminal-unavailable outcome', async () => { + const createSandbox = vi.fn(() => { + throw new AgentSandboxUnavailableError( + 'Sandbox operational configuration is incomplete', + 'provider_not_configured' + ); + }); + const result = await resolveTerminalWrapperClient( + { + env: {} as Env, + metadata: baseMetadata, + sessionId: baseMetadata.identity.sessionId, + }, + { createSandbox } + ); + + expect(result).toEqual({ + success: false, + error: 'Sandbox operational configuration is incomplete', + }); + }); }); diff --git a/services/cloud-agent-next/src/terminal/access.ts b/services/cloud-agent-next/src/terminal/access.ts index ff5bd35fab..97f95e72e9 100644 --- a/services/cloud-agent-next/src/terminal/access.ts +++ b/services/cloud-agent-next/src/terminal/access.ts @@ -1,5 +1,5 @@ import { createAgentSandbox } from '../agent-sandbox/factory.js'; -import type { AgentSandbox } from '../agent-sandbox/protocol.js'; +import { AgentSandboxUnavailableError, type AgentSandbox } from '../agent-sandbox/protocol.js'; import type { WrapperHealthResponse, WrapperPty } from '../kilo/wrapper-client.js'; import type { CloudAgentSessionState, OperationResult } from '../persistence/types.js'; import type { Env } from '../types.js'; @@ -68,9 +68,20 @@ export async function resolveTerminalWrapperClient( return { success: false, error: metadataResult.error }; } - const terminal = await deps - .createSandbox(params.env, metadataResult.data.metadata) - .getRunningTerminalClient(); + let terminal: Awaited>; + try { + terminal = await deps + .createSandbox(params.env, metadataResult.data.metadata) + .getRunningTerminalClient(); + } catch (error) { + if (error instanceof AgentSandboxUnavailableError) { + return { success: false, error: error.message }; + } + throw error; + } + if (terminal.status === 'capability-unavailable') { + return { success: false, error: terminal.message }; + } if (terminal.status === 'not-running') { return { success: false, diff --git a/services/cloud-agent-next/src/types.ts b/services/cloud-agent-next/src/types.ts index 03dcd60f6f..e9824e2a23 100644 --- a/services/cloud-agent-next/src/types.ts +++ b/services/cloud-agent-next/src/types.ts @@ -124,6 +124,8 @@ export type SandboxId = | `${string}__${string}` | `${string}__${string}__${string}`; +export type AgentSandboxProvider = 'cloudflare'; + /** Unique identifier for a session within a sandbox */ export type SessionId = `agent_${string}`; @@ -518,7 +520,7 @@ export type Env = { GITHUB_APP_SLUG?: string; /** GitHub App bot user ID for git commit email (e.g., '240665456') */ GITHUB_APP_BOT_USER_ID?: string; - /** Comma-separated org IDs that use per-session sandbox containers */ + /** Comma-separated org IDs that use per-session Cloudflare sandbox containers */ PER_SESSION_SANDBOX_ORG_IDS?: string; /** Comma-separated org IDs whose GitHub token uses credential containment, or `*` for all orgs */ GITHUB_TOKEN_CONTAINMENT_ORG_IDS?: string; diff --git a/services/cloud-agent-next/test/e2e/README.md b/services/cloud-agent-next/test/e2e/README.md index 6fd71b3492..b5e723411f 100644 --- a/services/cloud-agent-next/test/e2e/README.md +++ b/services/cloud-agent-next/test/e2e/README.md @@ -43,6 +43,15 @@ cloud-agent-next refactor. ## Running +> **Non-zero port offset:** the commands below use the default ports +> (`8794`/`8811`), which only match a zero-offset session. For any other +> session, first read the offset from `pnpm dev:status --json` +> (`portOffset` field), then prefix every driver invocation with +> `WORKER_URL=http://localhost:<8794 + portOffset>` and +> `FAKE_LLM_URL=http://localhost:<8811 + portOffset>`. Without these the +> driver silently hits the wrong Worker/fake-LLM and every scenario fails +> at connection. See the env-var table below for the full list. + Official SDK basic-chat acceptance (pinned `@kilocode/sdk/v2` `7.3.54`): ```bash @@ -111,7 +120,9 @@ in local dev, so the harness identifies each newly-created sandbox instead of killing every sandbox between cases. Kill scenarios only terminate the sandbox family created for that scenario. -Per-run overrides via env vars: +Per-run overrides via env vars. Defaults assume a zero-offset session; +for any other offset, compute the real ports from `pnpm dev:status --json` +(worker = `8794 + portOffset`, fake-LLM = `8811 + portOffset`): | Var | Default | |---|---| @@ -237,7 +248,33 @@ the newer `start` / `send` procedures. `prepareSession` requires confirm kilo is hitting it. - **`waitForGateEngaged` timed out** — kilo never reached the fake LLM. Most common cause: `KILO_OPENROUTER_BASE` still points at a real provider or the - fake service is not running. + fake service is not running. Confirm with `curl -s $FAKE_LLM_URL/test/requests` + (expect a rising `chatCompletions` count as kilo dials the fake) and + `tail -f dev/logs/fake-llm.log` — a stream that stays empty while a turn is + "preparing" means the wrapper never started, not a fake-LLM problem. +- **`Worker "git-token-service-dev" not found` in `cloud-agent-next.log`** — + the `GIT_TOKEN_SERVICE` service binding could not resolve. The Worker log + shows the failure as `Failed to issue Kilo session capability` and the turn + terminates immediately with `cloud.message.failed`. Cause: the + `cloudflare-git-token-service` dev process is up on its port but stale and + not heartbeating into the shared dev-registry (check + `.wrangler/dev-registry/` for a missing `git-token-service-dev` entry). Fix: + `pnpm dev:restart cloudflare-git-token-service`, then confirm the entry + reappears. The fake LLM is irrelevant here — kilo never gets far enough to + dial it. +- **Matrix fails intermittently with `preparing×N` and no terminal** — + environmental, not a regression. The `@cloudflare/containers` library's + container control connection sometimes returns 503 under Docker Desktop + load, triggering exponential-backoff retries that consume the scenario + timeout. The `smoke.ts` matrix now kills stale containers before starting + and uses a 120s per-scenario timeout, but this is not always enough. If + the matrix is flaky: (1) stop any competing dev session from another + worktree that also runs Cloud Agent sandboxes; (2) prune stopped containers + (`docker ps -a --filter status=exited --format '{{.Names}}' | rg + workerd-cloud-agent | xargs -r docker rm -f`); (3) restart + `cloud-agent-next` to clear stale DO alarm timers; (4) re-run the failing + scenario standalone — if it passes alone, the matrix failure was Docker + contention, not code. - **`releaseGate` returned 404** — the gate already went away, usually because the wrapper's request was aborted (e.g. by an `interruptSession`). Queue-interrupt-clears tolerates this; other scenarios treat it as an diff --git a/services/cloud-agent-next/test/e2e/client.ts b/services/cloud-agent-next/test/e2e/client.ts index 582d40b5e2..d00060404d 100644 --- a/services/cloud-agent-next/test/e2e/client.ts +++ b/services/cloud-agent-next/test/e2e/client.ts @@ -369,7 +369,7 @@ export async function fetchFakeRequests(fakeLlmUrl: string): Promise { const base = config.fakeLlmUrl.replace(/\/$/, ''); diff --git a/services/cloud-agent-next/test/e2e/lifecycle.ts b/services/cloud-agent-next/test/e2e/lifecycle.ts index bcbc567e17..e4a89b6587 100644 --- a/services/cloud-agent-next/test/e2e/lifecycle.ts +++ b/services/cloud-agent-next/test/e2e/lifecycle.ts @@ -54,7 +54,12 @@ export type LifecycleArgs = { * surface the web UI still uses. */ api?: ApiVersion; - /** Overall per-scenario timeout. Conservative default for cold-boot paths. */ + /** + * Overall per-scenario timeout. Conservative default for cold-boot paths. + * 120s gives margin over the wrapper startup path (3 attempts × 30s + * waitForPort) plus ensureSessionReady, which can collectively approach 90s + * under Docker contention. + */ timeoutMs?: number; }; @@ -91,7 +96,7 @@ async function snapshotSandboxIds(): Promise> { */ export async function lifecycleCold(args: LifecycleArgs): Promise { const start = Date.now(); - const { config, conversation, timeoutMs = 90_000, api = 'unified' } = args; + const { config, conversation, timeoutMs = 120_000, api = 'unified' } = args; try { const knownSandboxIds = await snapshotSandboxIds(); const sessionResult = await startSession(config, { prompt: fakeDirective(conversation) }, api); @@ -268,7 +273,7 @@ export async function lifecycleFollowup(args: LifecycleArgs): Promise { const start = Date.now(); - const { config, conversation, timeoutMs = 90_000, api = 'unified' } = args; + const { config, conversation, timeoutMs = 120_000, api = 'unified' } = args; const coldDirective = conversation && conversation !== '_' ? conversation : 'echo:hi'; const hotDirectives = ['echo:hot', 'slow:3:50', 'echo:followup']; const events: StreamEvent[] = []; @@ -371,7 +376,7 @@ export async function lifecycleColdHot(args: LifecycleArgs): Promise { const start = Date.now(); - const { config, conversation, timeoutMs = 90_000, api = 'unified' } = args; + const { config, conversation, timeoutMs = 120_000, api = 'unified' } = args; try { // 1. Bring a sandbox up with a warm-up echo. const knownSandboxIds = await snapshotSandboxIds(); @@ -462,7 +467,7 @@ export async function lifecycleExternalKill(args: LifecycleArgs): Promise { const start = Date.now(); - const { config, conversation, timeoutMs = 90_000, api = 'unified' } = args; + const { config, conversation, timeoutMs = 120_000, api = 'unified' } = args; const gateTag = 'killmid'; try { const knownSandboxIds = await snapshotSandboxIds(); @@ -484,7 +489,7 @@ export async function lifecycleKillMidFlight(args: LifecycleArgs): Promise { const start = Date.now(); - const { config, conversation, timeoutMs = 90_000, api = 'unified' } = args; + const { config, conversation, timeoutMs = 120_000, api = 'unified' } = args; const scenarioName = 'callback-completion'; const directive = conversation || 'echo:done'; const expectedText = directive.startsWith('echo:') ? directive.slice('echo:'.length) : undefined; @@ -1704,7 +1709,7 @@ export async function lifecycleCallbackBatchFollowup( }; } - const engaged = await waitForGateEngaged(config, gateTag, 90_000); + const engaged = await waitForGateEngaged(config, gateTag, 120_000); if (!engaged) { const events = [...stream.events]; stream.close(); @@ -1916,7 +1921,7 @@ export async function lifecycleCallbackBatchFollowup( */ export async function lifecycleCallbackInterrupt(args: LifecycleArgs): Promise { const start = Date.now(); - const { config, timeoutMs = 90_000, api = 'unified' } = args; + const { config, timeoutMs = 120_000, api = 'unified' } = args; const scenarioName = 'callback-interrupt'; const gateTag = 'callback-interrupt'; let sink: CallbackServerHandle | null = null; @@ -1946,7 +1951,7 @@ export async function lifecycleCallbackInterrupt(args: LifecycleArgs): Promise { model: process.env.E2E_MODEL ?? DEFAULT_CONFIG.model, }; + // Kill stale sandbox containers from previous runs before starting. + // Accumulated stopped/running containers degrade Docker Desktop performance + // and cause the preparing×7 wrapper-startup stall pattern. The baseline + // snapshot below only identifies *new* containers, so leftovers from prior + // runs would be skipped by cleanupMatrixSandboxes. + const staleContainers = await listSandboxContainers(); + if (staleContainers.length > 0) { + console.log(`smoke: cleaning ${staleContainers.length} stale sandbox container(s)`); + for (const container of staleContainers) { + await killSandboxFamily(container); + await waitForSandboxFamilyGone(container, 30_000); + } + } + const baselineSandboxIds = new Set( (await listSandboxContainers()).map(container => container.id) ); diff --git a/services/cloud-agent-next/test/integration/session/deletion-lifecycle.test.ts b/services/cloud-agent-next/test/integration/session/deletion-lifecycle.test.ts index 224173e7d8..196623e2f6 100644 --- a/services/cloud-agent-next/test/integration/session/deletion-lifecycle.test.ts +++ b/services/cloud-agent-next/test/integration/session/deletion-lifecycle.test.ts @@ -1,5 +1,6 @@ import { env, runInDurableObject } from 'cloudflare:test'; import { describe, expect, it } from 'vitest'; +import type { AgentSandboxLifecycle } from '../../../src/agent-sandbox/protocol.js'; import type { CloudAgentSession } from '../../../src/persistence/CloudAgentSession.js'; import { listPendingSessionMessages } from '../../../src/session/pending-messages.js'; import { getWrapperLease } from '../../../src/session/wrapper-runtime-state.js'; @@ -80,13 +81,17 @@ describe('session deletion physical cleanup', () => { ); const result = { metadata: await instance.getMetadata(), + storedMetadata: await instance.ctx.storage.get<{ identity: { sessionId: string } }>( + 'metadata' + ), lease: await getWrapperLease(instance.ctx.storage), }; await instance.ctx.storage.deleteAll(); return result; }); - expect(result.metadata?.identity.sessionId).toBe(sessionId); + expect(result.metadata).toBeNull(); + expect(result.storedMetadata?.identity.sessionId).toBe(sessionId); expect(result.lease).toMatchObject({ state: 'stop_needed', reason: 'session-delete', @@ -181,6 +186,9 @@ describe('session deletion physical cleanup', () => { ); const result = { metadata: await instance.getMetadata(), + storedMetadata: await instance.ctx.storage.get<{ identity: { sessionId: string } }>( + 'metadata' + ), alarm: await instance.ctx.storage.getAlarm(), nextAttemptAt, }; @@ -188,7 +196,8 @@ describe('session deletion physical cleanup', () => { return result; }); - expect(result.metadata?.identity.sessionId).toBe(sessionId); + expect(result.metadata).toBeNull(); + expect(result.storedMetadata?.identity.sessionId).toBe(sessionId); expect(result.alarm).toBe(result.nextAttemptAt); }); @@ -220,7 +229,8 @@ describe('session deletion physical cleanup', () => { expect(result.lease).toMatchObject({ state: 'stop_needed', attempts: 1 }); if (result.lease.state !== 'stop_needed') throw new Error('Expected pending wrapper cleanup'); - expect(result.alarm).toBe(result.lease.nextAttemptAt); + expect(result.alarm).not.toBeNull(); + expect(result.alarm).toBeLessThanOrEqual(result.lease.nextAttemptAt); }); it('rejects new message admission while explicit deletion is pending', async () => { @@ -277,16 +287,16 @@ describe('session deletion physical cleanup', () => { }; }); - const deletionPending = { + const deletedSession = { success: false, code: 'NOT_FOUND', - error: 'Session deletion is pending', + error: 'Session not found', }; expect(result).toEqual({ - submittedAdmission: deletionPending, - groupedInitialAdmission: deletionPending, - preparedInitialAdmission: deletionPending, - registration: { success: false, error: 'Session deletion is pending' }, + submittedAdmission: deletedSession, + groupedInitialAdmission: deletedSession, + preparedInitialAdmission: deletedSession, + registration: { success: false, error: 'Session is deleted' }, pendingMessages: [], }); }); @@ -364,13 +374,17 @@ describe('session deletion physical cleanup', () => { await instance.alarm(); const result = { metadata: await instance.getMetadata(), + storedMetadata: await instance.ctx.storage.get<{ identity: { sessionId: string } }>( + 'metadata' + ), lease: await getWrapperLease(instance.ctx.storage), }; await instance.ctx.storage.deleteAll(); return result; }); - expect(result.metadata?.identity.sessionId).toBe(sessionId); + expect(result.metadata).toBeNull(); + expect(result.storedMetadata?.identity.sessionId).toBe(sessionId); expect(result.lease).toMatchObject({ state: 'stop_needed', reason: 'session-delete', @@ -417,4 +431,45 @@ describe('session deletion physical cleanup', () => { expect(result.lease).toMatchObject({ state: 'stop_needed', attempts: 1 }); expect(result.alarm).toBe(result.nextAttemptAt); }); + + it('retains provider deferred-deletion entries through the payload purge', async () => { + const userId = 'user_delete_deferred'; + const sessionId = 'agent_delete_deferred'; + const stub = env.CLOUD_AGENT_SESSION.get( + env.CLOUD_AGENT_SESSION.idFromName(`${userId}:${sessionId}`) + ); + + const result = await runInDurableObject(stub, async instance => { + await registerReadySession(instance, { + sessionId, + userId, + prompt: 'deferred deletion', + mode: 'code', + model: 'test-model', + }); + await instance.ctx.storage.put('sensitive_payload', { token: 'secret' }); + + instance['sandboxLifecycle'] = { + reconcileCreateIntent: async () => undefined, + planDeletion: async () => ({ + kind: 'deferred', + entries: { provider_deletion_state: { remoteId: 'rm-123', status: 'pending' } }, + }), + reconcilePendingDeletion: async () => 'none', + } satisfies AgentSandboxLifecycle; + + await instance.deleteSession(); + return { + deletionIntent: await instance.ctx.storage.get('session_deletion_intent'), + providerState: await instance.ctx.storage.get('provider_deletion_state'), + metadata: await instance.ctx.storage.get('metadata'), + sensitivePayload: await instance.ctx.storage.get('sensitive_payload'), + }; + }); + + expect(result.deletionIntent).toEqual({ reason: 'explicit', startedAt: expect.any(Number) }); + expect(result.providerState).toEqual({ remoteId: 'rm-123', status: 'pending' }); + expect(result.metadata).toBeUndefined(); + expect(result.sensitivePayload).toBeUndefined(); + }); }); diff --git a/services/cloud-agent-next/test/integration/session/pending-messages.test.ts b/services/cloud-agent-next/test/integration/session/pending-messages.test.ts index 7fb734d39d..3e904d61ab 100644 --- a/services/cloud-agent-next/test/integration/session/pending-messages.test.ts +++ b/services/cloud-agent-next/test/integration/session/pending-messages.test.ts @@ -334,6 +334,13 @@ describe('pending session messages', () => { const result = await runInDurableObject(stub, async instance => { let acceptedMessageId: string | undefined; + const volatileEvents: Array<{ streamEventType: string; payload: string }> = []; + (instance as any).broadcastVolatileEvent = (event: { + streamEventType: string; + payload: string; + }) => { + volatileEvents.push(event); + }; (instance as any).orchestrator = { execute: async (plan: any) => { acceptedMessageId = plan.turn.messageId; @@ -364,13 +371,21 @@ describe('pending session messages', () => { await instance.alarm(); const pending = await listPendingSessionMessages(instance.ctx.storage); const acceptedMessages = await listNonTerminalAcceptedMessages(instance.ctx.storage); - return { acceptedMessageId, pending, acceptedMessages }; + return { acceptedMessageId, pending, acceptedMessages, volatileEvents }; }); expect(result.acceptedMessageId).toBe('msg_018f1e2d3c4bFlushMsgAbCdEf'); expect(result.pending).toHaveLength(0); expect(result.acceptedMessages).toHaveLength(1); expect(result.acceptedMessages[0]?.messageId).toBe('msg_018f1e2d3c4bFlushMsgAbCdEf'); + expect(result.volatileEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + streamEventType: 'cloud.status', + payload: JSON.stringify({ cloudStatus: { type: 'ready' } }), + }), + ]) + ); }); it('flushes canonical document attachment descriptors through the durable queue plan', async () => { diff --git a/services/cloud-agent-next/test/integration/session/start-execution-v2.test.ts b/services/cloud-agent-next/test/integration/session/start-execution-v2.test.ts index b55454e068..2cdd43ae56 100644 --- a/services/cloud-agent-next/test/integration/session/start-execution-v2.test.ts +++ b/services/cloud-agent-next/test/integration/session/start-execution-v2.test.ts @@ -3,13 +3,14 @@ */ import { env, runInDurableObject } from 'cloudflare:test'; -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { createPendingSessionMessage, listPendingSessionMessages, storePendingSessionMessage, } from '../../../src/session/pending-messages.js'; import { listNonTerminalAcceptedMessages } from '../../../src/session/session-message-state.js'; + import { groupedRegisterSessionInput, queueRegisteredInitialInput, @@ -54,7 +55,6 @@ describe('CloudAgentSession message admission', () => { messageId, outcome: 'queued', compatibilityDelivery: 'queued', - compatibilityDelivery: 'queued', }); expect(result.metadata?.initialMessage?.id).toBe(messageId); expect(result.pending).toHaveLength(1); @@ -136,6 +136,132 @@ describe('CloudAgentSession message admission', () => { expect(result.metadata).toBeNull(); }); + it('routes explicit and retention Cloudflare deletion through the DO sandbox lifecycle owner', async () => { + const explicitStub = env.CLOUD_AGENT_SESSION.get( + env.CLOUD_AGENT_SESSION.idFromName('user_cf_explicit_delete:agent_cf_explicit_delete') + ); + const retentionStub = env.CLOUD_AGENT_SESSION.get( + env.CLOUD_AGENT_SESSION.idFromName('user_cf_retention_delete:agent_cf_retention_delete') + ); + + const reasons = await runInDurableObject(explicitStub, async instance => { + await instance.registerSession( + groupedRegisterSessionInput({ + sessionId: 'agent_cf_explicit_delete', + userId: 'user_cf_explicit_delete', + prompt: 'delete explicit', + mode: 'code', + model: 'test-model', + }) + ); + const captured: string[] = []; + vi.spyOn(instance as any, 'deleteSandboxSessionResources').mockImplementation( + async (_metadata: unknown, reason: unknown) => { + captured.push(String(reason)); + } + ); + await instance.deleteSession(); + return captured; + }); + + const retained = await runInDurableObject(retentionStub, async instance => { + await instance.registerSession( + groupedRegisterSessionInput({ + sessionId: 'agent_cf_retention_delete', + userId: 'user_cf_retention_delete', + prompt: 'delete retention', + mode: 'code', + model: 'test-model', + }) + ); + const captured: string[] = []; + vi.spyOn(instance as any, 'deleteSandboxSessionResources').mockImplementation( + async (_metadata: unknown, reason: unknown) => { + captured.push(String(reason)); + } + ); + await instance.ctx.storage.put('last_activity', Date.now() - 91 * 24 * 60 * 60 * 1000); + await instance.alarm(); + return { captured, metadata: await instance.getMetadata() }; + }); + + expect(reasons).toEqual(['explicit']); + expect(retained.captured).toEqual(['retention-expired']); + expect(retained.metadata).toBeNull(); + }); + + it('reconciles committed Cloudflare deletion intent and retains it until cleanup succeeds', async () => { + const stub = env.CLOUD_AGENT_SESSION.get( + env.CLOUD_AGENT_SESSION.idFromName('user_cf_intent_recovery:agent_cf_intent_recovery') + ); + + const result = await runInDurableObject(stub, async instance => { + await instance.registerSession({ + ...groupedRegisterSessionInput({ + sessionId: 'agent_cf_intent_recovery', + userId: 'user_cf_intent_recovery', + prompt: 'recover interrupted deletion', + mode: 'code', + model: 'test-model', + }), + workspace: { sandboxId: 'usr-acde1234', sandboxProvider: 'cloudflare' }, + }); + await instance.ctx.storage.put('session_deletion_intent', { + reason: 'explicit', + startedAt: 12_000, + }); + const cleanup = vi + .spyOn(instance as any, 'deleteSandboxSessionResources') + .mockRejectedValueOnce(new Error('temporary Cloudflare cleanup failure')) + .mockResolvedValueOnce(undefined); + + await instance.alarm(); + const retained = { + metadata: await instance.ctx.storage.get('metadata'), + deletionIntent: await instance.ctx.storage.get('session_deletion_intent'), + alarm: await instance.ctx.storage.getAlarm(), + }; + await instance.alarm(); + return { + retained, + cleanupReasons: cleanup.mock.calls.map(call => call[1]), + keys: [...(await instance.ctx.storage.list()).keys()], + }; + }); + + expect(result.retained.metadata).toBeDefined(); + expect(result.retained.deletionIntent).toMatchObject({ reason: 'explicit', startedAt: 12_000 }); + expect(result.retained.alarm).toEqual(expect.any(Number)); + expect(result.cleanupReasons).toEqual(['explicit', 'explicit']); + expect(result.keys).toEqual([]); + }); + + it('persists the registered sandbox provider selection', async () => { + const userId = 'user_cloudflare_selection' as const; + const sessionId = 'agent_cloudflare_selection' as const; + const doId = env.CLOUD_AGENT_SESSION.idFromName(`${userId}:${sessionId}`); + const stub = env.CLOUD_AGENT_SESSION.get(doId); + + const workspace = await runInDurableObject(stub, async instance => { + await instance.registerSession({ + ...groupedRegisterSessionInput({ + sessionId, + userId, + prompt: 'stay on Cloudflare', + mode: 'code', + model: 'test-model', + }), + workspace: { + sandboxId: 'usr-abcdef', + sandboxProvider: 'cloudflare', + }, + }); + return (await instance.getMetadata())?.workspace; + }); + + expect(workspace?.sandboxProvider).toBe('cloudflare'); + }); + it('surfaces initial admission failure after retaining registered DO metadata', async () => { const userId = 'user_grouped_start_failure' as const; const sessionId = 'agent_grouped_start_failure' as const; @@ -220,6 +346,85 @@ describe('CloudAgentSession message admission', () => { expect(result.pending[0]?.messageId).toBe(messageId); }); + it('does not mutate persisted sandbox identity when a registration replay changes selection', async () => { + const userId = 'user_provider_replay' as const; + const sessionId = 'agent_provider_replay' as const; + const messageId = 'msg_018f1e2d3c4bBoundMsgAbCdEf'; + const doId = env.CLOUD_AGENT_SESSION.idFromName(`${userId}:${sessionId}`); + const stub = env.CLOUD_AGENT_SESSION.get(doId); + + const result = await runInDurableObject(stub, async instance => { + const initialInput = { + ...groupedRegisterSessionInput({ + sessionId, + userId, + prompt: 'preserve provider', + mode: 'code', + model: 'test-model', + }), + workspace: { + sandboxId: 'usr-abcdef' as const, + sandboxProvider: 'cloudflare' as const, + }, + message: { + initialTurn: { + type: 'prompt' as const, + messageId, + prompt: 'preserve provider', + }, + }, + }; + await instance.createSessionWithInitialAdmission(initialInput); + await instance.createSessionWithInitialAdmission({ + ...initialInput, + workspace: { + sandboxId: 'ses-abcdef', + sandboxProvider: 'cloudflare', + }, + }); + return { + metadata: await instance.getMetadata(), + }; + }); + + expect(result.metadata?.workspace?.sandboxProvider).toBe('cloudflare'); + expect(result.metadata?.workspace?.sandboxId).toBe('usr-abcdef'); + }); + + it('rejects metadata updates that replace the registered sandbox name', async () => { + const stub = env.CLOUD_AGENT_SESSION.get( + env.CLOUD_AGENT_SESSION.idFromName('user_runtime_identity:agent_runtime_identity') + ); + + const result = await runInDurableObject(stub, async instance => { + await instance.registerSession({ + ...groupedRegisterSessionInput({ + sessionId: 'agent_runtime_identity', + userId: 'user_runtime_identity', + prompt: 'immutable runtime identity', + mode: 'code', + model: 'test-model', + }), + workspace: { sandboxId: 'ses-abcdef', sandboxProvider: 'cloudflare' }, + }); + const metadata = await instance.getMetadata(); + if (!metadata) throw new Error('Expected metadata after registration'); + let error: string | undefined; + try { + await instance.updateMetadata({ + ...metadata, + workspace: { ...metadata.workspace, sandboxId: 'ses-fedcba' }, + }); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + return { error, metadata: await instance.getMetadata() }; + }); + + expect(result.error).toBe('Registered sandbox name cannot be changed'); + expect(result.metadata?.workspace?.sandboxId).toBe('ses-abcdef'); + }); + it('rejects a grouped replay that changes the immutable initial intent', async () => { const userId = 'user_grouped_start_mismatch' as const; const sessionId = 'agent_grouped_start_mismatch' as const; @@ -282,6 +487,7 @@ describe('CloudAgentSession message admission', () => { ...base, workspace: { sandboxId, + sandboxProvider: 'cloudflare' as const, sandboxRoute: { kind: 'shared' as const, routeKey, diff --git a/services/cloud-agent-next/test/unit/execution/orchestrator.test.ts b/services/cloud-agent-next/test/unit/execution/orchestrator.test.ts index 4525d2d8a8..af8de6ac93 100644 --- a/services/cloud-agent-next/test/unit/execution/orchestrator.test.ts +++ b/services/cloud-agent-next/test/unit/execution/orchestrator.test.ts @@ -275,6 +275,13 @@ describe('ExecutionError', () => { expect(error.code).toBe('WRAPPER_JOB_CONFLICT'); expect(error.retryable).toBe(false); }); + + it('sandboxCapabilityUnavailable creates non-retryable error', () => { + const error = ExecutionError.sandboxCapabilityUnavailable('Runtime delivery is disabled'); + + expect(error.code).toBe('SANDBOX_CAPABILITY_UNAVAILABLE'); + expect(error.retryable).toBe(false); + }); }); it('preserves cause for debugging', () => { @@ -295,10 +302,11 @@ describe('ExecutionError', () => { 'INVALID_REQUEST', 'SESSION_NOT_FOUND', 'WRAPPER_JOB_CONFLICT', + 'SANDBOX_CAPABILITY_UNAVAILABLE', ]; expect(retryableCodes).toHaveLength(4); - expect(permanentCodes).toHaveLength(3); + expect(permanentCodes).toHaveLength(4); }); });