fix(cloud-agent-next): recover sandbox after wrapper cleanup exhaustion

services/cloud-agent-next/src/persistence/CloudAgentSession.ts

@@ -90,7 +90,11 @@ import type {
 import { renderExecutionTurnContent } from '../execution/types.js';
 import type { Env as WorkerEnv, SandboxId } from '../types.js';
 import { deriveSharedSandboxId, generateSandboxId } from '../sandbox-id.js';
-import { recordSharedSandboxFailover } from '../shared-sandbox-route.js';
+import {
+  recordSharedSandboxFailover,
+  resolveSharedSandboxAssignment,
+} from '../shared-sandbox-route.js';
+import type { SHARED_SANDBOX_FAILOVER_SUFFIX } from '../shared-sandbox-route.js';
 
 import { resolveSecret, validateStreamTicket } from '../auth.js';
 import { resolveTerminalWrapperClient, type TerminalWrapperClient } from '../terminal/access.js';
@@ -438,6 +442,7 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
   }) => Promise<StopWrappersResult>;
   private sandboxSessionDeleter?: (reason: 'explicit' | 'retention-expired') => Promise<void>;
   private ephemeralSandboxDestroyer?: () => Promise<void>;
+  private postExhaustionSandboxDestroyer?: () => Promise<void>;
   private sharedSandboxFailoverRecorder?: (routeKey: SandboxId) => Promise<void>;
   private agentRuntime?: AgentRuntime;
   private sandboxLifecycle?: AgentSandboxLifecycle;
@@ -768,6 +773,17 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
           this.sharedSandboxFailoverRecorder
             ? this.sharedSandboxFailoverRecorder(routeKey)
             : recordSharedSandboxFailover(this.env.SHARED_SANDBOX_OVERRIDES, routeKey),
+        resolveSharedSandboxFailover: routeKey =>
+          resolveSharedSandboxAssignment(this.env.SHARED_SANDBOX_OVERRIDES, routeKey),
+        persistSharedSandboxRecoveryAssignment: input =>
+          this.persistSharedSandboxRecoveryAssignment(input),
+        destroyIsolatedSandbox: async metadata => {
+          if (this.postExhaustionSandboxDestroyer) {
+            await this.postExhaustionSandboxDestroyer();
+            return;
+          }
+          await createAgentSandbox(this.env, metadata).delete('recovery');
+        },
         requestAlarmAtOrBefore: deadline => this.scheduleAlarmAtOrBefore(deadline),
         getSessionIdForLogs: () => this.sessionId,
       });
@@ -1272,6 +1288,51 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
     return metadata ? parseSessionMetadata(metadata) : null;
   }
 
+  private async persistSharedSandboxRecoveryAssignment(input: {
+    routeKey: SandboxId;
+    sourceSandboxId: SandboxId;
+    replacementSandboxId: SandboxId;
+    suffix: typeof SHARED_SANDBOX_FAILOVER_SUFFIX;
+  }): Promise<void> {
+    const metadata = await this.getStoredMetadata();
+    if (!metadata?.workspace) throw new Error('Session workspace metadata is unavailable');
+    const route = metadata.workspace.sandboxRoute;
+    if (route?.kind !== 'shared' || route.routeKey !== input.routeKey) {
+      throw new Error('Session shared sandbox route no longer matches recovery action');
+    }
+    if (
+      metadata.workspace.sandboxId === input.replacementSandboxId &&
+      route.suffix === input.suffix
+    ) {
+      return;
+    }
+    if (metadata.workspace.sandboxId !== input.sourceSandboxId || route.suffix !== undefined) {
+      throw new Error('Session shared sandbox assignment no longer matches recovery action');
+    }
+    const updated = serializeSessionMetadata({
+      ...metadata,
+      workspace: {
+        ...metadata.workspace,
+        sandboxId: input.replacementSandboxId,
+        sandboxRoute: {
+          kind: 'shared',
+          routeKey: input.routeKey,
+          suffix: input.suffix,
+        },
+      },
+    });
+    await this.ctx.storage.put('metadata', updated);
+    logger
+      .withFields({
+        sessionId: metadata.identity.sessionId,
+        oldSandboxId: input.sourceSandboxId,
+        newSandboxId: input.replacementSandboxId,
+        routeKey: input.routeKey,
+        logTag: 'shared_sandbox_session_reassigned',
+      })
+      .warn('Moved session to shared sandbox failover assignment');
+  }
+
   async getMetadata(): Promise<SessionMetadata | null> {
     if (await this.hasDeletionIntent()) return null;
     return this.getStoredMetadata();
@@ -2412,19 +2473,6 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
 
       await this.getWrapperSupervisor().runMaintenance(now);
       await this.destroyEphemeralSandboxIfReady(now);
-
-      try {
-        await this.getMessageSettlementOutbox().repairTerminalEffects();
-      } catch (error) {
-        terminalEffectRepairRetryAt = Date.now() + PENDING_FLUSH_DEBOUNCE_MS;
-        logger
-          .withFields({
-            sessionId: this.sessionId,
-            error: error instanceof Error ? error.message : String(error),
-          })
-          .warn('Terminal effect repair failed; scheduled retry will continue recovery');
-      }
-      await this.retryPendingCallbacks(now);
       await this.getSessionMessageQueue().recoverPendingInterruption(async () => {
         await this.interruptAcceptedWrapperMessages();
       });
@@ -2437,6 +2485,34 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
       const flushOneResult = await this.flushOnePendingSessionMessage();
       pendingFlushRetryAt = flushOneResult.retryAt;
       remainingPendingCount = flushOneResult.remainingPendingCount;
+
+      const runtimeState = await getWrapperRuntimeState(this.ctx.storage);
+      const wrapperLease = await getWrapperLease(this.ctx.storage);
+      const deferTerminalEffects =
+        remainingPendingCount > 0 &&
+        isWrapperDeliveryHeld(runtimeState, wrapperLease) &&
+        !isWrapperCleanupExhausted(wrapperLease);
+      if (deferTerminalEffects) {
+        logger
+          .withFields({
+            sessionId: this.sessionId,
+            remainingPendingCount,
+          })
+          .debug('Deferring terminal effect repair while queued delivery is held');
+      } else {
+        try {
+          await this.getMessageSettlementOutbox().repairTerminalEffects();
+        } catch (error) {
+          terminalEffectRepairRetryAt = Date.now() + PENDING_FLUSH_DEBOUNCE_MS;
+          logger
+            .withFields({
+              sessionId: this.sessionId,
+              error: error instanceof Error ? error.message : String(error),
+            })
+            .warn('Terminal effect repair failed; scheduled retry will continue recovery');
+        }
+        await this.retryPendingCallbacks(now);
+      }
     } catch (error) {
       alarmWorkFailed = true;
       logger

services/cloud-agent-next/src/session/message-settlement-outbox.test.ts

@@ -720,6 +720,39 @@ describe('MessageSettlementOutbox', () => {
     expect(harness.callbackJobs[0].payload.messageId).toBe(firstMessageId);
   });
 
+  it('repairs a released terminal wrapper-run callback while the next run remains pending', async () => {
+    const harness = createHarness({ hasObservedWrapperIdle: false });
+    await putSessionMessageState(
+      harness.storage,
+      acceptedMessageState(firstMessageId, { url: 'https://example.com/repaired-batch' })
+    );
+    await storePendingSessionMessage(
+      harness.storage,
+      createPendingSessionMessage({
+        messageId: secondMessageId,
+        role: 'user',
+        content: 'next prompt',
+        createdAt: 3_000,
+      })
+    );
+
+    await harness.outbox.persistTerminalTransition(
+      firstMessageId,
+      {
+        kind: 'failed',
+        reason: 'wrapper became unhealthy',
+        completionSource: 'wrapper_failure',
+      },
+      { allowIdleBatchWithoutObservedIdle: true }
+    );
+
+    await harness.outbox.repairTerminalEffects();
+    await harness.outbox.repairTerminalEffects();
+
+    expect(harness.callbackJobs).toHaveLength(1);
+    expect(harness.callbackJobs[0].payload.messageId).toBe(firstMessageId);
+  });
+
   it('keeps a wrapper-run callback blocked while that run has a nonterminal accepted message', async () => {
     const harness = createHarness();
     await putSessionMessageState(

services/cloud-agent-next/src/session/message-settlement-outbox.ts

@@ -792,9 +792,21 @@ export function createMessageSettlementOutbox(
       }
       const batch = await getCurrentIdleBatchCallbackState();
       if (batch) {
-        await finalizeIdleBatchCallbackIfReady({
-          allowWithoutObservedIdle: batch.allowWithoutObservedIdle,
-        });
+        const representative = batch.representativeMessageId
+          ? await getSessionMessageState(storage, batch.representativeMessageId)
+          : null;
+        if (
+          representative?.wrapperRunId &&
+          (batch.wrapperTerminalWaitReleasedAt !== undefined ||
+            (batch.allowWithoutObservedIdle &&
+              representative.completionSource === 'wrapper_failure'))
+        ) {
+          await finalizeTerminalWrapperRunCallbackIfReady(representative.wrapperRunId);
+        } else {
+          await finalizeIdleBatchCallbackIfReady({
+            allowWithoutObservedIdle: batch.allowWithoutObservedIdle,
+          });
+        }
       }
     } catch (error) {
       await requestAlarmAtOrBefore(Date.now() + 1_000);

services/cloud-agent-next/src/session/wrapper-runtime-state.test.ts

@@ -252,6 +252,132 @@ describe('WrapperLease', () => {
     ).toEqual(exhausted);
   });
 
+  it('persists one post-exhaustion action and retries it with a durable capped deadline', () => {
+    const expectedLease = {
+      target: { kind: 'session' as const },
+      requestedAt: 100,
+      nextInstanceGeneration: 3,
+    };
+    const prepared = reduceSandboxRecoveryState(undefined, {
+      type: 'prepare_post_exhaustion',
+      recovery: {
+        kind: 'isolated-destroy',
+        sourceSandboxId: 'ses-abcdef' as SandboxId,
+        expectedLease,
+        attempts: 0,
+        nextAttemptAt: 300,
+      },
+    });
+    const duplicate = reduceSandboxRecoveryState(prepared, {
+      type: 'prepare_post_exhaustion',
+      recovery: {
+        kind: 'isolated-destroy',
+        sourceSandboxId: 'ses-competing' as SandboxId,
+        expectedLease,
+        attempts: 0,
+        nextAttemptAt: 301,
+      },
+    });
+    const retrying = reduceSandboxRecoveryState(duplicate, {
+      type: 'record_post_exhaustion_retry',
+      expectedLease,
+      expectedAttempts: 0,
+      nextAttemptAt: 2_300,
+      error: 'provider unavailable',
+    });
+
+    expect(duplicate).toEqual(prepared);
+    expect(retrying).toMatchObject({
+      postExhaustionRecovery: {
+        sourceSandboxId: 'ses-abcdef',
+        attempts: 1,
+        nextAttemptAt: 2_300,
+        lastError: 'provider unavailable',
+      },
+    });
+    expect(nextSandboxRecoveryDeadline(retrying)).toBe(2_300);
+  });
+
+  it('checkpoints a completed isolated destroy and keeps it across retry recording', () => {
+    const expectedLease = {
+      target: { kind: 'session' as const },
+      requestedAt: 100,
+      nextInstanceGeneration: 3,
+    };
+    const prepared = reduceSandboxRecoveryState(undefined, {
+      type: 'prepare_post_exhaustion',
+      recovery: {
+        kind: 'isolated-destroy',
+        sourceSandboxId: 'ses-abcdef' as SandboxId,
+        expectedLease,
+        attempts: 0,
+        nextAttemptAt: 300,
+      },
+    });
+
+    const destroyed = reduceSandboxRecoveryState(prepared, {
+      type: 'record_post_exhaustion_destroy',
+      expectedLease,
+      destroyedAt: 400,
+    });
+    expect(destroyed).toMatchObject({
+      postExhaustionRecovery: { kind: 'isolated-destroy', destroyedAt: 400 },
+    });
+    expect(
+      reduceSandboxRecoveryState(destroyed, {
+        type: 'record_post_exhaustion_destroy',
+        expectedLease: { ...expectedLease, requestedAt: 99 },
+        destroyedAt: 500,
+      })
+    ).toEqual(destroyed);
+
+    const retried = reduceSandboxRecoveryState(destroyed, {
+      type: 'record_post_exhaustion_retry',
+      expectedLease,
+      expectedAttempts: 0,
+      nextAttemptAt: 2_400,
+      error: 'settlement failed',
+    });
+    expect(retried).toMatchObject({
+      postExhaustionRecovery: { destroyedAt: 400, attempts: 1, nextAttemptAt: 2_400 },
+    });
+  });
+
+  it('clears only the exhausted lease captured by completed sandbox recovery', () => {
+    const requested = reduceWrapperLease(
+      { state: 'none', nextInstanceGeneration: 3 },
+      {
+        type: 'request_stop',
+        target: { kind: 'session' },
+        reason: 'unhealthy-wrapper',
+        now: 100,
+      }
+    );
+    if (requested.state !== 'stop_needed') throw new Error('Expected cleanup request');
+    const exhausted = reduceWrapperLease(
+      { ...requested, attempts: 5 },
+      { type: 'cleanup_exhausted', now: 300, error: 'still present' }
+    );
+    const expectedLease = {
+      target: { kind: 'session' as const },
+      requestedAt: 100,
+      nextInstanceGeneration: 3,
+    };
+
+    expect(
+      reduceWrapperLease(exhausted, {
+        type: 'sandbox_recovery_completed',
+        expectedLease: { ...expectedLease, requestedAt: 99 },
+      })
+    ).toEqual(exhausted);
+    expect(
+      reduceWrapperLease(exhausted, {
+        type: 'sandbox_recovery_completed',
+        expectedLease,
+      })
+    ).toEqual({ state: 'none', nextInstanceGeneration: 3 });
+  });
+
   it('counts only fresh list-processes timeouts and persists publication deadlines', () => {
     const routeKey = `usr-${'d'.repeat(48)}` as SandboxId;
     expect(