fix(plan): isolate embedded CLI and checkpoint benchmark stages

Author: icecrasher321

apps/sim/lib/mothership/request/lifecycle/run.test.ts

@@ -72,6 +72,7 @@ const {
     INTERNAL_API_SECRET: 'transport-test-secret-000000000000000000',
     COPILOT_API_KEY: undefined as string | undefined,
     MSHIP_SYSPROMPT_OVERRIDE: undefined as string | undefined,
+    MOTHERSHIP_BENCHMARK_URL: 'https://benchmark.test',
   },
 }))
 
@@ -280,6 +281,7 @@ describe('runCopilotLifecycle', () => {
     setEnvFlags({
       isHosted: false,
       isCopilotToolPermissionsEnabled: false,
+      isMothershipBenchmarkEnabled: false,
     })
     mockGetAutoAllowedTools.mockResolvedValue(new Set<string>())
     mockGetUserPermissionConfig.mockResolvedValue(null)
@@ -913,24 +915,30 @@ describe('runCopilotLifecycle', () => {
     })
   })
 
-  it('stamps server-owned transport over caller claims without exposing the instance secret', async () => {
-    let captured = ''
-    mockRunStreamLoop.mockImplementationOnce(async (_url: string, request: RequestInit) => {
-      captured = String(request.body)
-    })
-    await runCopilotLifecycle(
-      { message: 'hello', simConnection: { mode: 'direct' } },
-      {
-        userId: 'user-1',
-        workspaceId: 'ws-1',
-      }
-    )
-    expect(JSON.parse(captured).simConnection).toEqual({
-      mode: 'checkpoint',
-      channelId: expect.stringMatching(/^[a-f0-9]{64}$/),
-    })
-    expect(captured).not.toContain(mockEnv.INTERNAL_API_SECRET)
-  })
+  it.each(['/api/mothership', '/api/copilot', '/api/mothership/execute'])(
+    'forces benchmark checkpoint transport over caller claims on %s',
+    async (goRoute) => {
+      setEnvFlags({ isMothershipBenchmarkEnabled: true })
+      let captured = ''
+      mockRunStreamLoop.mockImplementationOnce(async (_url: string, request: RequestInit) => {
+        captured = String(request.body)
+      })
+      await runCopilotLifecycle(
+        { message: 'hello', simConnection: { mode: 'direct' } },
+        {
+          userId: 'user-1',
+          workspaceId: 'ws-1',
+          benchmark: 'distill',
+          goRoute,
+        }
+      )
+      expect(JSON.parse(captured).simConnection).toEqual({
+        mode: 'checkpoint',
+        channelId: expect.stringMatching(/^[a-f0-9]{64}$/),
+      })
+      expect(captured).not.toContain(mockEnv.INTERNAL_API_SECRET)
+    }
+  )
 
   it.each([false, true])(
     'attaches BYOK without letting a hidden hosted default override it (advanced=%s)',

apps/sim/lib/mothership/request/lifecycle/run.ts

@@ -1178,7 +1178,11 @@ async function runCheckpointLoop(
     ? getBenchmarkMothershipUrl()
     : await getMothershipBaseURL({ userId: options.userId })
   execContext.mothershipBaseURL = mothershipBaseURL
-  if (initialRoute === '/api/mothership' || initialRoute === '/api/copilot') {
+  if (
+    initialRoute === '/api/mothership' ||
+    initialRoute === '/api/copilot' ||
+    (initialRoute === '/api/mothership/execute' && options.benchmark)
+  ) {
     const simConnection = getSimConnection(options.benchmark ? 'checkpoint' : undefined)
     payload = { ...payload, simConnection }
   }

packages/sim-cli/src/embed-host-state.test.ts

@@ -0,0 +1,26 @@
+import { mkdtemp, readdir, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { runEmbeddedCli } from '#sim-cli/embed'
+
+const identity = { endpoint: 'https://sim.test', apiKey: 'fixture', workspaceId: 'workspace' }
+
+afterEach(() => vi.unstubAllEnvs())
+
+describe('embedded CLI host isolation', () => {
+  it.each([
+    { name: 'profile configuration', argv: ['configure', '--set-output', 'json'] },
+    { name: 'telemetry preferences', argv: ['telemetry', 'disable'] },
+  ])('rejects $name without persisting host settings', async ({ argv }) => {
+    const directory = await mkdtemp(join(tmpdir(), 'sim-embedded-host-'))
+    vi.stubEnv('SIM_CONFIG_DIR', directory)
+    try {
+      const result = await runEmbeddedCli(argv, identity)
+      expect.soft(result.exitCode).toBe(1)
+      expect.soft(await readdir(directory)).toEqual([])
+    } finally {
+      await rm(directory, { recursive: true, force: true })
+    }
+  })
+})

packages/sim-cli/src/embed.ts

@@ -132,7 +132,11 @@ export async function runEmbeddedCli(
     let exitCode = 0
     try {
       identity.signal?.throwIfAborted()
-      const program = buildProgram()
+      const program = buildProgram({
+        hostCommands: false,
+        helpText:
+          'The embedding application supplies authentication, endpoint and workspace scope.',
+      })
       configureEmbeddedOutput(program)
       await program.parseAsync(argv, { from: 'user' })
       // Commands that soft-fail (a failed run outcome, wait timeout) report through the

packages/sim-cli/src/program.ts

@@ -148,7 +148,13 @@ function addVersionOption(program: Command): void {
  * release.
  */
 export function buildProgram(
-  options: { version?: boolean; helpText?: string; program?: Command; env?: NodeJS.ProcessEnv } = {}
+  options: {
+    version?: boolean
+    helpText?: string
+    program?: Command
+    env?: NodeJS.ProcessEnv
+    hostCommands?: boolean
+  } = {}
 ): Command {
   const env = options.env ?? process.env
   const program = options.program ?? new Command()
@@ -164,17 +170,23 @@ export function buildProgram(
     .addOption(
       new Option('--output <format>', 'Output format for this command').choices([...OUTPUT_FORMATS])
     )
-  for (const command of [
-    loginCommand,
-    logoutCommand,
-    whoamiCommand,
-    profilesCommand,
-    configureCommand,
-  ])
-    program.addCommand(command())
-  const update = updateCommand()
-  program.addCommand(update)
-  program.addCommand(telemetryCommand())
+  if (options.hostCommands !== false) {
+    for (const command of [
+      loginCommand,
+      logoutCommand,
+      whoamiCommand,
+      profilesCommand,
+      configureCommand,
+    ])
+      program.addCommand(command())
+    const update = updateCommand()
+    program.addCommand(update)
+    program.addCommand(telemetryCommand())
+    program.hook('preAction', async (_program, command) => {
+      if (command === update) return
+      await announceUpdateIfAvailable()
+    })
+  }
   program.addCommand(cliCommand())
 
   for (const command of buildGeneratedCommands()) {
@@ -190,11 +202,6 @@ export function buildProgram(
     detectCodingAgent(env) !== undefined && command.commands.length > 0 ? AGENT_DISCOVERY_NOTE : ''
   )
 
-  program.hook('preAction', async (_program, command) => {
-    if (command === update) return
-    await announceUpdateIfAvailable()
-  })
-
   refuseHelpAfterUnknownCommand(program)
   assertNoReservedProgramFlags(program)