From 2007048b2845d98384cd8c2864e318a76f7a69bb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 21:58:23 -0700 Subject: [PATCH 1/4] test(desktop): stop the agent's terminal commands in the real app, with real shells and real tmux An Electron suite for terminal cancel, which ships without a feature flag: Stop, sign-out (by the web app's logout and by the session cookie going away), switching Terminal off, the launch after an interrupted sign-out or a crash, and untracked runs on tmux without pane options. It asserts what is running and which panes exist, never which calls were made. - Agent and user commands ignore SIGHUP, SIGINT and SIGTERM, so only Sim's own stop can end them and a stray stop would show; each launch gets its own tmux server, and every process name carries a per-run suffix. - The fixture Sim and its helpers move to e2e/executor-sim.ts, shared with the background executor suite. - The macOS E2E job installs tmux so the tmux scenarios run in CI; elsewhere they skip when tmux is missing. --- .github/workflows/desktop-e2e.yml | 4 + apps/desktop/e2e/background-executor.spec.ts | 556 ++--------------- apps/desktop/e2e/executor-sim.ts | 486 +++++++++++++++ apps/desktop/e2e/terminal-cancel.spec.ts | 611 +++++++++++++++++++ 4 files changed, 1157 insertions(+), 500 deletions(-) create mode 100644 apps/desktop/e2e/executor-sim.ts create mode 100644 apps/desktop/e2e/terminal-cancel.spec.ts diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index fb673b362c8..410464cefa2 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -81,6 +81,10 @@ jobs: working-directory: apps/desktop run: bunx playwright install chromium + # The terminal-cancel suite drives a real tmux server; without tmux its tmux scenarios skip. + - name: Install tmux + run: brew install tmux + - name: Run Playwright _electron smoke suite working-directory: apps/desktop run: bunx playwright test diff --git a/apps/desktop/e2e/background-executor.spec.ts b/apps/desktop/e2e/background-executor.spec.ts index 8482e27bcbd..72ec81b2018 100644 --- a/apps/desktop/e2e/background-executor.spec.ts +++ b/apps/desktop/e2e/background-executor.spec.ts @@ -1,20 +1,23 @@ -import { execFileSync } from 'node:child_process' -import { createHash } from 'node:crypto' import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs' -import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { fileURLToPath } from 'node:url' -import { - type ElectronApplication, - _electron as electron, - expect, - type Page, - test, -} from '@playwright/test' +import { type ElectronApplication, expect, test } from '@playwright/test' import type { SimDesktopApi } from '@sim/desktop-bridge' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { + type FixtureCall, + FixtureSim, + launch, + processRunning, + RECONCILE_MS, + readFileSafe, + refFor, + registeredDevice, + settled, + sha256, + WORKSPACE, +} from './executor-sim' /** * The Sim desktop app's background executor against a fixture Sim that speaks the executor's @@ -25,39 +28,9 @@ import { sleep } from '@sim/utils/helpers' * report at BACKGROUND_EXECUTOR_REPORT_PATH. */ -const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) -const WORKSPACE = 'ws-e2e' const CHAT_A = 'chat-browser-a' const CHAT_B = 'chat-terminal-b' const CHAT_C = 'chat-idle-c' -const LEASE_RENEW_MS = 1_000 -const RECONCILE_MS = 2_000 - -type CallStatus = 'pending' | 'awaiting_approval' | 'running' | 'completed' | 'failed' | 'cancelled' - -interface Completion { - status: string - message: string - data?: Record - outcome: 'recorded' | 'duplicate' | 'superseded' - at: number -} - -interface FixtureCall { - toolCallId: string - toolName: string - args: Record - chatId: string - deviceId: string - status: CallStatus - issuedAt: number - claimedAt?: number - token?: string - claims: number - renewals: number - completions: Completion[] - acknowledged: boolean -} interface ReportCheck { name: string @@ -68,399 +41,8 @@ interface ReportCheck { const report: ReportCheck[] = [] -interface ImportedEntry { - toolCallId: string - kind: string - sourceName: string - relativePath: string - sha256?: string - bytes?: number -} - -function sha256(bytes: Buffer): string { - return createHash('sha256').update(bytes).digest('hex') -} - -class FixtureSim { - readonly calls = new Map() - readonly devices = new Map() - readonly requests: string[] = [] - readonly hits = new Map() - readonly streams = new Map>() - readonly imported: ImportedEntry[] = [] - enabled = true - offline = false - droppedWhileOffline = 0 - private server: Server | null = null - origin = '' - private nextToken = 0 - - async start(): Promise { - this.server = createServer((request, response) => { - void this.handle(request, response) - }) - await new Promise((resolve) => this.server?.listen(0, '127.0.0.1', resolve)) - const address = this.server.address() - if (!address || typeof address === 'string') throw new Error('Missing fixture address') - this.origin = `http://127.0.0.1:${address.port}` - } - - async stop(): Promise { - for (const streams of this.streams.values()) for (const stream of streams) stream.destroy() - this.server?.closeAllConnections() - await new Promise((resolve) => this.server?.close(() => resolve())) - } - - reset(): void { - this.calls.clear() - this.devices.clear() - this.requests.length = 0 - this.hits.clear() - this.imported.length = 0 - this.enabled = true - this.offline = false - this.droppedWhileOffline = 0 - } - - /** Persists a call for a device and rings its doorbell, as Sim's pre-persist and offer do. */ - issue( - deviceId: string, - chatId: string, - toolName: string, - args: Record, - status: CallStatus = 'pending' - ): string { - const toolCallId = `call-${this.calls.size + 1}-${toolName}` - this.calls.set(toolCallId, { - toolCallId, - toolName, - args, - chatId, - deviceId, - status, - issuedAt: Date.now(), - claims: 0, - renewals: 0, - completions: [], - acknowledged: false, - }) - this.ring(deviceId, status === 'awaiting_approval' ? 'approval' : 'call') - return toolCallId - } - - /** Stop from any surface: Sim settles the call and tells the device to cancel. */ - stopCall(toolCallId: string): void { - const call = this.requireCall(toolCallId) - call.status = 'cancelled' - this.ring(call.deviceId, 'cancel') - } - - approve(toolCallId: string): void { - const call = this.requireCall(toolCallId) - call.status = 'pending' - this.ring(call.deviceId, 'approval') - } - - requireCall(toolCallId: string): FixtureCall { - const call = this.calls.get(toolCallId) - if (!call) throw new Error(`No fixture call ${toolCallId}`) - return call - } - - ring(deviceId: string, reason: string): void { - for (const stream of this.streams.get(deviceId) ?? []) { - stream.write(`event: inbox_changed\ndata: ${JSON.stringify({ reason })}\n\n`) - } - } - - private async raw(request: IncomingMessage): Promise { - const chunks: Buffer[] = [] - for await (const chunk of request) chunks.push(Buffer.from(chunk)) - return Buffer.concat(chunks) - } - - private async body(request: IncomingMessage): Promise> { - const text = (await this.raw(request)).toString() - return text ? (JSON.parse(text) as Record) : {} - } - - /** Stores one entry of a claimed, running import, as Sim's import route does. */ - private async importEntry( - url: URL, - request: IncomingMessage, - response: ServerResponse - ): Promise { - const query = Object.fromEntries(url.searchParams) - const call = this.calls.get(query.toolCallId ?? '') - if ( - request.method !== 'PUT' || - !call || - call.deviceId !== query.deviceId || - call.toolName !== 'import_local_files' || - call.token !== request.headers['x-sim-execution-token'] || - call.status !== 'running' - ) { - this.json(response, 404, { error: 'Desktop import not found' }) - return - } - // As Sim's route does: a file must declare its length, and arrive whole. - if (query.kind === 'file' && request.headers['content-length'] === undefined) { - this.json(response, 411, { error: 'A file import must declare its length' }) - return - } - const content = await this.raw(request) - if (query.kind === 'file' && content.length !== Number(request.headers['content-length'])) { - this.json(response, 400, { error: 'The file did not arrive whole' }) - return - } - const entry: ImportedEntry = { - toolCallId: call.toolCallId, - kind: query.kind ?? '', - sourceName: query.sourceName ?? '', - relativePath: query.relativePath ?? '', - ...(query.kind === 'file' ? { sha256: sha256(content), bytes: content.length } : {}), - } - this.imported.push(entry) - this.json(response, 200, { - id: `entry-${this.imported.length}`, - name: entry.relativePath.split('/').at(-1) || entry.sourceName, - }) - } - - private json(response: ServerResponse, status: number, body: unknown): void { - response.writeHead(status, { 'Content-Type': 'application/json' }) - response.end(JSON.stringify(body)) - } - - private signedIn(request: IncomingMessage): boolean { - return request.headers.cookie?.includes('better-auth.session_token=fixture') ?? false - } - - private async handle(request: IncomingMessage, response: ServerResponse): Promise { - const url = new URL(request.url ?? '/', this.origin) - const path = url.pathname - this.requests.push(`${request.method} ${path}`) - if (path.startsWith('/api/desktop/') && this.offline) { - this.droppedWhileOffline += 1 - request.socket.destroy() - return - } - if (path === '/api/auth/get-session') { - this.json( - response, - 200, - this.signedIn(request) ? { user: { id: 'user-e2e' }, session: { id: 'session-e2e' } } : null - ) - return - } - if (path.startsWith('/api/desktop/') && !this.signedIn(request)) { - this.json(response, 401, { error: 'Unauthorized' }) - return - } - if (path === '/api/desktop/devices' && request.method === 'POST') { - const body = await this.body(request) - if (this.enabled) { - this.devices.set(String(body.deviceId), { - name: String(body.name), - platform: String(body.platform), - }) - } - this.json(response, 200, { - enabled: this.enabled, - protocolVersion: 1, - leaseMs: 60_000, - leaseRenewMs: LEASE_RENEW_MS, - reconcileMs: RECONCILE_MS, - }) - return - } - const deviceId = url.searchParams.get('deviceId') - if (path === '/api/desktop/inbox/stream') { - if (!deviceId || !this.devices.has(deviceId)) { - this.json(response, 401, { error: 'Unregistered' }) - return - } - response.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }) - response.write(': connected\n\n') - const streams = this.streams.get(deviceId) ?? new Set() - streams.add(response) - this.streams.set(deviceId, streams) - request.on('close', () => streams.delete(response)) - return - } - if (path === '/api/desktop/inbox') { - if (!deviceId || !this.devices.has(deviceId)) { - this.json(response, 401, { error: 'Unregistered' }) - return - } - const items = [...this.calls.values()] - .filter((call) => call.deviceId === deviceId) - .flatMap((call) => { - if (call.status === 'pending' && !call.token) - return [ - { - kind: 'call', - toolCallId: call.toolCallId, - toolName: call.toolName, - chatId: call.chatId, - workspaceId: WORKSPACE, - createdAt: new Date(call.issuedAt).toISOString(), - }, - ] - if (call.status === 'awaiting_approval') - return [ - { - kind: 'approval_needed', - toolCallId: call.toolCallId, - toolName: call.toolName, - chatId: call.chatId, - chatTitle: 'Fix CI', - workspaceId: WORKSPACE, - summary: String((call.args.args as { command?: string } | undefined)?.command), - }, - ] - if (call.token && call.status === 'cancelled' && !call.acknowledged) - return [{ kind: 'cancel', toolCallId: call.toolCallId }] - return [] - }) - this.json(response, 200, { items }) - return - } - if (path === '/api/desktop/tool/import') { - await this.importEntry(url, request, response) - return - } - if (path.startsWith('/api/desktop/tool/')) { - const body = await this.body(request) - const call = this.calls.get(String(body.toolCallId)) - if (!call || call.deviceId !== body.deviceId) { - this.json(response, 404, { error: 'Desktop tool call not found' }) - return - } - if (path === '/api/desktop/tool/claim') { - if (call.status !== 'pending' || call.token) { - this.json(response, 404, { error: 'This call is no longer waiting for this device' }) - return - } - call.claims += 1 - call.status = 'running' - call.claimedAt = Date.now() - call.token = `token-${++this.nextToken}` - this.json(response, 200, { - toolName: call.toolName, - args: call.args, - chatId: call.chatId, - workspaceId: WORKSPACE, - executionToken: call.token, - }) - return - } - if (body.executionToken !== call.token) { - this.json(response, 404, { error: 'Desktop tool call not found' }) - return - } - if (path === '/api/desktop/tool/lease') { - call.renewals += 1 - if (call.status !== 'running') { - this.json(response, 410, { error: 'This call was stopped or settled. Stop running it.' }) - return - } - this.json(response, 200, { renewed: true }) - return - } - if (path === '/api/desktop/tool/complete') { - const status = String(body.status) - const settled = - status === 'success' ? 'completed' : status === 'cancelled' ? 'cancelled' : 'failed' - const outcome = - call.status === 'running' - ? 'recorded' - : call.completions.length > 0 - ? 'duplicate' - : 'superseded' - if (outcome === 'recorded') call.status = settled - if (call.status === 'cancelled') call.acknowledged = true - call.completions.push({ - status, - message: String(body.message ?? ''), - ...(body.data && typeof body.data === 'object' - ? { data: body.data as Record } - : {}), - outcome, - at: Date.now(), - }) - this.json(response, 200, { - outcome, - status: call.status === 'running' ? settled : call.status, - }) - return - } - } - if (path === '/hit') { - const chat = url.searchParams.get('chat') ?? '' - this.hits.set(chat, (this.hits.get(chat) ?? 0) + 1) - this.json(response, 200, { count: this.hits.get(chat) }) - return - } - response.writeHead(200, { - 'Content-Type': 'text/html', - 'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; SameSite=Lax; Path=/', - }) - response.end( - path === '/counter' - ? `Counter ${url.searchParams.get('chat')} - -

0

` - : `Sim fixture

${path}

` - ) - } -} - const sim = new FixtureSim() -async function launch(userData: string): Promise<{ app: ElectronApplication; window: Page }> { - const app = await electron.launch({ - args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], - cwd: DESKTOP_DIR, - env: { - ...process.env, - SIM_DESKTOP_ORIGIN: sim.origin, - SIM_DESKTOP_USER_DATA: userData, - }, - }) - // A dialog listener stops Playwright auto-dismissing page dialogs, so the desktop's own handling - // decides their outcome exactly as it does in production. - const leaveDialogsToDesktop = (page: Page) => page.on('dialog', () => {}) - app.context().pages().forEach(leaveDialogsToDesktop) - app.context().on('page', leaveDialogsToDesktop) - const window = await app.firstWindow() - return { app, window } -} - -/** The device this app registered, once it has opened its doorbell. */ -async function registeredDevice(knownDevices: ReadonlySet = new Set()): Promise { - let deviceId: string | undefined - await expect - .poll( - () => { - deviceId = [...sim.devices.keys()].find( - (id) => !knownDevices.has(id) && (sim.streams.get(id)?.size ?? 0) > 0 - ) - return deviceId - }, - { timeout: 30_000 } - ) - .toBeTruthy() - return deviceId as string -} - -async function settled(toolCallId: string, timeout = 60_000): Promise { - await expect - .poll(() => sim.requireCall(toolCallId).completions.length, { timeout }) - .toBeGreaterThan(0) - return sim.requireCall(toolCallId).completions[0] as Completion -} - async function check(name: string, body: () => Promise): Promise { const startedAt = Date.now() try { @@ -477,24 +59,6 @@ async function check(name: string, body: () => Promise): Promise { } } -function refFor(outline: string, name: string): number { - const match = outline - .split('\n') - .find((line) => line.includes(`"${name}"`) && /\[ref=\d+\]/.test(line)) - ?.match(/\[ref=(\d+)\]/) - if (!match) throw new Error(`No reference for ${name}: ${outline}`) - return Number(match[1]) -} - -function processRunning(pattern: string): boolean { - try { - execFileSync('pgrep', ['-f', pattern]) - return true - } catch { - return false - } -} - test.describe('background executor', () => { let app: ElectronApplication | null = null @@ -521,10 +85,10 @@ test.describe('background executor', () => { test('A: two chats run browser and terminal work while the user is elsewhere and reloads', async () => { const userData = mkdtempSync(join(tmpdir(), 'sim-executor-a-')) - const launched = await launch(userData) + const launched = await launch(sim, userData) app = launched.app const window = launched.window - const deviceId = await registeredDevice() + const deviceId = await registeredDevice(sim) await window.goto(`${sim.origin}/workspace/${WORKSPACE}/chat/${CHAT_C}`) const marker = join(userData, 'terminal-marker.txt') @@ -535,7 +99,7 @@ test.describe('background executor', () => { const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', { url: `${sim.origin}/counter?chat=A`, }) - const completion = await settled(opened) + const completion = await settled(sim, opened) expect(completion.status, completion.message).toBe('success') }) const opened = [...sim.calls.values()][0] as FixtureCall @@ -559,7 +123,7 @@ test.describe('background executor', () => { await check('A: every call completes exactly once with its own token', async () => { for (const id of [...clicks, ...terminalRuns, localRead]) { - const completion = await settled(id, 90_000) + const completion = await settled(sim, id, 90_000) const call = sim.requireCall(id) expect(completion.status, `${call.toolName}: ${completion.message}`).toBe('success') expect(completion.outcome).toBe('recorded') @@ -580,7 +144,7 @@ test.describe('background executor', () => { await check("A: chat B cannot see chat A's tabs", async () => { const listed = sim.issue(deviceId, CHAT_B, 'browser_list_tabs', {}) - const completion = await settled(listed) + const completion = await settled(sim, listed) expect(JSON.stringify(completion.data)).not.toContain('/counter') }) @@ -606,8 +170,8 @@ test.describe('background executor', () => { test('B: a result produced while offline is delivered once after reconnecting', async () => { const userData = mkdtempSync(join(tmpdir(), 'sim-executor-b-')) - app = (await launch(userData)).app - const deviceId = await registeredDevice() + app = (await launch(sim, userData)).app + const deviceId = await registeredDevice(sim) const run = sim.issue(deviceId, CHAT_B, 'terminal', { operation: 'run', @@ -624,7 +188,7 @@ test.describe('background executor', () => { sim.offline = false await check('B: the result arrives once after reconnecting', async () => { - const completion = await settled(run, 60_000) + const completion = await settled(sim, run, 60_000) expect(completion.status).toBe('success') expect(JSON.stringify(completion.data)).toContain('offline-done') await sleep(3_000) @@ -635,8 +199,8 @@ test.describe('background executor', () => { test('C: a crash mid-command reports the outcome as unknown and never reruns it', async () => { const userData = mkdtempSync(join(tmpdir(), 'sim-executor-c-')) const marker = join(userData, 'crash-marker.txt') - const first = await launch(userData) - const deviceId = await registeredDevice() + const first = await launch(sim, userData) + const deviceId = await registeredDevice(sim) const run = sim.issue(deviceId, CHAT_B, 'terminal', { operation: 'run', @@ -646,10 +210,10 @@ test.describe('background executor', () => { first.app.process().kill('SIGKILL') sim.streams.get(deviceId)?.clear() - app = (await launch(userData)).app + app = (await launch(sim, userData)).app await check('C: the restarted app reports the lost result as outcome unknown', async () => { - const completion = await settled(run, 60_000) + const completion = await settled(sim, run, 60_000) expect(completion.data).toMatchObject({ outcomeUnknown: true, doNotRetry: true }) }) @@ -661,9 +225,9 @@ test.describe('background executor', () => { test('D: a folder import lands in Sim while the user is in another chat', async () => { const userData = mkdtempSync(join(tmpdir(), 'sim-executor-d-')) - const launched = await launch(userData) + const launched = await launch(sim, userData) app = launched.app - const deviceId = await registeredDevice() + const deviceId = await registeredDevice(sim) const source = join(userData, 'Reports') mkdirSync(join(source, 'q3'), { recursive: true }) writeFileSync(join(source, 'notes.txt'), 'remember the numbers') @@ -679,7 +243,7 @@ test.describe('background executor', () => { }) await check('D: the import completes with every entry it stored', async () => { - const completion = await settled(call, 60_000) + const completion = await settled(sim, call, 60_000) expect(completion.status).toBe('success') expect(completion.data).toMatchObject({ success: true, @@ -722,13 +286,13 @@ test.describe('background executor', () => { test('E: Stop from another chat stops a running browser wait and terminal command', async () => { const userData = mkdtempSync(join(tmpdir(), 'sim-executor-e-')) - app = (await launch(userData)).app - const deviceId = await registeredDevice() + app = (await launch(sim, userData)).app + const deviceId = await registeredDevice(sim) const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', { url: `${sim.origin}/counter?chat=E`, }) - await settled(opened) + await settled(sim, opened) const wait = sim.issue(deviceId, CHAT_A, 'browser_wait_for', { text: 'never appears', timeoutMs: 30_000, @@ -746,8 +310,8 @@ test.describe('background executor', () => { sim.stopCall(run) await check('E: both stopped calls are acknowledged within seconds', async () => { - await settled(wait, 10_000) - await settled(run, 15_000) + await settled(sim, wait, 10_000) + await settled(sim, run, 15_000) expect(sim.requireCall(wait).completions[0]?.outcome).toBe('superseded') expect(sim.requireCall(run).completions[0]?.outcome).toBe('superseded') expect((sim.requireCall(wait).completions[0]?.at ?? 0) - stoppedAt).toBeLessThan(5_000) @@ -760,7 +324,7 @@ test.describe('background executor', () => { test('F: a call waiting for approval in a background chat notifies, then runs once approved', async () => { const userData = mkdtempSync(join(tmpdir(), 'sim-executor-f-')) - app = (await launch(userData)).app + app = (await launch(sim, userData)).app await app.evaluate(({ Notification }) => { const shown: Array<{ title: string; body: string }> = [] const target = globalThis as typeof globalThis & { __shownNotifications?: typeof shown } @@ -769,7 +333,7 @@ test.describe('background executor', () => { shown.push({ title: this.title, body: this.body }) } }) - const deviceId = await registeredDevice() + const deviceId = await registeredDevice(sim) const gated = sim.issue( deviceId, @@ -806,22 +370,22 @@ test.describe('background executor', () => { sim.approve(gated) const approvedAt = Date.now() await check('F: the approved call is claimed promptly and runs once', async () => { - const completion = await settled(gated) + const completion = await settled(sim, gated) expect((sim.requireCall(gated).claimedAt ?? 0) - approvedAt).toBeLessThan(1_500) expect(JSON.stringify(completion.data)).toContain('approved-run') }) }) test('G: only the device a turn is bound to claims its calls', async () => { - const first = await launch(mkdtempSync(join(tmpdir(), 'sim-executor-g1-'))) - const firstDevice = await registeredDevice() - const second = await launch(mkdtempSync(join(tmpdir(), 'sim-executor-g2-'))) - const secondDevice = await registeredDevice(new Set([firstDevice])) + const first = await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-g1-'))) + const firstDevice = await registeredDevice(sim) + const second = await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-g2-'))) + const secondDevice = await registeredDevice(sim, new Set([firstDevice])) app = first.app const call = sim.issue(firstDevice, CHAT_A, 'browser_list_tabs', {}) await check('G: the bound device runs it and the other never claims it', async () => { - await settled(call) + await settled(sim, call) expect(sim.requireCall(call).claims).toBe(1) expect( sim.requests.filter((request) => request === 'POST /api/desktop/tool/claim').length @@ -833,7 +397,7 @@ test.describe('background executor', () => { test('H: a device Sim has not enabled offers no binding', async () => { sim.enabled = false - const launched = await launch(mkdtempSync(join(tmpdir(), 'sim-executor-h-'))) + const launched = await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-h-'))) app = launched.app await launched.window.goto(`${sim.origin}/workspace/${WORKSPACE}/chat/${CHAT_C}`) await expect.poll(() => sim.requests.includes('POST /api/desktop/devices')).toBe(true) @@ -852,9 +416,9 @@ test.describe('background executor', () => { sim.enabled = true await launched.app.close() - const enabled = await launch(mkdtempSync(join(tmpdir(), 'sim-executor-h2-'))) + const enabled = await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-h2-'))) app = enabled.app - const deviceId = await registeredDevice() + const deviceId = await registeredDevice(sim) await enabled.window.goto(`${sim.origin}/workspace/${WORKSPACE}/chat/${CHAT_C}`) await check('H: an enabled device offers itself for binding', async () => { const device = await enabled.window.evaluate(() => @@ -867,12 +431,12 @@ test.describe('background executor', () => { }) test('I: the agent yields its page while the user works in it, then takes it back', async () => { - app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-i-')))).app - const deviceId = await registeredDevice() + app = (await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-i-')))).app + const deviceId = await registeredDevice(sim) const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', { url: `${sim.origin}/counter?chat=I`, }) - const outline = ((await settled(opened)).data?.snapshot as { outline: string }).outline + const outline = ((await settled(sim, opened)).data?.snapshot as { outline: string }).outline const button = refFor(outline, 'Count visit') const typeInAgentPage = () => @@ -892,7 +456,7 @@ test.describe('background executor', () => { const lastUserInputAt = Date.now() await check('I: the click waits until the user stops, then runs once', async () => { - const completion = await settled(click, 30_000) + const completion = await settled(sim, click, 30_000) expect(completion.status, completion.message).toBe('success') expect(completion.at - lastUserInputAt).toBeGreaterThanOrEqual(3_000) await expect.poll(() => sim.hits.get('I')).toBe(1) @@ -900,12 +464,12 @@ test.describe('background executor', () => { }) test('I: an action the user never stops working long enough for does not run', async () => { - app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-i2-')))).app - const deviceId = await registeredDevice() + app = (await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-i2-')))).app + const deviceId = await registeredDevice(sim) const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', { url: `${sim.origin}/counter?chat=I2`, }) - const outline = ((await settled(opened)).data?.snapshot as { outline: string }).outline + const outline = ((await settled(sim, opened)).data?.snapshot as { outline: string }).outline const button = refFor(outline, 'Count visit') const typeInAgentPage = () => app?.evaluate(({ webContents }) => { @@ -927,7 +491,7 @@ test.describe('background executor', () => { })() await check('I: the click reports it never ran, instead of timing out', async () => { - const completion = await settled(click, 60_000) + const completion = await settled(sim, click, 60_000) typing = false await keepTyping expect(completion.status).toBe('error') @@ -938,7 +502,7 @@ test.describe('background executor', () => { }) test('J: the machine stays awake only while a chat has work running', async () => { - app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-j-')))).app + app = (await launch(sim, mkdtempSync(join(tmpdir(), 'sim-executor-j-')))).app await app.evaluate(({ powerSaveBlocker }) => { const log: string[] = [] const active = new Set() @@ -957,7 +521,7 @@ test.describe('background executor', () => { } powerSaveBlocker.isStarted = (id) => active.has(id) }) - const deviceId = await registeredDevice() + const deviceId = await registeredDevice(sim) const sleepLog = () => app?.evaluate(() => (globalThis as { __sleepBlocks?: string[] }).__sleepBlocks ?? []) @@ -969,18 +533,10 @@ test.describe('background executor', () => { await expect.poll(sleepLog, { timeout: 15_000 }).toEqual(['start:prevent-app-suspension']) }) await check('J: it is released once the result is delivered', async () => { - await settled(run) + await settled(sim, run) await expect .poll(sleepLog, { timeout: 10_000 }) .toEqual(['start:prevent-app-suspension', 'stop']) }) }) }) - -function readFileSafe(path: string): string { - try { - return readFileSync(path, 'utf8') - } catch { - return '' - } -} diff --git a/apps/desktop/e2e/executor-sim.ts b/apps/desktop/e2e/executor-sim.ts new file mode 100644 index 00000000000..1273052f7ef --- /dev/null +++ b/apps/desktop/e2e/executor-sim.ts @@ -0,0 +1,486 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import { fileURLToPath } from 'node:url' +import { + type ElectronApplication, + _electron as electron, + expect, + type Page, +} from '@playwright/test' + +/** + * A fixture Sim that speaks the background executor's device protocol (register, inbox, + * doorbell, claim, lease, complete, import) the way Sim's routes do, and the helpers the + * executor's Electron suites share. + */ + +const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) +export const WORKSPACE = 'ws-e2e' +export const LEASE_RENEW_MS = 1_000 +export const RECONCILE_MS = 2_000 + +type CallStatus = 'pending' | 'awaiting_approval' | 'running' | 'completed' | 'failed' | 'cancelled' + +export interface Completion { + status: string + message: string + data?: Record + outcome: 'recorded' | 'duplicate' | 'superseded' + at: number +} + +export interface FixtureCall { + toolCallId: string + toolName: string + args: Record + chatId: string + deviceId: string + status: CallStatus + issuedAt: number + claimedAt?: number + token?: string + claims: number + renewals: number + completions: Completion[] + acknowledged: boolean +} + +export interface ImportedEntry { + toolCallId: string + kind: string + sourceName: string + relativePath: string + sha256?: string + bytes?: number +} + +export function sha256(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex') +} + +export class FixtureSim { + readonly calls = new Map() + readonly devices = new Map() + readonly requests: string[] = [] + readonly hits = new Map() + readonly streams = new Map>() + readonly imported: ImportedEntry[] = [] + enabled = true + offline = false + droppedWhileOffline = 0 + private server: Server | null = null + origin = '' + private nextToken = 0 + + async start(): Promise { + this.server = createServer((request, response) => { + void this.handle(request, response) + }) + await new Promise((resolve) => this.server?.listen(0, '127.0.0.1', resolve)) + const address = this.server.address() + if (!address || typeof address === 'string') throw new Error('Missing fixture address') + this.origin = `http://127.0.0.1:${address.port}` + } + + async stop(): Promise { + for (const streams of this.streams.values()) for (const stream of streams) stream.destroy() + this.server?.closeAllConnections() + await new Promise((resolve) => this.server?.close(() => resolve())) + } + + reset(): void { + this.calls.clear() + this.devices.clear() + this.requests.length = 0 + this.hits.clear() + this.imported.length = 0 + this.enabled = true + this.offline = false + this.droppedWhileOffline = 0 + } + + /** Persists a call for a device and rings its doorbell, as Sim's pre-persist and offer do. */ + issue( + deviceId: string, + chatId: string, + toolName: string, + args: Record, + status: CallStatus = 'pending' + ): string { + const toolCallId = `call-${this.calls.size + 1}-${toolName}` + this.calls.set(toolCallId, { + toolCallId, + toolName, + args, + chatId, + deviceId, + status, + issuedAt: Date.now(), + claims: 0, + renewals: 0, + completions: [], + acknowledged: false, + }) + this.ring(deviceId, status === 'awaiting_approval' ? 'approval' : 'call') + return toolCallId + } + + /** Stop from any surface: Sim settles the call and tells the device to cancel. */ + stopCall(toolCallId: string): void { + const call = this.requireCall(toolCallId) + call.status = 'cancelled' + this.ring(call.deviceId, 'cancel') + } + + approve(toolCallId: string): void { + const call = this.requireCall(toolCallId) + call.status = 'pending' + this.ring(call.deviceId, 'approval') + } + + requireCall(toolCallId: string): FixtureCall { + const call = this.calls.get(toolCallId) + if (!call) throw new Error(`No fixture call ${toolCallId}`) + return call + } + + ring(deviceId: string, reason: string): void { + for (const stream of this.streams.get(deviceId) ?? []) { + stream.write(`event: inbox_changed\ndata: ${JSON.stringify({ reason })}\n\n`) + } + } + + private async raw(request: IncomingMessage): Promise { + const chunks: Buffer[] = [] + for await (const chunk of request) chunks.push(Buffer.from(chunk)) + return Buffer.concat(chunks) + } + + private async body(request: IncomingMessage): Promise> { + const text = (await this.raw(request)).toString() + return text ? (JSON.parse(text) as Record) : {} + } + + /** Stores one entry of a claimed, running import, as Sim's import route does. */ + private async importEntry( + url: URL, + request: IncomingMessage, + response: ServerResponse + ): Promise { + const query = Object.fromEntries(url.searchParams) + const call = this.calls.get(query.toolCallId ?? '') + if ( + request.method !== 'PUT' || + !call || + call.deviceId !== query.deviceId || + call.toolName !== 'import_local_files' || + call.token !== request.headers['x-sim-execution-token'] || + call.status !== 'running' + ) { + this.json(response, 404, { error: 'Desktop import not found' }) + return + } + // As Sim's route does: a file must declare its length, and arrive whole. + if (query.kind === 'file' && request.headers['content-length'] === undefined) { + this.json(response, 411, { error: 'A file import must declare its length' }) + return + } + const content = await this.raw(request) + if (query.kind === 'file' && content.length !== Number(request.headers['content-length'])) { + this.json(response, 400, { error: 'The file did not arrive whole' }) + return + } + const entry: ImportedEntry = { + toolCallId: call.toolCallId, + kind: query.kind ?? '', + sourceName: query.sourceName ?? '', + relativePath: query.relativePath ?? '', + ...(query.kind === 'file' ? { sha256: sha256(content), bytes: content.length } : {}), + } + this.imported.push(entry) + this.json(response, 200, { + id: `entry-${this.imported.length}`, + name: entry.relativePath.split('/').at(-1) || entry.sourceName, + }) + } + + private json(response: ServerResponse, status: number, body: unknown): void { + response.writeHead(status, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(body)) + } + + private signedIn(request: IncomingMessage): boolean { + return request.headers.cookie?.includes('better-auth.session_token=fixture') ?? false + } + + private async handle(request: IncomingMessage, response: ServerResponse): Promise { + const url = new URL(request.url ?? '/', this.origin) + const path = url.pathname + this.requests.push(`${request.method} ${path}`) + if (path.startsWith('/api/desktop/') && this.offline) { + this.droppedWhileOffline += 1 + request.socket.destroy() + return + } + if (path === '/api/auth/get-session') { + this.json( + response, + 200, + this.signedIn(request) ? { user: { id: 'user-e2e' }, session: { id: 'session-e2e' } } : null + ) + return + } + if (path.startsWith('/api/desktop/') && !this.signedIn(request)) { + this.json(response, 401, { error: 'Unauthorized' }) + return + } + if (path === '/api/desktop/devices' && request.method === 'POST') { + const body = await this.body(request) + if (this.enabled) { + this.devices.set(String(body.deviceId), { + name: String(body.name), + platform: String(body.platform), + }) + } + this.json(response, 200, { + enabled: this.enabled, + protocolVersion: 1, + leaseMs: 60_000, + leaseRenewMs: LEASE_RENEW_MS, + reconcileMs: RECONCILE_MS, + }) + return + } + const deviceId = url.searchParams.get('deviceId') + if (path === '/api/desktop/inbox/stream') { + if (!deviceId || !this.devices.has(deviceId)) { + this.json(response, 401, { error: 'Unregistered' }) + return + } + response.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }) + response.write(': connected\n\n') + const streams = this.streams.get(deviceId) ?? new Set() + streams.add(response) + this.streams.set(deviceId, streams) + request.on('close', () => streams.delete(response)) + return + } + if (path === '/api/desktop/inbox') { + if (!deviceId || !this.devices.has(deviceId)) { + this.json(response, 401, { error: 'Unregistered' }) + return + } + const items = [...this.calls.values()] + .filter((call) => call.deviceId === deviceId) + .flatMap((call) => { + if (call.status === 'pending' && !call.token) + return [ + { + kind: 'call', + toolCallId: call.toolCallId, + toolName: call.toolName, + chatId: call.chatId, + workspaceId: WORKSPACE, + createdAt: new Date(call.issuedAt).toISOString(), + }, + ] + if (call.status === 'awaiting_approval') + return [ + { + kind: 'approval_needed', + toolCallId: call.toolCallId, + toolName: call.toolName, + chatId: call.chatId, + chatTitle: 'Fix CI', + workspaceId: WORKSPACE, + summary: String((call.args.args as { command?: string } | undefined)?.command), + }, + ] + if (call.token && call.status === 'cancelled' && !call.acknowledged) + return [{ kind: 'cancel', toolCallId: call.toolCallId }] + return [] + }) + this.json(response, 200, { items }) + return + } + if (path === '/api/desktop/tool/import') { + await this.importEntry(url, request, response) + return + } + if (path.startsWith('/api/desktop/tool/')) { + const body = await this.body(request) + const call = this.calls.get(String(body.toolCallId)) + if (!call || call.deviceId !== body.deviceId) { + this.json(response, 404, { error: 'Desktop tool call not found' }) + return + } + if (path === '/api/desktop/tool/claim') { + if (call.status !== 'pending' || call.token) { + this.json(response, 404, { error: 'This call is no longer waiting for this device' }) + return + } + call.claims += 1 + call.status = 'running' + call.claimedAt = Date.now() + call.token = `token-${++this.nextToken}` + this.json(response, 200, { + toolName: call.toolName, + args: call.args, + chatId: call.chatId, + workspaceId: WORKSPACE, + executionToken: call.token, + }) + return + } + if (body.executionToken !== call.token) { + this.json(response, 404, { error: 'Desktop tool call not found' }) + return + } + if (path === '/api/desktop/tool/lease') { + call.renewals += 1 + if (call.status !== 'running') { + this.json(response, 410, { error: 'This call was stopped or settled. Stop running it.' }) + return + } + this.json(response, 200, { renewed: true }) + return + } + if (path === '/api/desktop/tool/complete') { + const status = String(body.status) + const settled = + status === 'success' ? 'completed' : status === 'cancelled' ? 'cancelled' : 'failed' + const outcome = + call.status === 'running' + ? 'recorded' + : call.completions.length > 0 + ? 'duplicate' + : 'superseded' + if (outcome === 'recorded') call.status = settled + if (call.status === 'cancelled') call.acknowledged = true + call.completions.push({ + status, + message: String(body.message ?? ''), + ...(body.data && typeof body.data === 'object' + ? { data: body.data as Record } + : {}), + outcome, + at: Date.now(), + }) + this.json(response, 200, { + outcome, + status: call.status === 'running' ? settled : call.status, + }) + return + } + } + if (path === '/hit') { + const chat = url.searchParams.get('chat') ?? '' + this.hits.set(chat, (this.hits.get(chat) ?? 0) + 1) + this.json(response, 200, { count: this.hits.get(chat) }) + return + } + if (path === '/session-ended') { + // Another account signing in replaces the session: the old session cookie is cleared. + response.writeHead(200, { + 'Content-Type': 'text/html', + 'Set-Cookie': 'better-auth.session_token=; Max-Age=0; HttpOnly; SameSite=Lax; Path=/', + }) + response.end('Signed in elsewhere') + return + } + response.writeHead(200, { + 'Content-Type': 'text/html', + 'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; SameSite=Lax; Path=/', + }) + response.end( + path === '/counter' + ? `Counter ${url.searchParams.get('chat')} + +

0

` + : `Sim fixture

${path}

` + ) + } +} + +export async function launch( + sim: FixtureSim, + userData: string, + env: Record = {} +): Promise<{ app: ElectronApplication; window: Page }> { + const app = await electron.launch({ + args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], + cwd: DESKTOP_DIR, + env: { + ...process.env, + SIM_DESKTOP_ORIGIN: sim.origin, + SIM_DESKTOP_USER_DATA: userData, + ...env, + }, + }) + // A dialog listener stops Playwright auto-dismissing page dialogs, so the desktop's own handling + // decides their outcome exactly as it does in production. + const leaveDialogsToDesktop = (page: Page) => page.on('dialog', () => {}) + app.context().pages().forEach(leaveDialogsToDesktop) + app.context().on('page', leaveDialogsToDesktop) + const window = await app.firstWindow() + return { app, window } +} + +/** The device this app registered, once it has opened its doorbell. */ +export async function registeredDevice( + sim: FixtureSim, + knownDevices: ReadonlySet = new Set() +): Promise { + let deviceId: string | undefined + await expect + .poll( + () => { + deviceId = [...sim.devices.keys()].find( + (id) => !knownDevices.has(id) && (sim.streams.get(id)?.size ?? 0) > 0 + ) + return deviceId + }, + { timeout: 30_000 } + ) + .toBeTruthy() + return deviceId as string +} + +export async function settled( + sim: FixtureSim, + toolCallId: string, + timeout = 60_000 +): Promise { + await expect + .poll(() => sim.requireCall(toolCallId).completions.length, { timeout }) + .toBeGreaterThan(0) + return sim.requireCall(toolCallId).completions[0] as Completion +} + +export function refFor(outline: string, name: string): number { + const match = outline + .split('\n') + .find((line) => line.includes(`"${name}"`) && /\[ref=\d+\]/.test(line)) + ?.match(/\[ref=(\d+)\]/) + if (!match) throw new Error(`No reference for ${name}: ${outline}`) + return Number(match[1]) +} + +export function processRunning(pattern: string): boolean { + try { + execFileSync('pgrep', ['-f', pattern]) + return true + } catch { + return false + } +} + +export function readFileSafe(path: string): string { + try { + return readFileSync(path, 'utf8') + } catch { + return '' + } +} diff --git a/apps/desktop/e2e/terminal-cancel.spec.ts b/apps/desktop/e2e/terminal-cancel.spec.ts new file mode 100644 index 00000000000..0d734390b33 --- /dev/null +++ b/apps/desktop/e2e/terminal-cancel.spec.ts @@ -0,0 +1,611 @@ +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdtempSync, readdirSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { type ElectronApplication, expect, type Page, test } from '@playwright/test' +import type { SimDesktopApi } from '@sim/desktop-bridge' +import { sleep } from '@sim/utils/helpers' +import { FixtureSim, launch, processRunning, registeredDevice, settled } from './executor-sim' + +/** + * Stopping the agent's terminal commands in the real app, with real shells and a real tmux + * server: Stop, sign-out (by the web app's logout and by the session cookie going away), switching + * Terminal off, and the launch-time recovery of an interrupted sign-out. What is asserted is the + * machine's own state (which processes still run, which tmux panes still exist), never which + * calls were made. + * + * Agent commands in a plain shell ignore SIGHUP, SIGINT and SIGTERM, so closing the shell cannot + * end them by accident: only Sim's own stop, which escalates to SIGKILL on the command's process + * group, does. The user's commands ignore the same signals, so a stop that reached them would + * show. Each launch gets its own tmux server (TMUX_TMPDIR), which the app's shells and its tmux + * calls inherit. The tmux scenarios skip where tmux is not installed. + */ + +const CHAT_PLAIN = 'chat-cancel-plain' +const CHAT_TMUX = 'chat-cancel-tmux' +const TMUX_SESSION = 'agent-e2e' +/** Commands that only a SIGKILL ends, so a shell closing around them proves nothing. */ +const STUBBORN = "trap '' HUP INT TERM" + +const sim = new FixtureSim() +/** The app's own log output for the current test, printed when it fails. */ +const appOutput: string[] = [] + +/** The process tree and tmux's clients, for a failure that is about who started what. */ +function diagnose(): string { + const run = (command: string, args: string[]) => { + try { + return execFileSync(command, args).toString() + } catch (error) { + return String(error) + } + } + return [ + run('ps', ['-eo', 'pid,ppid,pgid,args']) + .split('\n') + .filter((line) => /tmux|sleep|bash|zsh/.test(line)) + .join('\n'), + ].join('\n') +} +const REAL_TMUX = findTmux() + +/** + * Unique to this run of the suite, so its processes are never confused with those of another run on + * the same machine (CI boxes run several at once). + */ +const RUN_NONCE = String(Math.floor(Math.random() * 1_000_000)).padStart(6, '0') + +/** A sleep that runs for about `seconds` and that only this test names. */ +function slept(seconds: number): string { + return `sleep ${seconds}.${RUN_NONCE}` +} + +/** The command line a stubborn sleep runs as, unique to one test by its duration. */ +function stubborn(seconds: number): string { + return `bash -c "${STUBBORN}; ${slept(seconds)}"` +} + +function findTmux(): string | null { + try { + return execFileSync('sh', ['-c', 'command -v tmux']).toString().trim() || null + } catch { + return null + } +} + +/** One launch's own tmux server, and a way to read and reshape it as the user would. */ +class TmuxServer { + readonly dir = mkdtempSync('/tmp/sim-tmux-') + + run(args: string[]): string { + return execFileSync(REAL_TMUX ?? 'tmux', args, { + env: { ...process.env, TMUX_TMPDIR: this.dir }, + }).toString() + } + + /** Every pane: its id, its run tag (empty when untagged) and the command it was started with. */ + panes(): Array<{ id: string; runId: string; command: string }> { + try { + return this.run([ + 'list-panes', + '-a', + '-F', + '#{pane_id}\t#{@sim-run-id}\t#{pane_start_command}', + ]) + .trim() + .split('\n') + .filter(Boolean) + .map((line) => { + const [id = '', runId = '', command = ''] = line.split('\t') + return { id, runId, command } + }) + } catch { + return [] + } + } + + /** The pane an agent run opened, found by the run script it was started with. */ + runPane(): { id: string; runId: string } | undefined { + return this.panes().find((pane) => pane.command.includes('run.sh')) + } + + hasPane(id: string): boolean { + return this.panes().some((pane) => pane.id === id) + } + + /** The user splits a pane of their own beside the agent's, running their own command. */ + split(target: string, command: string): string { + return this.run(['split-window', '-d', '-t', target, '-P', '-F', '#{pane_id}', command]).trim() + } + + clients(): number { + try { + return this.run(['list-clients']).trim().split('\n').filter(Boolean).length + } catch { + return 0 + } + } + + kill(): void { + try { + this.run(['kill-server']) + } catch { + // Already gone. + } + } +} + +/** A `tmux` ahead of the real one on PATH that refuses pane options, as tmux before 3.0 does. */ +function oldTmuxPath(): string { + const dir = mkdtempSync(join(tmpdir(), 'sim-old-tmux-')) + const binary = join(dir, 'tmux') + writeFileSync( + binary, + `#!/bin/sh +if [ "$1" = set-option ] && [ "$2" = -p ]; then + printf 'tmux: unknown option -- p\\nusage: set-option [-aFgosquw] [-t target-window] option [value]\\n' >&2 + exit 1 +fi +exec ${JSON.stringify(REAL_TMUX)} "$@" +` + ) + chmodSync(binary, 0o755) + return `${dir}:${process.env.PATH ?? ''}` +} + +function bridge(window: Page) { + return { + /** The user opens a terminal tab in a chat's terminal panel and starts a command in it. */ + async userRuns(scope: string, command: string): Promise { + const terminalId = await window.evaluate(async (scopeId) => { + const terminal = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }) + .simDesktop.terminal + await terminal.activateScope(scopeId) + terminal.setVisible?.(true, scopeId) + terminal.setFocused(true, scopeId) + const opened = await terminal.openTerminal(undefined, scopeId) + return opened.activeTerminalId + }, scope) + if (!terminalId) throw new Error('No terminal tab opened') + // Typing is gated on a real keypress in the window, as it is for the panel's own input. + await window.keyboard.press('Shift') + await window.evaluate( + ([id, text, scopeId]) => + ( + globalThis as typeof globalThis & { simDesktop: SimDesktopApi } + ).simDesktop.terminal.write(id, text, scopeId), + [terminalId, `${command}\r`, scope] as const + ) + }, + /** The user puts a chat away: its terminals close, though tmux runs started in them go on. */ + async putAway(scope: string): Promise { + return window.evaluate(async (scopeId) => { + const terminal = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }) + .simDesktop.terminal + await terminal.activateScope(scopeId) + return terminal.suspendScope(scopeId) + }, scope) + }, + async setTerminalEnabled(enabled: boolean): Promise { + await window.evaluate( + (value) => + ( + globalThis as typeof globalThis & { simDesktop: SimDesktopApi } + ).simDesktop.settings.setPreference('terminalEnabled', value), + enabled + ) + }, + } +} + +/** The agent runs a plain-shell command in a chat's terminal; resolves once it is running. */ +async function agentRunsPlain(deviceId: string, seconds: number): Promise { + const call = sim.issue(deviceId, CHAT_PLAIN, 'terminal', { + operation: 'run', + args: { command: stubborn(seconds), waitSeconds: 120 }, + }) + await expect.poll(() => processRunning(`${slept(seconds)}`), { timeout: 30_000 }).toBe(true) + return call +} + +/** + * The agent's terminal attaches to tmux (the agent starts a tmux client in it), and the agent then + * runs a command, which tmux gives its own tagged pane. Resolves with the call and that pane. + */ +/** The agent starts a tmux client in its chat's terminal, so its runs go to tmux panes. */ +async function agentAttachesTmux(deviceId: string, tmux: TmuxServer): Promise { + const before = tmux.clients() + sim.issue(deviceId, CHAT_TMUX, 'terminal', { + operation: 'run', + args: { command: `tmux new-session -A -s ${TMUX_SESSION}`, waitSeconds: 2 }, + }) + await expect.poll(() => tmux.clients(), { timeout: 30_000 }).toBe(before + 1) + // The terminal's attachment is looked up again once its short cache lapses. + await sleep(3_500) +} + +/** + * The agent runs a command in tmux, which gives it its own tagged pane; with a short + * `waitSeconds` the call hands it back as still running, with that pane. + */ +async function agentRunsInTmux( + deviceId: string, + tmux: TmuxServer, + seconds: number, + waitSeconds = 120 +): Promise<{ call: string; pane: { id: string; runId: string } }> { + await agentAttachesTmux(deviceId, tmux) + // It ignores Ctrl-C, so stopping it takes closing its pane, the step that must spare the user's. + const call = sim.issue(deviceId, CHAT_TMUX, 'terminal', { + operation: 'run', + args: { + command: `bash -c "trap '' INT; echo collect-me; ${slept(seconds)}"`, + waitSeconds, + }, + }) + await expect.poll(() => processRunning(`${slept(seconds)}`), { timeout: 30_000 }).toBe(true) + await expect.poll(() => tmux.runPane(), { timeout: 10_000 }).toBeTruthy() + return { call, pane: tmux.runPane() as { id: string; runId: string } } +} + +async function claimed(call: string): Promise { + await expect.poll(() => sim.requireCall(call).claims, { timeout: 30_000 }).toBe(1) +} + +test.describe('terminal cancel', () => { + let app: ElectronApplication | null = null + let tmux: TmuxServer + const leftovers: string[] = [] + + test.beforeAll(async () => { + await sim.start() + }) + + test.beforeEach(() => { + tmux = new TmuxServer() + }) + + test.afterEach(async () => { + const testInfo = test.info() + if (testInfo.status !== testInfo.expectedStatus) { + // What each call came back with, and the panes tmux held, explain most failures. + const calls = [...sim.calls.values()].map((call) => ({ + toolCallId: call.toolCallId, + args: call.args, + status: call.status, + completions: call.completions.map(({ status, message, data }) => ({ + status, + message, + data, + })), + })) + await testInfo.attach('calls', { + body: JSON.stringify({ calls, panes: tmux.panes() }, null, 2), + }) + console.log(JSON.stringify({ calls, panes: tmux.panes() }, null, 2)) + console.log(appOutput.join('').slice(-6_000)) + console.log(diagnose()) + console.log( + (() => { + try { + return tmux.run(['list-clients', '-F', '#{client_pid} #{client_tty} #{client_session}']) + } catch (error) { + return String(error) + } + })() + ) + } + appOutput.length = 0 + // Quitting with a command still running in a tab asks first (natively, on macOS), and nobody + // is there to answer, so a close that does not finish promptly ends the app instead. + const closing = app + const closed = await Promise.race([ + closing?.close().then( + () => true, + () => true + ) ?? Promise.resolve(true), + sleep(5_000).then(() => false), + ]) + if (!closed) closing?.process().kill('SIGKILL') + app = null + tmux.kill() + for (const pattern of leftovers.splice(0)) { + try { + execFileSync('pkill', ['-9', '-f', pattern]) + } catch { + // Nothing left. + } + } + sim.reset() + }) + + test.afterAll(async () => { + await sim.stop() + }) + + async function start( + name: string, + env: Record = {} + ): Promise<{ window: Page; deviceId: string; userData: string }> { + const userData = mkdtempSync(join(tmpdir(), `sim-cancel-${name}-`)) + const launched = await launch(sim, userData, { TMUX_TMPDIR: tmux.dir, ...env }) + app = launched.app + launched.app.process().stdout?.on('data', (chunk: Buffer) => appOutput.push(chunk.toString())) + launched.app.process().stderr?.on('data', (chunk: Buffer) => appOutput.push(chunk.toString())) + const deviceId = await registeredDevice(sim) + await launched.window.goto(`${sim.origin}/workspace/ws-e2e/chat/${CHAT_PLAIN}`) + return { window: launched.window, deviceId, userData } + } + + test('Stop ends the agent command in a plain shell and leaves the user command running', async () => { + const { window, deviceId } = await start('stop-plain') + leftovers.push(slept(701), slept(702)) + // The agent's terminal comes first; the user's own tab opens beside it. + const call = await agentRunsPlain(deviceId, 701) + await bridge(window).userRuns(CHAT_PLAIN, stubborn(702)) + await expect.poll(() => processRunning(slept(702)), { timeout: 30_000 }).toBe(true) + + sim.stopCall(call) + + await settled(sim, call, 20_000) + await expect.poll(() => processRunning(slept(701)), { timeout: 15_000 }).toBe(false) + expect(processRunning(slept(702))).toBe(true) + }) + + test('Stop interrupts a tagged tmux run and closes only its pane, never the user pane beside it', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { deviceId } = await start('stop-tmux') + leftovers.push(slept(711), slept(712)) + const { call, pane } = await agentRunsInTmux(deviceId, tmux, 711) + expect(pane.runId).not.toBe('') + const users = tmux.split(pane.id, stubborn(712)) + + sim.stopCall(call) + + await settled(sim, call, 20_000) + await expect.poll(() => processRunning(slept(711)), { timeout: 15_000 }).toBe(false) + await expect.poll(() => tmux.hasPane(pane.id), { timeout: 10_000 }).toBe(false) + expect(tmux.hasPane(users)).toBe(true) + expect(processRunning(slept(712))).toBe(true) + }) + + test('signing out stops every agent command and leaves the user commands running', async () => { + const { window, deviceId } = await start('sign-out') + leftovers.push(slept(721), slept(722), slept(723), slept(724)) + const plain = await agentRunsPlain(deviceId, 721) + await claimed(plain) + await bridge(window).userRuns(CHAT_PLAIN, stubborn(722)) + await expect.poll(() => processRunning(slept(722)), { timeout: 30_000 }).toBe(true) + const run = REAL_TMUX ? await agentRunsInTmux(deviceId, tmux, 723) : null + const users = run ? tmux.split(run.pane.id, stubborn(724)) : null + + // The web app signs out by navigating to its login page. + await window.goto(`${sim.origin}/login?fromLogout=true`) + + await expect.poll(() => processRunning(slept(721)), { timeout: 30_000 }).toBe(false) + expect(processRunning(slept(722))).toBe(true) + if (run && users) { + await expect.poll(() => processRunning(slept(723)), { timeout: 15_000 }).toBe(false) + await expect.poll(() => tmux.hasPane(run.pane.id), { timeout: 10_000 }).toBe(false) + expect(tmux.hasPane(users)).toBe(true) + expect(processRunning(slept(724))).toBe(true) + } + }) + + test('a session ended by another account signing in stops every agent command', async () => { + const { window, deviceId } = await start('account') + leftovers.push(slept(731), slept(733)) + await agentRunsPlain(deviceId, 731) + const run = REAL_TMUX ? await agentRunsInTmux(deviceId, tmux, 733) : null + + // Another account signing in replaces the session: the old session cookie goes away. + await window.goto(`${sim.origin}/session-ended`) + + await expect.poll(() => processRunning(slept(731)), { timeout: 30_000 }).toBe(false) + if (run) { + await expect.poll(() => processRunning(slept(733)), { timeout: 15_000 }).toBe(false) + await expect.poll(() => tmux.hasPane(run.pane.id), { timeout: 10_000 }).toBe(false) + } + }) + + test('switching Terminal off stops every agent command and leaves the user commands running', async () => { + const { window, deviceId } = await start('terminal-off') + leftovers.push(slept(741), slept(743), slept(744)) + await agentRunsPlain(deviceId, 741) + const run = REAL_TMUX ? await agentRunsInTmux(deviceId, tmux, 743) : null + const users = run ? tmux.split(run.pane.id, stubborn(744)) : null + + await bridge(window).setTerminalEnabled(false) + + await expect.poll(() => processRunning(slept(741)), { timeout: 30_000 }).toBe(false) + if (run && users) { + await expect.poll(() => processRunning(slept(743)), { timeout: 15_000 }).toBe(false) + await expect.poll(() => tmux.hasPane(run.pane.id), { timeout: 10_000 }).toBe(false) + expect(tmux.hasPane(users)).toBe(true) + expect(processRunning(slept(744))).toBe(true) + } + }) + + test('signing out stops a tmux run whose chat was put away', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { window, deviceId, userData } = await start('put-away') + leftovers.push(slept(801)) + const { pane } = await agentRunsInTmux(deviceId, tmux, 801) + + // No live terminal holds the run any more; only its record does. + expect(await bridge(window).putAway(CHAT_TMUX)).toBe(true) + await sleep(1_000) + expect(processRunning(slept(801))).toBe(true) + + await window.goto(`${sim.origin}/login?fromLogout=true`) + + await expect.poll(() => processRunning(slept(801)), { timeout: 30_000 }).toBe(false) + expect(tmux.hasPane(pane.id)).toBe(false) + await expect.poll(() => recordedRuns(userData), { timeout: 10_000 }).toEqual([]) + }) + + /** The app dies without a word: its shells go with it, and its tmux runs outlive it. */ + function crash(): void { + app?.process().kill('SIGKILL') + app = null + } + + /** The runs this app has recorded for a later process to find. */ + function recordedRuns(userData: string): string[] { + try { + return readdirSync(join(userData, 'terminal-runs')) + } catch { + return [] + } + } + + test('the launch after an interrupted sign-out stops the tmux run the previous app left running', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { deviceId, userData } = await start('recovery') + leftovers.push(slept(751)) + const { pane } = await agentRunsInTmux(deviceId, tmux, 751) + + // The app dies mid sign-out: its recovery marker is on disk and the tmux run outlives it. + crash() + writeFileSync( + join(userData, 'account-data-teardown-required.json'), + JSON.stringify({ version: 2, kind: 'account', origin: sim.origin }) + ) + expect(processRunning(slept(751))).toBe(true) + + app = (await launch(sim, userData, { TMUX_TMPDIR: tmux.dir })).app + + await expect.poll(() => processRunning(slept(751)), { timeout: 30_000 }).toBe(false) + expect(tmux.hasPane(pane.id)).toBe(false) + await expect.poll(() => recordedRuns(userData), { timeout: 10_000 }).toEqual([]) + }) + + test('the launch after a crash stops the tmux run its call can no longer report', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { deviceId, userData } = await start('crash-live') + leftovers.push(slept(771), slept(772)) + const { pane } = await agentRunsInTmux(deviceId, tmux, 771) + const users = tmux.split(pane.id, stubborn(772)) + + // Same user, plain crash: the journal settles the call as outcome unknown, and nothing will + // ever read what the command does. + crash() + expect(processRunning(slept(771))).toBe(true) + app = (await launch(sim, userData, { TMUX_TMPDIR: tmux.dir })).app + + await expect.poll(() => processRunning(slept(771)), { timeout: 30_000 }).toBe(false) + expect(tmux.hasPane(pane.id)).toBe(false) + expect(tmux.hasPane(users)).toBe(true) + expect(processRunning(slept(772))).toBe(true) + await expect.poll(() => recordedRuns(userData), { timeout: 10_000 }).toEqual([]) + }) + + test('the launch after a crash only forgets a tmux run that had already finished', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { deviceId, userData } = await start('crash-finished') + leftovers.push(slept(6), slept(782)) + // Handed back as still running, it then finishes on its own. + const { call, pane } = await agentRunsInTmux(deviceId, tmux, 6, 2) + await settled(sim, call) + const users = tmux.split(pane.id, stubborn(782)) + await expect.poll(() => tmux.hasPane(pane.id), { timeout: 20_000 }).toBe(false) + expect(recordedRuns(userData)).toHaveLength(1) + + crash() + app = (await launch(sim, userData, { TMUX_TMPDIR: tmux.dir })).app + + await expect.poll(() => recordedRuns(userData), { timeout: 30_000 }).toEqual([]) + expect(tmux.hasPane(users)).toBe(true) + expect(processRunning(slept(782))).toBe(true) + }) + + test('a run handed back as still going survives a quit and the relaunch, and can still be read', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { deviceId, userData } = await start('collectable') + leftovers.push(slept(811)) + const { call, pane } = await agentRunsInTmux(deviceId, tmux, 811, 2) + const handedBack = await settled(sim, call) + expect(handedBack.data).toMatchObject({ status: 'running', pane: pane.id }) + + // Quitting stops no tmux run; the relaunch leaves one the model may come back to. + const quitting = app + app = null + const quit = await Promise.race([ + quitting?.close().then(() => true) ?? Promise.resolve(true), + sleep(10_000).then(() => false), + ]) + if (!quit) quitting?.process().kill('SIGKILL') + expect(processRunning(slept(811))).toBe(true) + app = (await launch(sim, userData, { TMUX_TMPDIR: tmux.dir })).app + // Same install, same device: it is back once its doorbell is open again. + const relaunched = await registeredDevice(sim) + await sleep(5_000) + expect(processRunning(slept(811))).toBe(true) + expect(tmux.hasPane(pane.id)).toBe(true) + expect(recordedRuns(userData)).toHaveLength(1) + + // The model comes back to the pane it was handed. + await agentAttachesTmux(relaunched, tmux) + const read = sim.issue(relaunched, CHAT_TMUX, 'terminal', { + operation: 'read', + args: { pane: pane.id, lines: 50 }, + }) + const output = await settled(sim, read) + expect(output.status, output.message).toBe('success') + expect(JSON.stringify(output.data)).toContain('collect-me') + }) + + test("never touches a pane that took a recorded run's id after tmux restarted", async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { deviceId, userData } = await start('stale-record') + leftovers.push(slept(791), slept(792)) + const { pane } = await agentRunsInTmux(deviceId, tmux, 791) + crash() + + // tmux restarts, ending the run with it, and the user's own pane gets the run's old id. + tmux.kill() + // The old server takes a moment to let go of its socket. + await expect + .poll( + () => { + try { + tmux.run(['-f', '/dev/null', 'new-session', '-d', '-s', 'users', stubborn(792)]) + return true + } catch { + return false + } + }, + { timeout: 10_000 } + ) + .toBe(true) + for (let panes = 0; panes < 10 && !tmux.hasPane(pane.id); panes += 1) { + tmux.split('users', stubborn(792)) + } + expect(processRunning(slept(791))).toBe(false) + + app = (await launch(sim, userData, { TMUX_TMPDIR: tmux.dir })).app + + await expect.poll(() => recordedRuns(userData), { timeout: 30_000 }).toEqual([]) + expect(tmux.hasPane(pane.id)).toBe(true) + expect(processRunning(slept(792))).toBe(true) + }) + + test('on tmux without pane options a run goes ahead untracked, and Stop and sign-out leave it alone', async () => { + test.skip(!REAL_TMUX, 'tmux is not installed on this machine') + const { window, deviceId } = await start('untracked', { PATH: oldTmuxPath() }) + leftovers.push(slept(761)) + const { call, pane } = await agentRunsInTmux(deviceId, tmux, 761) + expect(pane.runId).toBe('') + + sim.stopCall(call) + await settled(sim, call, 20_000) + await sleep(6_000) + expect(processRunning(slept(761))).toBe(true) + expect(tmux.hasPane(pane.id)).toBe(true) + + await window.goto(`${sim.origin}/login?fromLogout=true`) + await sleep(8_000) + expect(processRunning(slept(761))).toBe(true) + expect(tmux.hasPane(pane.id)).toBe(true) + }) +}) From f8ce0081608ced6173c156c7606ca23dec3d1110 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 00:43:31 -0700 Subject: [PATCH 2/4] test(desktop): draw the run nonce from the shared random helpers, and keep the lease constant private --- apps/desktop/e2e/executor-sim.ts | 2 +- apps/desktop/e2e/terminal-cancel.spec.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/desktop/e2e/executor-sim.ts b/apps/desktop/e2e/executor-sim.ts index 1273052f7ef..af7a6d3da12 100644 --- a/apps/desktop/e2e/executor-sim.ts +++ b/apps/desktop/e2e/executor-sim.ts @@ -18,7 +18,7 @@ import { const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) export const WORKSPACE = 'ws-e2e' -export const LEASE_RENEW_MS = 1_000 +const LEASE_RENEW_MS = 1_000 export const RECONCILE_MS = 2_000 type CallStatus = 'pending' | 'awaiting_approval' | 'running' | 'completed' | 'failed' | 'cancelled' diff --git a/apps/desktop/e2e/terminal-cancel.spec.ts b/apps/desktop/e2e/terminal-cancel.spec.ts index 0d734390b33..4c33d3e3117 100644 --- a/apps/desktop/e2e/terminal-cancel.spec.ts +++ b/apps/desktop/e2e/terminal-cancel.spec.ts @@ -5,6 +5,7 @@ import { join } from 'node:path' import { type ElectronApplication, expect, type Page, test } from '@playwright/test' import type { SimDesktopApi } from '@sim/desktop-bridge' import { sleep } from '@sim/utils/helpers' +import { randomInt } from '@sim/utils/random' import { FixtureSim, launch, processRunning, registeredDevice, settled } from './executor-sim' /** @@ -53,7 +54,7 @@ const REAL_TMUX = findTmux() * Unique to this run of the suite, so its processes are never confused with those of another run on * the same machine (CI boxes run several at once). */ -const RUN_NONCE = String(Math.floor(Math.random() * 1_000_000)).padStart(6, '0') +const RUN_NONCE = String(randomInt(0, 1_000_000)).padStart(6, '0') /** A sleep that runs for about `seconds` and that only this test names. */ function slept(seconds: number): string { From aa1ee574dbb3f81b56214ee663a6adf63dbc3943 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 00:44:39 -0700 Subject: [PATCH 3/4] test(desktop): attach terminal-cancel diagnostics instead of printing them, write a suite report, and keep the user pane through a session ended elsewhere --- .github/workflows/desktop-e2e.yml | 1 + apps/desktop/e2e/terminal-cancel.spec.ts | 62 +++++++++++++----------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index 410464cefa2..2981d76cf48 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -90,6 +90,7 @@ jobs: run: bunx playwright test env: BACKGROUND_EXECUTOR_REPORT_PATH: test-results/background-executor-report.json + TERMINAL_CANCEL_REPORT_PATH: test-results/terminal-cancel-report.json - name: Upload test results if: failure() diff --git a/apps/desktop/e2e/terminal-cancel.spec.ts b/apps/desktop/e2e/terminal-cancel.spec.ts index 4c33d3e3117..7e8345384d4 100644 --- a/apps/desktop/e2e/terminal-cancel.spec.ts +++ b/apps/desktop/e2e/terminal-cancel.spec.ts @@ -29,24 +29,22 @@ const TMUX_SESSION = 'agent-e2e' const STUBBORN = "trap '' HUP INT TERM" const sim = new FixtureSim() -/** The app's own log output for the current test, printed when it fails. */ +/** The app's own log output for the current test, attached when it fails. */ const appOutput: string[] = [] +/** Each scenario's outcome, written to `TERMINAL_CANCEL_REPORT_PATH` when the suite ends. */ +const report: Array<{ name: string; status: string; durationMs: number }> = [] -/** The process tree and tmux's clients, for a failure that is about who started what. */ -function diagnose(): string { - const run = (command: string, args: string[]) => { - try { - return execFileSync(command, args).toString() - } catch (error) { - return String(error) - } - } - return [ - run('ps', ['-eo', 'pid,ppid,pgid,args']) +/** The shells, sleeps and tmux processes running, for a failure about who started what. */ +function processes(): string { + try { + return execFileSync('ps', ['-eo', 'pid,ppid,pgid,args']) + .toString() .split('\n') .filter((line) => /tmux|sleep|bash|zsh/.test(line)) - .join('\n'), - ].join('\n') + .join('\n') + } catch (error) { + return String(error) + } } const REAL_TMUX = findTmux() @@ -268,6 +266,11 @@ test.describe('terminal cancel', () => { test.afterEach(async () => { const testInfo = test.info() + report.push({ + name: testInfo.title, + status: testInfo.status ?? 'unknown', + durationMs: testInfo.duration, + }) if (testInfo.status !== testInfo.expectedStatus) { // What each call came back with, and the panes tmux held, explain most failures. const calls = [...sim.calls.values()].map((call) => ({ @@ -283,18 +286,8 @@ test.describe('terminal cancel', () => { await testInfo.attach('calls', { body: JSON.stringify({ calls, panes: tmux.panes() }, null, 2), }) - console.log(JSON.stringify({ calls, panes: tmux.panes() }, null, 2)) - console.log(appOutput.join('').slice(-6_000)) - console.log(diagnose()) - console.log( - (() => { - try { - return tmux.run(['list-clients', '-F', '#{client_pid} #{client_tty} #{client_session}']) - } catch (error) { - return String(error) - } - })() - ) + await testInfo.attach('app-output', { body: appOutput.join('').slice(-6_000) }) + await testInfo.attach('processes', { body: processes() }) } appOutput.length = 0 // Quitting with a command still running in a tab asks first (natively, on macOS), and nobody @@ -322,6 +315,13 @@ test.describe('terminal cancel', () => { test.afterAll(async () => { await sim.stop() + const reportPath = process.env.TERMINAL_CANCEL_REPORT_PATH + if (reportPath) { + writeFileSync( + reportPath, + JSON.stringify({ suite: 'terminal-cancel', checks: report }, null, 2) + ) + } }) async function start( @@ -393,19 +393,23 @@ test.describe('terminal cancel', () => { } }) - test('a session ended by another account signing in stops every agent command', async () => { + test('a session ended by another account signing in stops every agent command and leaves the user pane running', async () => { const { window, deviceId } = await start('account') - leftovers.push(slept(731), slept(733)) + leftovers.push(slept(731), slept(733), slept(734)) await agentRunsPlain(deviceId, 731) const run = REAL_TMUX ? await agentRunsInTmux(deviceId, tmux, 733) : null + const users = run ? tmux.split(run.pane.id, stubborn(734)) : null + if (users) await expect.poll(() => processRunning(slept(734)), { timeout: 15_000 }).toBe(true) // Another account signing in replaces the session: the old session cookie goes away. await window.goto(`${sim.origin}/session-ended`) await expect.poll(() => processRunning(slept(731)), { timeout: 30_000 }).toBe(false) - if (run) { + if (run && users) { await expect.poll(() => processRunning(slept(733)), { timeout: 15_000 }).toBe(false) await expect.poll(() => tmux.hasPane(run.pane.id), { timeout: 10_000 }).toBe(false) + expect(tmux.hasPane(users)).toBe(true) + expect(processRunning(slept(734))).toBe(true) } }) From 53aab9b7028a823123bd23906a6762e3ea38c3ec Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 01:25:56 -0700 Subject: [PATCH 4/4] test(desktop): add each terminal-cancel check to the report as it finishes, so a retry keeps the original failure --- apps/desktop/e2e/terminal-cancel.spec.ts | 40 +++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/apps/desktop/e2e/terminal-cancel.spec.ts b/apps/desktop/e2e/terminal-cancel.spec.ts index 7e8345384d4..9b1e08a9d04 100644 --- a/apps/desktop/e2e/terminal-cancel.spec.ts +++ b/apps/desktop/e2e/terminal-cancel.spec.ts @@ -1,5 +1,5 @@ import { execFileSync } from 'node:child_process' -import { chmodSync, mkdtempSync, readdirSync, writeFileSync } from 'node:fs' +import { chmodSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { type ElectronApplication, expect, type Page, test } from '@playwright/test' @@ -31,8 +31,32 @@ const STUBBORN = "trap '' HUP INT TERM" const sim = new FixtureSim() /** The app's own log output for the current test, attached when it fails. */ const appOutput: string[] = [] -/** Each scenario's outcome, written to `TERMINAL_CANCEL_REPORT_PATH` when the suite ends. */ -const report: Array<{ name: string; status: string; durationMs: number }> = [] +interface ReportCheck { + name: string + status: string + durationMs: number + retry: number +} + +/** + * Adds a scenario's outcome to the report at `TERMINAL_CANCEL_REPORT_PATH` as soon as it is known. + * Playwright replaces the worker after a failure, so the file, not the worker's memory, holds what + * came before: a failure and its retry both stay in it. + */ +function reportCheck(check: ReportCheck): void { + const reportPath = process.env.TERMINAL_CANCEL_REPORT_PATH + if (!reportPath) return + let checks: ReportCheck[] = [] + try { + checks = (JSON.parse(readFileSync(reportPath, 'utf8')) as { checks: ReportCheck[] }).checks + } catch { + // The first check of the run. + } + writeFileSync( + reportPath, + JSON.stringify({ suite: 'terminal-cancel', checks: [...checks, check] }, null, 2) + ) +} /** The shells, sleeps and tmux processes running, for a failure about who started what. */ function processes(): string { @@ -266,10 +290,11 @@ test.describe('terminal cancel', () => { test.afterEach(async () => { const testInfo = test.info() - report.push({ + reportCheck({ name: testInfo.title, status: testInfo.status ?? 'unknown', durationMs: testInfo.duration, + retry: testInfo.retry, }) if (testInfo.status !== testInfo.expectedStatus) { // What each call came back with, and the panes tmux held, explain most failures. @@ -315,13 +340,6 @@ test.describe('terminal cancel', () => { test.afterAll(async () => { await sim.stop() - const reportPath = process.env.TERMINAL_CANCEL_REPORT_PATH - if (reportPath) { - writeFileSync( - reportPath, - JSON.stringify({ suite: 'terminal-cancel', checks: report }, null, 2) - ) - } }) async function start(