improvement(ci): run the live desktop suite unless every change is clearly unrelated, and fail open

- The change check skips only docs, the other apps and published content; any change elsewhere,
  and any failure to fetch or diff the base, runs the suite
- settled() discounts only requests a hold is keeping from Sim right now
- The warm-up's Stop turn waits for its stream to close instead of on a promise that never settles

Author: waleedlatif1

.github/scripts/desktop-live-changes.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Prints `changed=false` only when every file a pull request changes is clearly unrelated to the
+# live desktop suite, and `changed=true` otherwise, including when the diff cannot be worked out.
+#
+# Usage: desktop-live-changes.sh <base-sha>
+set -u
+
+unrelated='^(apps/docs/|apps/pii/|apps/sim/content/|packages/(python-sdk|ts-sdk)/)|\.mdx?$|(^|/)LICENSE$'
+
+base=${1:-}
+run() {
+  echo "changed=true"
+  echo "Running the live desktop suite: $1" >&2
+  exit 0
+}
+[ -n "$base" ] || run 'no base commit'
+git fetch --quiet --depth=1 origin "$base" || run "could not fetch $base"
+names=$(git diff --name-only "$base" HEAD) || run "could not diff against $base"
+[ -n "$names" ] || run 'no changed files listed'
+if printf '%s\n' "$names" | grep -qvE "$unrelated"; then
+  run 'a change may affect it'
+fi
+echo "changed=false"
+echo 'Skipping the live desktop suite: every change is unrelated to it' >&2

.github/workflows/test-build.yml

@@ -442,35 +442,26 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
-  # Pull requests run the live desktop suite only when they touch what it exercises. The diff
-  # comes from git against the pull request's base, so it needs no API access.
+  # Pull requests skip the live desktop suite only when every change is clearly unrelated to the
+  # app it drives (docs, other apps, published content). Anything else, and any failure to work
+  # out the diff, runs it: a pull request that skipped it wrongly would first fail on staging.
   desktop-live-changes:
     name: Detect desktop tool changes
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     outputs:
-      changed: ${{ github.event_name != 'pull_request' || steps.diff.outputs.changed == 'true' }}
+      changed: ${{ github.event_name != 'pull_request' || steps.diff.outputs.changed != 'false' }}
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
         if: github.event_name == 'pull_request'
         with:
           fetch-depth: 2
-      - name: Diff desktop tool paths
+      - name: Diff against the pull request's base
         id: diff
         if: github.event_name == 'pull_request'
         env:
           BASE: ${{ github.event.pull_request.base.sha }}
-        run: |
-          git fetch --quiet --depth=1 origin "$BASE"
-          paths='^(\.github/workflows/test-build\.yml|\.github/scripts/stop-session\.sh|bun\.lock'
-          paths="$paths|apps/desktop/|apps/realtime/|packages/(browser-protocol|db|desktop-bridge|terminal-protocol)/"
-          paths="$paths|apps/sim/app/api/(auth|copilot|desktop|mothership)/|apps/sim/app/workspace/[^/]+/(home|chat)/"
-          paths="$paths|apps/sim/lib/(auth|desktop|mothership)/|apps/sim/stores/)"
-          if git diff --name-only "$BASE" HEAD | grep -qE "$paths"; then
-            echo "changed=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-          fi
+        run: bash .github/scripts/desktop-live-changes.sh "$BASE" >> "$GITHUB_OUTPUT"
 
   # Desktop tools in the real Electron app against a local app, on its own runner: the
   # Electron app, the dev app and its realtime server together outgrow the http-e2e runner.

apps/desktop/e2e/desktop-tools-live-sim.spec.ts

@@ -182,7 +182,8 @@ test.describe('desktop tools against a live Sim', () => {
       )
       agent.script('[warm-stop]', async (turn) => {
         turn.text('Stopping soon.')
-        await new Promise<void>(() => {})
+        // The leg stays open until Stop ends it.
+        await turn.closed
       })
       const page = await openApp(user, 'Warm chat', COMPILE_MS)
       await send(page, '[warm-up] read and import', COMPILE_MS)

apps/desktop/e2e/fixtures/live-sim.ts

@@ -170,6 +170,8 @@ export class SimProxy {
   rewrittenChatBodies = 0
   private readonly server: Server
   private holds: HeldRequest[] = []
+  /** Requests a hold is keeping from Sim right now. */
+  private readonly heldEntries = new Set<ProxiedRequest>()
   private chatBodyRewrite: ((body: Record<string, unknown>) => void) | undefined
   private readonly sockets = new Set<Duplex>()
 
@@ -248,9 +250,12 @@ export class SimProxy {
     let quietSince = Date.now()
     for (;;) {
       const waiting = this.requests.filter(
-        (entry) => entry.status === undefined && entry.clientClosedAt === undefined
+        (entry) =>
+          entry.status === undefined &&
+          entry.clientClosedAt === undefined &&
+          !this.heldEntries.has(entry)
       )
-      if (waiting.length > this.holds.length) quietSince = Date.now()
+      if (waiting.length > 0) quietSince = Date.now()
       else if (Date.now() - quietSince >= 3_000) return
       if (Date.now() > deadline)
         throw new Error(
@@ -287,7 +292,10 @@ export class SimProxy {
     const held = this.holds.find((candidate) => candidate.matches(method, url.pathname))
     if (held) {
       this.holds = this.holds.filter((candidate) => candidate !== held)
-      if (!(await held.hold(entry, response)) && !held.deliverIfAbandoned) return
+      this.heldEntries.add(entry)
+      const deliver = await held.hold(entry, response)
+      this.heldEntries.delete(entry)
+      if (!deliver && !held.deliverIfAbandoned) return
     }
     if (this.chatBodyRewrite && method === 'POST' && url.pathname === '/api/mothership/chat') {
       const parsed: Record<string, unknown> = JSON.parse(body.toString('utf8'))
@@ -346,6 +354,9 @@ interface Resume {
 /** A chat turn Sim opened against the agent, written to as the scripted model acts. */
 class AgentTurn {
   readonly toolCallIds: string[] = []
+  /** Settles once this leg's stream has ended, from either side. */
+  readonly closed: Promise<void>
+  private markClosed!: () => void
   private seq = 0
   private readonly keepAlive: ReturnType<typeof setInterval>
   private ended = false
@@ -355,6 +366,9 @@ class AgentTurn {
     readonly streamId: string,
     private readonly response: ServerResponse
   ) {
+    this.closed = new Promise((resolve) => {
+      this.markClosed = resolve
+    })
     response.writeHead(200, {
       'Content-Type': 'text/event-stream',
       'Cache-Control': 'no-cache',
@@ -432,6 +446,7 @@ class AgentTurn {
     this.ended = true
     clearInterval(this.keepAlive)
     this.response.end()
+    this.markClosed()
   }
 }