[rush-client-core] Bound the daemon startup reservation and surface startup errors

apps/rush-cli-client/src/launchClient.ts

@@ -177,6 +177,7 @@ export async function launchClientAsync(
       writeStreamAsync(stream === 'stderr' ? process.stderr : process.stdout, bytes)
   });
   let outcome: DaemonClientOutcome | undefined;
+  let restartFailure: DaemonClientError | undefined;
   const discoveryLines: string[] = [];
   const writeDiscoveryAsync = async (): Promise<void> => {
     if (discoveryLines.length > 0) {
@@ -226,8 +227,13 @@ export async function launchClientAsync(
         : undefined
     });
   } catch (error) {
+    if (!(error instanceof DaemonClientError)) throw error;
+    if (!abort.signal.aborted) {
+      // A restart handoff fails only before the request executes, so in-process fallback cannot replay work.
+      if (error.code !== 'startupFailed') throw error;
+      restartFailure = error;
+    }
     // After cancellation, a transport failure (e.g. the cancellation deadline) still means "cancelled".
-    if (!abort.signal.aborted || !(error instanceof DaemonClientError)) throw error;
     outcome = undefined;
   } finally {
     for (const signal of CANCELLATION_SIGNALS) process.removeListener(signal, onSignal);
@@ -237,6 +243,12 @@ export async function launchClientAsync(
       await client.closeAsync();
     }
   }
+  if (restartFailure) {
+    agentRenderer?.dispose();
+    process.stderr.write(`rush-client: ${restartFailure.message} Using in-process Rush.\n`);
+    launchInProcess(route.argv, rushx, selectedVersion);
+    return;
+  }
   if (outcome === undefined || isCancelledOutcome(outcome, abort.signal.aborted)) {
     const exitCode: number = getSignalExitCode(cancellationSignal ?? 'SIGINT');
     agentRenderer?.finish({ exitCode, errorMessage: 'cancelled' });

common/changes/@rushstack/rush-cli-client/startup-wedge_2026-09-24-01-30.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-cli-client",
+      "comment": "Print the daemon's error message for failed results, and fall back to in-process Rush when a pre-execution daemon restart cannot start.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-cli-client",
+  "email": "selarkin@microsoft.com"
+}

common/changes/@rushstack/rush-client-core/startup-wedge_2026-09-24-01-30.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-client-core",
+      "comment": "Make the daemon startup reservation bounded and verifiable: record owner and launcher PIDs, release it when the launcher exits before readiness, accept a ready daemon despite a reservation, reclaim stale reservations without waiting for the deadline, and include the launcher log's last error lines in startup failures.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-client-core",
+  "email": "selarkin@microsoft.com"
+}

common/changes/@rushstack/rush-daemon/startup-wedge_2026-09-24-01-30.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-daemon",
+      "comment": "Validate a request's Rush environment before planning a process restart, so an invalid value fails the request with the native message and keeps the current daemon running.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-daemon",
+  "email": "selarkin@microsoft.com"
+}

libraries/rush-client-core/README.md

@@ -48,7 +48,7 @@ aware `LockFile` for the first-start mutex, including kernel-enforced exclusive
 file sharing on Windows. The winning client rechecks readiness,
 reclaims only an absent/dead owner, and reserves `<lockfilePath>.starting` before
 handing the explicit command to a detached startup helper. The helper spawns without
-a shell and retains that reservation until the daemon completes hello/ping readiness,
+a shell and holds that reservation until the daemon completes hello/ping readiness,
 independently of whether the requesting client survives. Clients still await
 hello/pong under bounded backoff. Stdout/stderr go to `<lockfilePath>.log`. No PID
 is killed. While holding the mutex with no startup reservation, stale leftovers are
@@ -61,14 +61,34 @@ which removes the record, socket and reservation after the same no-listener/no-l
 The helper uses a stable tool cwd, and the starting client awaits its exit after
 readiness. The explicit launcher's cwd is unchanged.
 
-An unresolved startup reservation is never automatically reclaimed based on PID
-liveness or elapsed time. If the helper cannot establish readiness, subsequent starts
-fail closed instead of risking a second detached daemon. Only a known spawn failure
-(no executable started) releases the reservation immediately. An arbitrary launcher
-can spawn descendants, so its exit is not proof that another launch is safe.
-Recovery of an abandoned reservation requires operator confirmation that the original
-startup cannot still publish an endpoint; normal successful startup releases it
-automatically. Cancellation stops the client waiting, not the detached handoff.
+The startup reservation is bounded and verifiable. It is a JSON record of its token,
+creation time, startup timeout, the process responsible for releasing it (the starting
+client until the helper is spawned, then the helper, with its start time) and the
+launcher PID once spawned. The reservation is released when:
+
+- the daemon completes hello/ping readiness (by the helper);
+- no executable could be started, or the launcher the helper started exits before
+  publishing an endpoint (by the helper). The client reports this immediately with the
+  last lines of `<lockfilePath>.log`, such as the launcher's own error;
+- any client finds a ready endpoint whose published ownership record matches the
+  hello/ping status while the reservation's owner is gone. Such an endpoint is used even
+  while a reservation exists; a live helper releases its own reservation;
+- a client holding the first-start mutex finds it stale: neither its owner nor its
+  launcher is alive, or it has outlived its own startup timeout by a fixed 60-second
+  grace period (this bounds PID reuse and a wedged launcher). Unrecognized records, such
+  as token-only reservations from older clients, have no verifiable owner and become
+  stale by file age. A stale reservation is reclaimed without waiting for the deadline.
+
+Every check-then-write of the reservation (owner/launcher updates, release, stale
+reclaim, and cleanup after a dead owner) holds a short `<lockfile>-reservation` lock,
+so a resumed stale owner can never overwrite or remove a replacement reservation.
+
+Otherwise, a live reservation makes later starts wait for readiness, then fail with the
+reservation's owner/launcher state instead of launching a second daemon. A launcher that
+misses the deadline but later binds is still accepted. If a stale reservation is
+reclaimed while an old launcher is merely suspended, the transport's bind-time ownership
+check still rejects whichever daemon binds second, so one workspace never has two
+serving daemons. Cancellation stops the client waiting, not the detached handoff.
 
 If a wire-compatible daemon reports the wrong implementation version and an explicit
 replacement launcher is available, startup serializes replacement under that same mutex.
@@ -129,6 +149,5 @@ graph reference client. A successful handshake is still transport readiness, not
 guarantee that every command or configuration is supported. Version-selected daemon
 installation and incompatible-protocol replacement remain
 separate integration work; this core package does not construct an engine.
-The startup helper and durable pre-bind reservation protect concurrent first-invocations
-even if the original client dies. They do not add a new daemon protocol or authorize
-automatic recovery of ambiguous launcher failures.
+The startup helper and bounded pre-bind reservation protect concurrent first-invocations
+even if the original client dies. They do not add a new daemon protocol.

libraries/rush-client-core/src/DaemonLogFile.ts

@@ -1,9 +1,51 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
 
+import * as fs from 'node:fs';
+
 import type { IDaemonPaths } from '@rushstack/rush-daemon-transport';
 
+const MAX_LOG_TAIL_BYTES: number = 16384;
+const MAX_LOG_TAIL_LINES: number = 3;
+const MAX_LOG_LINE_LENGTH: number = 500;
+
 /** The workspace launcher's persistent stdout/stderr log, independent of daemon lifetime. @beta */
 export function getDaemonLogFilePath(paths: IDaemonPaths): string {
   return `${paths.lockfilePath}.log`;
 }
+
+/** Returns the log's current size, used to scope later diagnostics to one startup attempt. */
+export function getDaemonLogFileSize(paths: IDaemonPaths): number {
+  return fs.statSync(getDaemonLogFilePath(paths), { throwIfNoEntry: false })?.size ?? 0;
+}
+
+/**
+ * Formats the last launcher log lines written after `fromOffset`, preferring error lines and omitting
+ * stack frames, so startup failures show their real cause. Returns an empty string if nothing is available.
+ */
+export function formatDaemonLogTail(paths: IDaemonPaths, fromOffset: number = 0): string {
+  let text: string;
+  try {
+    const fd: number = fs.openSync(getDaemonLogFilePath(paths), 'r');
+    try {
+      const size: number = fs.fstatSync(fd).size;
+      const start: number = Math.max(fromOffset > size ? 0 : fromOffset, size - MAX_LOG_TAIL_BYTES);
+      const buffer: Buffer = Buffer.alloc(size - start);
+      fs.readSync(fd, buffer, 0, buffer.length, start);
+      text = buffer.toString('utf8');
+    } finally {
+      fs.closeSync(fd);
+    }
+  } catch {
+    return '';
+  }
+  const meaningful: string[] = text
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !line.startsWith('at '));
+  const errors: string[] = meaningful.filter((line) => /error/i.test(line));
+  const lines: string[] = (errors.length > 0 ? errors : meaningful)
+    .slice(-MAX_LOG_TAIL_LINES)
+    .map((line) => (line.length > MAX_LOG_LINE_LENGTH ? `${line.slice(0, MAX_LOG_LINE_LENGTH)}...` : line));
+  return lines.length > 0 ? `\nLast launcher log lines:\n  ${lines.join('\n  ')}\n` : '';
+}

libraries/rush-client-core/src/DaemonStartup.ts

@@ -2,9 +2,7 @@
 // See LICENSE in the project root for license information.
 
 import { spawn, type ChildProcess } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
 import { once } from 'node:events';
-import * as fs from 'node:fs';
 import { setTimeout as delayAsync } from 'node:timers/promises';
 
 import {
@@ -16,6 +14,18 @@ import {
 import type { IDaemonStartCommand } from './connectOrStartDaemon';
 import { DaemonClient } from './DaemonClient';
 import { DaemonClientError } from './DaemonClientError';
+import {
+  assertDaemonStartupReservation,
+  getDaemonStartupFilePath,
+  releaseDaemonStartup,
+  updateDaemonStartupReservation
+} from './DaemonStartupReservation';
+
+export {
+  getDaemonStartupFilePath,
+  releaseDaemonStartup,
+  reserveDaemonStartup
+} from './DaemonStartupReservation';
 
 export interface IDaemonStartupOptions {
   readonly paths: IDaemonPaths;
@@ -24,35 +34,16 @@ export interface IDaemonStartupOptions {
   readonly timeoutMs: number;
 }
 
-export function getDaemonStartupFilePath(paths: IDaemonPaths): string {
-  return `${paths.lockfilePath}.starting`;
-}
-
-export function reserveDaemonStartup(paths: IDaemonPaths): string {
-  const token: string = randomUUID();
-  fs.writeFileSync(getDaemonStartupFilePath(paths), token, { flag: 'wx', mode: 0o600 });
-  return token;
-}
-
-function assertReservation(paths: IDaemonPaths, token: string): void {
-  if (fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8') !== token) {
-    throw new DaemonClientError('startupFailed', 'The daemon startup reservation changed ownership.');
-  }
-}
-
-export function releaseDaemonStartup(paths: IDaemonPaths, token: string): void {
-  assertReservation(paths, token);
-  fs.unlinkSync(getDaemonStartupFilePath(paths));
-}
-
 /**
- * Runs independently of the requesting client. Once spawn succeeds, only protocol readiness releases
- * the reservation: an arbitrary launcher may outlive its parent or spawn descendants.
- * Failure before readiness deliberately leaves a durable reservation instead of guessing that a PID is safe.
+ * Runs independently of the requesting client. The reservation records this helper and its launcher PID,
+ * so later starters can verify whether the startup can still make progress. Protocol readiness releases
+ * the reservation; so does a launcher that exits without publishing an endpoint. A deadline miss while the
+ * launcher is still alive keeps the reservation, which later becomes stale when both processes are gone or
+ * the bounded grace period elapses.
  */
 export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Promise<void> {
   const { paths, startCommand: start, token, timeoutMs } = options;
-  assertReservation(paths, token);
+  assertDaemonStartupReservation(paths, token);
   let child: ChildProcess;
   let closed: Promise<void> | undefined;
   try {
@@ -72,6 +63,7 @@ export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Pro
     throw error;
   }
   child.unref();
+  if (child.pid !== undefined) updateDaemonStartupReservation(paths, token, { launcherPid: child.pid });
 
   const deadline: number = Date.now() + timeoutMs;
   let backoffMs: number = 50;
@@ -104,16 +96,24 @@ export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Pro
     }
     if (child.exitCode !== null || child.signalCode !== null) {
       await closed;
+      // The launcher this helper started is gone without publishing an endpoint. If it left a descendant
+      // that binds later, the transport's bind-time ownership check still rejects a second daemon.
+      releaseDaemonStartup(paths, token);
       throw new DaemonClientError(
         'startupFailed',
-        `Launcher exited (${child.exitCode ?? child.signalCode}) before protocol readiness; startup reservation retained.`
+        `Daemon launcher ${describeExit(child)} before protocol readiness; startup reservation released.`
       );
     }
     await delayAsync(Math.min(backoffMs, Math.max(1, deadline - Date.now())));
     backoffMs = Math.min(500, backoffMs * 2);
   }
   throw new DaemonClientError(
     'startupFailed',
-    `Timed out awaiting daemon readiness; startup reservation retained at ${getDaemonStartupFilePath(paths)}.`
+    `Timed out awaiting daemon readiness; the startup reservation at ${getDaemonStartupFilePath(paths)} ` +
+      `is kept while launcher PID ${child.pid} is alive and becomes stale when it exits.`
   );
 }
+
+export function describeExit(child: ChildProcess): string {
+  return child.signalCode ? `was terminated (${child.signalCode})` : `exited (${child.exitCode})`;
+}