diff --git a/apps/rush-cli-client/src/launchClient.ts b/apps/rush-cli-client/src/launchClient.ts index 6ae500af55..895c8dd08f 100644 --- a/apps/rush-cli-client/src/launchClient.ts +++ b/apps/rush-cli-client/src/launchClient.ts @@ -177,6 +177,7 @@ export async function launchClientAsync( writeStreamAsync(stream === 'stderr' ? process.stderr : process.stdout, bytes) }); let outcome: DaemonClientOutcome | undefined; + let restartFailure: DaemonClientError | undefined; const discoveryLines: string[] = []; const writeDiscoveryAsync = async (): Promise => { if (discoveryLines.length > 0) { @@ -226,8 +227,13 @@ export async function launchClientAsync( : undefined }); } catch (error) { + if (!(error instanceof DaemonClientError)) throw error; + if (!abort.signal.aborted) { + // A restart handoff fails only before the request executes, so in-process fallback cannot replay work. + if (error.code !== 'startupFailed') throw error; + restartFailure = error; + } // After cancellation, a transport failure (e.g. the cancellation deadline) still means "cancelled". - if (!abort.signal.aborted || !(error instanceof DaemonClientError)) throw error; outcome = undefined; } finally { for (const signal of CANCELLATION_SIGNALS) process.removeListener(signal, onSignal); @@ -237,6 +243,12 @@ export async function launchClientAsync( await client.closeAsync(); } } + if (restartFailure) { + agentRenderer?.dispose(); + process.stderr.write(`rush-client: ${restartFailure.message} Using in-process Rush.\n`); + launchInProcess(route.argv, rushx, selectedVersion); + return; + } if (outcome === undefined || isCancelledOutcome(outcome, abort.signal.aborted)) { const exitCode: number = getSignalExitCode(cancellationSignal ?? 'SIGINT'); agentRenderer?.finish({ exitCode, errorMessage: 'cancelled' }); diff --git a/common/changes/@rushstack/rush-cli-client/startup-wedge_2026-09-24-01-30.json b/common/changes/@rushstack/rush-cli-client/startup-wedge_2026-09-24-01-30.json new file mode 100644 index 0000000000..99df5b2bca --- /dev/null +++ b/common/changes/@rushstack/rush-cli-client/startup-wedge_2026-09-24-01-30.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-cli-client", + "comment": "Print the daemon's error message for failed results, and fall back to in-process Rush when a pre-execution daemon restart cannot start.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-cli-client", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-client-core/startup-wedge_2026-09-24-01-30.json b/common/changes/@rushstack/rush-client-core/startup-wedge_2026-09-24-01-30.json new file mode 100644 index 0000000000..bb3c446821 --- /dev/null +++ b/common/changes/@rushstack/rush-client-core/startup-wedge_2026-09-24-01-30.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-client-core", + "comment": "Make the daemon startup reservation bounded and verifiable: record owner and launcher PIDs, release it when the launcher exits before readiness, accept a ready daemon despite a reservation, reclaim stale reservations without waiting for the deadline, and include the launcher log's last error lines in startup failures.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-client-core", + "email": "selarkin@microsoft.com" +} diff --git a/common/changes/@rushstack/rush-daemon/startup-wedge_2026-09-24-01-30.json b/common/changes/@rushstack/rush-daemon/startup-wedge_2026-09-24-01-30.json new file mode 100644 index 0000000000..da5c34cf5e --- /dev/null +++ b/common/changes/@rushstack/rush-daemon/startup-wedge_2026-09-24-01-30.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-daemon", + "comment": "Validate a request's Rush environment before planning a process restart, so an invalid value fails the request with the native message and keeps the current daemon running.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-daemon", + "email": "selarkin@microsoft.com" +} diff --git a/libraries/rush-client-core/README.md b/libraries/rush-client-core/README.md index a91c95b331..4dced7838c 100644 --- a/libraries/rush-client-core/README.md +++ b/libraries/rush-client-core/README.md @@ -48,7 +48,7 @@ aware `LockFile` for the first-start mutex, including kernel-enforced exclusive file sharing on Windows. The winning client rechecks readiness, reclaims only an absent/dead owner, and reserves `.starting` before handing the explicit command to a detached startup helper. The helper spawns without -a shell and retains that reservation until the daemon completes hello/ping readiness, +a shell and holds that reservation until the daemon completes hello/ping readiness, independently of whether the requesting client survives. Clients still await hello/pong under bounded backoff. Stdout/stderr go to `.log`. No PID is killed. While holding the mutex with no startup reservation, stale leftovers are @@ -61,14 +61,34 @@ which removes the record, socket and reservation after the same no-listener/no-l The helper uses a stable tool cwd, and the starting client awaits its exit after readiness. The explicit launcher's cwd is unchanged. -An unresolved startup reservation is never automatically reclaimed based on PID -liveness or elapsed time. If the helper cannot establish readiness, subsequent starts -fail closed instead of risking a second detached daemon. Only a known spawn failure -(no executable started) releases the reservation immediately. An arbitrary launcher -can spawn descendants, so its exit is not proof that another launch is safe. -Recovery of an abandoned reservation requires operator confirmation that the original -startup cannot still publish an endpoint; normal successful startup releases it -automatically. Cancellation stops the client waiting, not the detached handoff. +The startup reservation is bounded and verifiable. It is a JSON record of its token, +creation time, startup timeout, the process responsible for releasing it (the starting +client until the helper is spawned, then the helper, with its start time) and the +launcher PID once spawned. The reservation is released when: + +- the daemon completes hello/ping readiness (by the helper); +- no executable could be started, or the launcher the helper started exits before + publishing an endpoint (by the helper). The client reports this immediately with the + last lines of `.log`, such as the launcher's own error; +- any client finds a ready endpoint whose published ownership record matches the + hello/ping status while the reservation's owner is gone. Such an endpoint is used even + while a reservation exists; a live helper releases its own reservation; +- a client holding the first-start mutex finds it stale: neither its owner nor its + launcher is alive, or it has outlived its own startup timeout by a fixed 60-second + grace period (this bounds PID reuse and a wedged launcher). Unrecognized records, such + as token-only reservations from older clients, have no verifiable owner and become + stale by file age. A stale reservation is reclaimed without waiting for the deadline. + +Every check-then-write of the reservation (owner/launcher updates, release, stale +reclaim, and cleanup after a dead owner) holds a short `-reservation` lock, +so a resumed stale owner can never overwrite or remove a replacement reservation. + +Otherwise, a live reservation makes later starts wait for readiness, then fail with the +reservation's owner/launcher state instead of launching a second daemon. A launcher that +misses the deadline but later binds is still accepted. If a stale reservation is +reclaimed while an old launcher is merely suspended, the transport's bind-time ownership +check still rejects whichever daemon binds second, so one workspace never has two +serving daemons. Cancellation stops the client waiting, not the detached handoff. If a wire-compatible daemon reports the wrong implementation version and an explicit replacement launcher is available, startup serializes replacement under that same mutex. @@ -129,6 +149,5 @@ graph reference client. A successful handshake is still transport readiness, not guarantee that every command or configuration is supported. Version-selected daemon installation and incompatible-protocol replacement remain separate integration work; this core package does not construct an engine. -The startup helper and durable pre-bind reservation protect concurrent first-invocations -even if the original client dies. They do not add a new daemon protocol or authorize -automatic recovery of ambiguous launcher failures. +The startup helper and bounded pre-bind reservation protect concurrent first-invocations +even if the original client dies. They do not add a new daemon protocol. diff --git a/libraries/rush-client-core/src/DaemonLogFile.ts b/libraries/rush-client-core/src/DaemonLogFile.ts index 42e4a1dbe7..e5a209cad8 100644 --- a/libraries/rush-client-core/src/DaemonLogFile.ts +++ b/libraries/rush-client-core/src/DaemonLogFile.ts @@ -1,9 +1,51 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as fs from 'node:fs'; + import type { IDaemonPaths } from '@rushstack/rush-daemon-transport'; +const MAX_LOG_TAIL_BYTES: number = 16384; +const MAX_LOG_TAIL_LINES: number = 3; +const MAX_LOG_LINE_LENGTH: number = 500; + /** The workspace launcher's persistent stdout/stderr log, independent of daemon lifetime. @beta */ export function getDaemonLogFilePath(paths: IDaemonPaths): string { return `${paths.lockfilePath}.log`; } + +/** Returns the log's current size, used to scope later diagnostics to one startup attempt. */ +export function getDaemonLogFileSize(paths: IDaemonPaths): number { + return fs.statSync(getDaemonLogFilePath(paths), { throwIfNoEntry: false })?.size ?? 0; +} + +/** + * Formats the last launcher log lines written after `fromOffset`, preferring error lines and omitting + * stack frames, so startup failures show their real cause. Returns an empty string if nothing is available. + */ +export function formatDaemonLogTail(paths: IDaemonPaths, fromOffset: number = 0): string { + let text: string; + try { + const fd: number = fs.openSync(getDaemonLogFilePath(paths), 'r'); + try { + const size: number = fs.fstatSync(fd).size; + const start: number = Math.max(fromOffset > size ? 0 : fromOffset, size - MAX_LOG_TAIL_BYTES); + const buffer: Buffer = Buffer.alloc(size - start); + fs.readSync(fd, buffer, 0, buffer.length, start); + text = buffer.toString('utf8'); + } finally { + fs.closeSync(fd); + } + } catch { + return ''; + } + const meaningful: string[] = text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith('at ')); + const errors: string[] = meaningful.filter((line) => /error/i.test(line)); + const lines: string[] = (errors.length > 0 ? errors : meaningful) + .slice(-MAX_LOG_TAIL_LINES) + .map((line) => (line.length > MAX_LOG_LINE_LENGTH ? `${line.slice(0, MAX_LOG_LINE_LENGTH)}...` : line)); + return lines.length > 0 ? `\nLast launcher log lines:\n ${lines.join('\n ')}\n` : ''; +} diff --git a/libraries/rush-client-core/src/DaemonStartup.ts b/libraries/rush-client-core/src/DaemonStartup.ts index 8fcd8e8477..a3573ef683 100644 --- a/libraries/rush-client-core/src/DaemonStartup.ts +++ b/libraries/rush-client-core/src/DaemonStartup.ts @@ -2,9 +2,7 @@ // See LICENSE in the project root for license information. import { spawn, type ChildProcess } from 'node:child_process'; -import { randomUUID } from 'node:crypto'; import { once } from 'node:events'; -import * as fs from 'node:fs'; import { setTimeout as delayAsync } from 'node:timers/promises'; import { @@ -16,6 +14,18 @@ import { import type { IDaemonStartCommand } from './connectOrStartDaemon'; import { DaemonClient } from './DaemonClient'; import { DaemonClientError } from './DaemonClientError'; +import { + assertDaemonStartupReservation, + getDaemonStartupFilePath, + releaseDaemonStartup, + updateDaemonStartupReservation +} from './DaemonStartupReservation'; + +export { + getDaemonStartupFilePath, + releaseDaemonStartup, + reserveDaemonStartup +} from './DaemonStartupReservation'; export interface IDaemonStartupOptions { readonly paths: IDaemonPaths; @@ -24,35 +34,16 @@ export interface IDaemonStartupOptions { readonly timeoutMs: number; } -export function getDaemonStartupFilePath(paths: IDaemonPaths): string { - return `${paths.lockfilePath}.starting`; -} - -export function reserveDaemonStartup(paths: IDaemonPaths): string { - const token: string = randomUUID(); - fs.writeFileSync(getDaemonStartupFilePath(paths), token, { flag: 'wx', mode: 0o600 }); - return token; -} - -function assertReservation(paths: IDaemonPaths, token: string): void { - if (fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8') !== token) { - throw new DaemonClientError('startupFailed', 'The daemon startup reservation changed ownership.'); - } -} - -export function releaseDaemonStartup(paths: IDaemonPaths, token: string): void { - assertReservation(paths, token); - fs.unlinkSync(getDaemonStartupFilePath(paths)); -} - /** - * Runs independently of the requesting client. Once spawn succeeds, only protocol readiness releases - * the reservation: an arbitrary launcher may outlive its parent or spawn descendants. - * Failure before readiness deliberately leaves a durable reservation instead of guessing that a PID is safe. + * Runs independently of the requesting client. The reservation records this helper and its launcher PID, + * so later starters can verify whether the startup can still make progress. Protocol readiness releases + * the reservation; so does a launcher that exits without publishing an endpoint. A deadline miss while the + * launcher is still alive keeps the reservation, which later becomes stale when both processes are gone or + * the bounded grace period elapses. */ export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Promise { const { paths, startCommand: start, token, timeoutMs } = options; - assertReservation(paths, token); + assertDaemonStartupReservation(paths, token); let child: ChildProcess; let closed: Promise | undefined; try { @@ -72,6 +63,7 @@ export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Pro throw error; } child.unref(); + if (child.pid !== undefined) updateDaemonStartupReservation(paths, token, { launcherPid: child.pid }); const deadline: number = Date.now() + timeoutMs; let backoffMs: number = 50; @@ -104,9 +96,12 @@ export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Pro } if (child.exitCode !== null || child.signalCode !== null) { await closed; + // The launcher this helper started is gone without publishing an endpoint. If it left a descendant + // that binds later, the transport's bind-time ownership check still rejects a second daemon. + releaseDaemonStartup(paths, token); throw new DaemonClientError( 'startupFailed', - `Launcher exited (${child.exitCode ?? child.signalCode}) before protocol readiness; startup reservation retained.` + `Daemon launcher ${describeExit(child)} before protocol readiness; startup reservation released.` ); } await delayAsync(Math.min(backoffMs, Math.max(1, deadline - Date.now()))); @@ -114,6 +109,11 @@ export async function runDaemonStartupAsync(options: IDaemonStartupOptions): Pro } throw new DaemonClientError( 'startupFailed', - `Timed out awaiting daemon readiness; startup reservation retained at ${getDaemonStartupFilePath(paths)}.` + `Timed out awaiting daemon readiness; the startup reservation at ${getDaemonStartupFilePath(paths)} ` + + `is kept while launcher PID ${child.pid} is alive and becomes stale when it exits.` ); } + +export function describeExit(child: ChildProcess): string { + return child.signalCode ? `was terminated (${child.signalCode})` : `exited (${child.exitCode})`; +} diff --git a/libraries/rush-client-core/src/DaemonStartupReservation.ts b/libraries/rush-client-core/src/DaemonStartupReservation.ts new file mode 100644 index 0000000000..a4e0903b1b --- /dev/null +++ b/libraries/rush-client-core/src/DaemonStartupReservation.ts @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { randomUUID } from 'node:crypto'; +import * as fs from 'node:fs'; + +import type { IDaemonPaths } from '@rushstack/rush-daemon-transport'; + +import { DaemonClientError } from './DaemonClientError'; +import { withStartupReservationLock } from './StartupLock'; + +/** + * How long a reservation may outlive its own startup deadline while a recorded process is still alive. + * This bounds PID reuse and a wedged launcher; the transport's bind-time ownership check still prevents + * two daemons from serving one endpoint if the original launcher later resumes. + */ +export const DAEMON_STARTUP_RESERVATION_GRACE_MS: number = 60000; + +/** The durable `.starting` record. */ +export interface IDaemonStartupReservation { + readonly token: string; + readonly createdAt: string; + readonly timeoutMs: number; + /** The process responsible for releasing the reservation: the starting client, then the detached helper. */ + readonly ownerPid: number; + readonly ownerStartedAt: string; + /** The explicit launcher (typically the daemon itself) once the helper has spawned it. */ + readonly launcherPid?: number; +} + +export function getDaemonStartupFilePath(paths: IDaemonPaths): string { + return `${paths.lockfilePath}.starting`; +} + +export function reserveDaemonStartup(paths: IDaemonPaths, timeoutMs: number): string { + const now: string = new Date().toISOString(); + const reservation: IDaemonStartupReservation = { + token: randomUUID(), + createdAt: now, + timeoutMs, + ownerPid: process.pid, + ownerStartedAt: new Date(Date.now() - process.uptime() * 1000).toISOString() + }; + fs.writeFileSync(getDaemonStartupFilePath(paths), JSON.stringify(reservation), { + flag: 'wx', + mode: 0o600 + }); + return reservation.token; +} + +/** Marks a reservation file that is present but is not a recognized record. */ +export const UNRECOGNIZED_DAEMON_STARTUP_RESERVATION: 'unrecognized' = 'unrecognized'; + +export type DaemonStartupReservationRecord = + | IDaemonStartupReservation + | typeof UNRECOGNIZED_DAEMON_STARTUP_RESERVATION + | undefined; + +/** Returns the parsed reservation, `undefined` if absent, or `UNRECOGNIZED_DAEMON_STARTUP_RESERVATION`. */ +export function readDaemonStartupReservation( + paths: IDaemonPaths +): DaemonStartupReservationRecord { + let text: string; + try { + text = fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8'); + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) return undefined; + throw error; + } + try { + const record: unknown = JSON.parse(text); + return isReservation(record) ? record : UNRECOGNIZED_DAEMON_STARTUP_RESERVATION; + } catch { + return UNRECOGNIZED_DAEMON_STARTUP_RESERVATION; + } +} + +function assertOwnedReservation(paths: IDaemonPaths, token: string): IDaemonStartupReservation { + const reservation: DaemonStartupReservationRecord = readDaemonStartupReservation(paths); + if (typeof reservation !== 'object' || reservation.token !== token) { + throw new DaemonClientError('startupFailed', 'The daemon startup reservation changed ownership.'); + } + return reservation; +} + +export function assertDaemonStartupReservation(paths: IDaemonPaths, token: string): void { + assertOwnedReservation(paths, token); +} + +/** + * Atomically records a new owner or launcher PID, but only while the token still owns the reservation. + * All reservation mutations hold the reservation lock, so a resumed stale owner can never overwrite or + * remove a replacement reservation between its ownership check and its write. + */ +export function updateDaemonStartupReservation( + paths: IDaemonPaths, + token: string, + update: Partial> +): void { + withStartupReservationLock(paths, () => replaceOwnedReservation(paths, token, update)); +} + +function replaceOwnedReservation( + paths: IDaemonPaths, + token: string, + update: Partial +): void { + const reservation: IDaemonStartupReservation = assertOwnedReservation(paths, token); + const filePath: string = getDaemonStartupFilePath(paths); + const temporaryPath: string = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + fs.writeFileSync(temporaryPath, JSON.stringify({ ...reservation, ...update }), { + flag: 'wx', + mode: 0o600 + }); + try { + fs.renameSync(temporaryPath, filePath); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + throw error; + } +} + +export function releaseDaemonStartup(paths: IDaemonPaths, token: string): void { + withStartupReservationLock(paths, () => { + assertOwnedReservation(paths, token); + fs.rmSync(getDaemonStartupFilePath(paths), { force: true }); + }); +} + +/** True if the process responsible for releasing the reservation is gone. */ +export function isDaemonStartupOwnerGone(reservation: IDaemonStartupReservation): boolean { + return !isProcessAlive(reservation.ownerPid); +} + +/** + * A reservation is stale when neither its owner nor its launcher is alive, or when it has outlived its own + * startup deadline by the grace period. Unrecognized (for example legacy token-only) records have no + * verifiable owner, so only their age counts. + */ +function isDaemonStartupReservationStale(paths: IDaemonPaths, timeoutMs: number): boolean { + const reservation: DaemonStartupReservationRecord = readDaemonStartupReservation(paths); + if (reservation === undefined) return false; + if (reservation === UNRECOGNIZED_DAEMON_STARTUP_RESERVATION) { + const stats: fs.Stats | undefined = fs.statSync(getDaemonStartupFilePath(paths), { + throwIfNoEntry: false + }); + return !!stats && Date.now() - stats.mtimeMs > timeoutMs + DAEMON_STARTUP_RESERVATION_GRACE_MS; + } + const expiresAt: number = + Date.parse(reservation.createdAt) + reservation.timeoutMs + DAEMON_STARTUP_RESERVATION_GRACE_MS; + const launcherGone: boolean = + reservation.launcherPid === undefined || !isProcessAlive(reservation.launcherPid); + return (isDaemonStartupOwnerGone(reservation) && launcherGone) || Date.now() > expiresAt; +} + +/** Removes the reservation if it is stale, atomically with respect to other reservation mutations. */ +export function reclaimStaleDaemonStartupReservation(paths: IDaemonPaths, timeoutMs: number): boolean { + return withStartupReservationLock(paths, () => { + if (!isDaemonStartupReservationStale(paths, timeoutMs)) return false; + fs.rmSync(getDaemonStartupFilePath(paths), { force: true }); + return true; + }); +} + +/** + * Removes the reservation file only if it still matches the observed record, so a concurrent new + * reservation is never removed. + */ +export function removeDaemonStartupReservation( + paths: IDaemonPaths, + observed: DaemonStartupReservationRecord +): void { + withStartupReservationLock(paths, () => removeMatchingReservation(paths, observed)); +} + +function removeMatchingReservation(paths: IDaemonPaths, observed: DaemonStartupReservationRecord): void { + const current: DaemonStartupReservationRecord = readDaemonStartupReservation(paths); + if (current === undefined || observed === undefined) return; + const matches: boolean = + typeof observed === 'object' + ? typeof current === 'object' && current.token === observed.token + : current === observed; + if (!matches) return; + fs.rmSync(getDaemonStartupFilePath(paths), { force: true }); +} + +export function describeDaemonStartupReservation(paths: IDaemonPaths): string { + const reservation: DaemonStartupReservationRecord = readDaemonStartupReservation(paths); + if (typeof reservation !== 'object') return 'unrecognized reservation record'; + const ageSeconds: number = Math.max(0, Math.round((Date.now() - Date.parse(reservation.createdAt)) / 1000)); + const launcher: string = + reservation.launcherPid === undefined + ? 'not spawned' + : `PID ${reservation.launcherPid} ${isProcessAlive(reservation.launcherPid) ? 'alive' : 'exited'}`; + return ( + `owner PID ${reservation.ownerPid} ${isProcessAlive(reservation.ownerPid) ? 'alive' : 'exited'}, ` + + `launcher ${launcher}, age ${ageSeconds}s` + ); +} + +function isReservation(record: unknown): record is IDaemonStartupReservation { + if (typeof record !== 'object' || record === null) return false; + const value: Partial> = record as Partial< + Record + >; + return ( + typeof value.token === 'string' && + typeof value.createdAt === 'string' && + Number.isFinite(Date.parse(value.createdAt)) && + typeof value.timeoutMs === 'number' && + Number.isFinite(value.timeoutMs) && + isPid(value.ownerPid) && + typeof value.ownerStartedAt === 'string' && + (value.launcherPid === undefined || isPid(value.launcherPid)) + ); +} + +function isPid(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM means the PID exists but belongs to another user. + return !hasErrorCode(error, 'ESRCH'); + } +} + +function hasErrorCode(error: unknown, code: string): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === code; +} diff --git a/libraries/rush-client-core/src/StartupLock.ts b/libraries/rush-client-core/src/StartupLock.ts index 32f0d7152d..6937b8bf1a 100644 --- a/libraries/rush-client-core/src/StartupLock.ts +++ b/libraries/rush-client-core/src/StartupLock.ts @@ -6,6 +6,8 @@ import * as path from 'node:path'; import { LockFile } from '@rushstack/node-core-library'; import type { IDaemonPaths } from '@rushstack/rush-daemon-transport'; +import { DaemonClientError } from './DaemonClientError'; + export interface IStartupLock { releaseAsync(): Promise; } @@ -18,3 +20,34 @@ export async function tryAcquireStartupLockAsync(paths: IDaemonPaths): Promise lock.release() } : undefined; } + +const RESERVATION_LOCK_TIMEOUT_MS: number = 5000; +const RESERVATION_LOCK_RETRY_MS: number = 10; + +/** + * Runs a short, synchronous read-check-write of the startup reservation while holding a dedicated lock. + * The start lock cannot be used because a waiting client holds it for the whole startup, while the helper + * must still be able to update or release its own reservation. + */ +export function withStartupReservationLock(paths: IDaemonPaths, action: () => T): T { + const folder: string = path.dirname(paths.lockfilePath); + const name: string = `${path.basename(paths.lockfilePath)}-reservation`; + const deadline: number = Date.now() + RESERVATION_LOCK_TIMEOUT_MS; + const sleeper: Int32Array = new Int32Array(new SharedArrayBuffer(4)); + let lock: LockFile | undefined = LockFile.tryAcquire(folder, name); + while (!lock) { + if (Date.now() >= deadline) { + throw new DaemonClientError( + 'startupFailed', + `Timed out waiting for the daemon startup reservation lock in ${folder}.` + ); + } + Atomics.wait(sleeper, 0, 0, RESERVATION_LOCK_RETRY_MS); + lock = LockFile.tryAcquire(folder, name); + } + try { + return action(); + } finally { + lock.release(); + } +} diff --git a/libraries/rush-client-core/src/connectOrStartDaemon.ts b/libraries/rush-client-core/src/connectOrStartDaemon.ts index 83cc47ddee..f4848ab167 100644 --- a/libraries/rush-client-core/src/connectOrStartDaemon.ts +++ b/libraries/rush-client-core/src/connectOrStartDaemon.ts @@ -20,7 +20,8 @@ import { import { DaemonClient, type IDaemonClientConnectOptions } from './DaemonClient'; import { DaemonClientError } from './DaemonClientError'; -import { getDaemonLogFilePath } from './DaemonLogFile'; +import { formatDaemonLogTail, getDaemonLogFilePath } from './DaemonLogFile'; +import { describeExit, type IDaemonStartupOptions } from './DaemonStartup'; import { DAEMON_RESET_HINT, hasErrorCode, @@ -31,16 +32,24 @@ import { type DaemonOwnership } from './DaemonOwnership'; import { + describeDaemonStartupReservation, getDaemonStartupFilePath, - reserveDaemonStartup, + isDaemonStartupOwnerGone, + readDaemonStartupReservation, + reclaimStaleDaemonStartupReservation, releaseDaemonStartup, - type IDaemonStartupOptions -} from './DaemonStartup'; + removeDaemonStartupReservation, + reserveDaemonStartup, + updateDaemonStartupReservation, + type DaemonStartupReservationRecord +} from './DaemonStartupReservation'; import { tryAcquireStartupLockAsync, type IStartupLock } from './StartupLock'; interface IStartupHelper { readonly child: ChildProcess; readonly closed: Promise; + /** Launcher log size before this startup, so failures report only this attempt's output. */ + readonly logOffset: number; } /** A version-selected launch command supplied by the embedding application, never guessed by the core. @beta */ @@ -107,12 +116,17 @@ export async function connectOrStartDaemonAsync( `No ready daemon at ${options.paths.socketPath}; auto-start is disabled.` ); } - return await startDaemonAsync({ ...options, startCommand, resolveStartCommandAsync: undefined }, deadline); + return await startDaemonAsync( + { ...options, startCommand, resolveStartCommandAsync: undefined }, + deadline, + timeoutMs + ); } async function startDaemonAsync( options: IConnectOrStartDaemonOptions & { readonly startCommand: IDaemonStartCommand }, - deadline: number + deadline: number, + timeoutMs: number ): Promise { ensureDaemonRuntimeDir(options.paths); let lock: IStartupLock | undefined; @@ -133,10 +147,15 @@ async function startDaemonAsync( while (fs.lstatSync(getDaemonStartupFilePath(options.paths), { throwIfNoEntry: false })) { const ready: DaemonClient | undefined = await tryConnectAsync(options, deadline); if (ready) return ready; + // Holding the start lock, a verifiably abandoned reservation can be reclaimed without waiting. + if (reclaimStaleDaemonStartupReservation(options.paths, timeoutMs)) continue; if (Date.now() >= deadline) { throw startupError( options, - `has an unresolved startup handoff at ${getDaemonStartupFilePath(options.paths)}; refusing another launch. ${DAEMON_RESET_HINT}` + `has an unresolved startup handoff at ${getDaemonStartupFilePath(options.paths)} ` + + `(${describeDaemonStartupReservation(options.paths)}); refusing another launch until it ` + + `becomes ready or stale. ${DAEMON_RESET_HINT}`, + formatDaemonLogTail(options.paths) ); } await delayAsync(Math.min(100, Math.max(1, deadline - Date.now())), undefined, { @@ -174,7 +193,8 @@ async function startDaemonAsync( await waitForHelperExitAsync(helper, options, deadline); throw startupError( options, - `failed: Unable to start ${options.startCommand.command}; helper exited (${child.exitCode ?? child.signalCode}) before readiness` + `failed: Unable to start ${options.startCommand.command}; startup helper ${describeExit(child)} before readiness`, + formatDaemonLogTail(options.paths, helper.logOffset) ); } await delayAsync(Math.min(backoffMs, Math.max(1, deadline - Date.now())), undefined, { @@ -182,7 +202,11 @@ async function startDaemonAsync( }); backoffMs = Math.min(500, backoffMs * 2); } - throw startupError(options, 'timed out awaiting hello/ping readiness'); + throw startupError( + options, + 'timed out awaiting hello/ping readiness', + formatDaemonLogTail(options.paths, helper.logOffset) + ); } finally { await lock.releaseAsync(); } @@ -256,15 +280,16 @@ async function tryConnectAsync( socketPath: options.paths.socketPath, timeoutMs: Math.min(options.timeoutMs ?? 1000, Math.max(1, deadline - Date.now())) }); - // Do not expose a just-started daemon to shutdown/restart until the helper finishes the handoff. - let pendingStartup: boolean = true; + // A reservation normally means the helper has not finished the handoff. An endpoint that answers + // hello/ping and matches the published ownership record is nevertheless ready, so accept it. + let accepted: boolean = false; try { options.abortSignal?.throwIfAborted(); - pendingStartup = !!fs.lstatSync(getDaemonStartupFilePath(options.paths), { throwIfNoEntry: false }); + accepted = await isReadyDespiteReservationAsync(client, options.paths); } finally { - if (pendingStartup) await client.closeAsync(); + if (!accepted) await client.closeAsync(); } - return pendingStartup ? undefined : client; + return accepted ? client : undefined; } catch (error) { options.abortSignal?.throwIfAborted(); if ( @@ -292,6 +317,19 @@ async function tryConnectAsync( } } +async function isReadyDespiteReservationAsync(client: DaemonClient, paths: IDaemonPaths): Promise { + const reservation: DaemonStartupReservationRecord = readDaemonStartupReservation(paths); + if (reservation === undefined) return true; + const { pid } = await client.status; + const owner: IDaemonLockfile | undefined = readDaemonLockfile(paths.lockfilePath); + if (!isDaemonOwnership(owner) || owner.pid !== pid || owner.socketPath !== paths.socketPath) return false; + // A live helper releases its own reservation; clean up only after an owner that can no longer do so. + if (typeof reservation !== 'object' || isDaemonStartupOwnerGone(reservation)) { + removeDaemonStartupReservation(paths, reservation); + } + return true; +} + async function waitForHandoffAsync( options: IConnectOrStartDaemonOptions, deadline: number @@ -420,7 +458,8 @@ async function spawnDetachedAsync( } fs.fchmodSync(logFd, 0o600); } - const token: string = reserveDaemonStartup(options.paths); + const logOffset: number = stats.size; + const token: string = reserveDaemonStartup(options.paths, Math.max(1, deadline - Date.now())); let helper: IStartupHelper | undefined; try { const child: ChildProcess = spawn(process.execPath, [path.join(__dirname, 'runDaemonStartup.js')], { @@ -431,7 +470,8 @@ async function spawnDetachedAsync( }); helper = { child, - closed: new Promise((resolve) => child.once('close', () => resolve())) + closed: new Promise((resolve) => child.once('close', () => resolve())), + logOffset }; await once(child, 'spawn'); } catch (error) { @@ -453,6 +493,11 @@ async function spawnDetachedAsync( }; let delivered: boolean = false; try { + // Record the helper before handing off, so the reservation stays live exactly as long as the helper. + updateDaemonStartupReservation(options.paths, token, { + ownerPid: child.pid!, + ownerStartedAt: new Date().toISOString() + }); await new Promise((resolve, reject) => { child.send(startup, (error) => (error ? reject(error) : resolve())); }); @@ -492,9 +537,13 @@ async function waitForHelperExitAsync( } } -function startupError(options: IConnectOrStartDaemonOptions, reason: string): DaemonClientError { +function startupError( + options: IConnectOrStartDaemonOptions, + reason: string, + logTail: string = '' +): DaemonClientError { return new DaemonClientError( 'startupFailed', - `Daemon startup ${reason}. Inspect ${getDaemonLogFilePath(options.paths)} and retry, or use --no-daemon.` + `Daemon startup ${reason}. Inspect ${getDaemonLogFilePath(options.paths)} and retry, or use --no-daemon.${logTail}` ); } diff --git a/libraries/rush-client-core/src/test/DaemonStartupReservation.test.ts b/libraries/rush-client-core/src/test/DaemonStartupReservation.test.ts new file mode 100644 index 0000000000..2662680f0d --- /dev/null +++ b/libraries/rush-client-core/src/test/DaemonStartupReservation.test.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import { LockFile } from '@rushstack/node-core-library'; +import type { IDaemonPaths } from '@rushstack/rush-daemon-transport'; + +import { + getDaemonStartupFilePath, + reclaimStaleDaemonStartupReservation, + releaseDaemonStartup, + reserveDaemonStartup, + updateDaemonStartupReservation +} from '../DaemonStartupReservation'; + +describe('daemon startup reservation mutations', () => { + let folder: string; + let paths: IDaemonPaths; + + beforeEach(() => { + folder = fs.mkdtempSync(path.join(os.tmpdir(), 'rush-reservation-')); + paths = { + runtimeDir: folder, + socketPath: path.join(folder, 'd.sock'), + lockfilePath: path.join(folder, 'daemon.pid.json') + }; + }); + + afterEach(() => { + fs.rmSync(folder, { recursive: true, force: true }); + }); + + it('never lets a stale owner update or release a replacement reservation', () => { + const staleToken: string = reserveDaemonStartup(paths, 0); + fs.rmSync(getDaemonStartupFilePath(paths)); + reserveDaemonStartup(paths, 1000); + const replacement: string = fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8'); + expect(() => updateDaemonStartupReservation(paths, staleToken, { launcherPid: 1 })).toThrow( + 'changed ownership' + ); + expect(() => releaseDaemonStartup(paths, staleToken)).toThrow('changed ownership'); + expect(fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8')).toBe(replacement); + }); + + it('serializes reclaim with other reservation mutations', () => { + reserveDaemonStartup(paths, 1000); + const lock: LockFile | undefined = LockFile.tryAcquire(folder, 'daemon.pid.json-reservation'); + expect(lock).toBeDefined(); + const started: number = Date.now(); + try { + expect(() => reclaimStaleDaemonStartupReservation(paths, 1000)).toThrow('reservation lock'); + } finally { + lock?.release(); + } + expect(Date.now() - started).toBeGreaterThanOrEqual(4000); + expect(reclaimStaleDaemonStartupReservation(paths, 1000)).toBe(false); + expect(fs.existsSync(getDaemonStartupFilePath(paths))).toBe(true); + }, 15000); +}); diff --git a/libraries/rush-client-core/src/test/connectOrStartDaemon.test.ts b/libraries/rush-client-core/src/test/connectOrStartDaemon.test.ts index e49f287909..633bb9b87f 100644 --- a/libraries/rush-client-core/src/test/connectOrStartDaemon.test.ts +++ b/libraries/rush-client-core/src/test/connectOrStartDaemon.test.ts @@ -163,6 +163,11 @@ describe('detached daemon startup', () => { }); expect((await client.status).pid).toBe(daemonPid); await client.closeAsync(); + // Successors accept the ready daemon at once; the live helper then releases its own reservation. + const releaseDeadline: number = Date.now() + 5000; + while (fs.existsSync(getDaemonStartupFilePath(paths)) && Date.now() < releaseDeadline) { + await delayAsync(20); + } expect(fs.existsSync(getDaemonStartupFilePath(paths))).toBe(false); }, 15000); @@ -180,21 +185,96 @@ describe('detached daemon startup', () => { expect(fs.existsSync(path.join(folder, 'starts'))).toBe(false); }); - it('does not infer safe retry from a launcher exiting before ownership publication', async () => { + it('releases the reservation and reports the launcher error when it exits before readiness', async () => { const startupPath: string = getDaemonStartupFilePath(paths); const failing: IConnectOrStartDaemonOptions = { ...options, startCommand: { ...options.startCommand!, args: [path.join(folder, 'missing-entry.js')] } }; - await expect(connectOrStartDaemonAsync(failing)).rejects.toThrow('Unable to start'); - const contents: string = fs.readFileSync(startupPath, 'utf8'); - await expect(connectOrStartDaemonAsync({ ...options, startupTimeoutMs: 100 })).rejects.toThrow( + const started: number = Date.now(); + await expect(connectOrStartDaemonAsync(failing)).rejects.toThrow( + /helper exited \(1\) before readiness[\s\S]*Last launcher log lines:[\s\S]*startup reservation released/ + ); + expect(Date.now() - started).toBeLessThan(options.startupTimeoutMs!); + expect(fs.existsSync(startupPath)).toBe(false); + expect(fs.existsSync(path.join(folder, 'starts'))).toBe(false); + const client = await connectOrStartDaemonAsync(options); + await client.closeAsync(); + expect(fs.readFileSync(path.join(folder, 'starts'), 'utf8').trim().split('\n')).toHaveLength(1); + }); + + async function getExitedPidAsync(): Promise { + const exited: ChildProcess = spawn(process.execPath, ['-e', ''], { stdio: 'ignore' }); + await once(exited, 'exit'); + return exited.pid!; + } + + function writeReservation(ownerPid: number, launcherPid?: number): string { + const contents: string = JSON.stringify({ + token: 'fixture', + createdAt: new Date().toISOString(), + timeoutMs: 1000, + ownerPid, + ownerStartedAt: new Date().toISOString(), + launcherPid + }); + fs.writeFileSync(getDaemonStartupFilePath(paths), contents); + return contents; + } + + it('reclaims a stale reservation whose owner and launcher are dead and starts at once', async () => { + writeReservation(await getExitedPidAsync(), await getExitedPidAsync()); + const started: number = Date.now(); + const client = await connectOrStartDaemonAsync(options); + await client.closeAsync(); + expect(Date.now() - started).toBeLessThan(options.startupTimeoutMs!); + expect(fs.existsSync(getDaemonStartupFilePath(paths))).toBe(false); + expect(fs.readFileSync(path.join(folder, 'starts'), 'utf8').trim().split('\n')).toHaveLength(1); + }); + + it('reclaims an unrecognized reservation only after the bounded age', async () => { + const startupPath: string = getDaemonStartupFilePath(paths); + fs.writeFileSync(startupPath, 'legacy-token'); + await expect(connectOrStartDaemonAsync({ ...options, startupTimeoutMs: 200 })).rejects.toThrow( 'unresolved startup handoff' ); - expect(fs.readFileSync(startupPath, 'utf8')).toBe(contents); + const old: Date = new Date(Date.now() - 10 * 60 * 1000); + fs.utimesSync(startupPath, old, old); + const client = await connectOrStartDaemonAsync(options); + await client.closeAsync(); + expect(fs.existsSync(startupPath)).toBe(false); + }); + + it('keeps waiting on a reservation whose launcher is still alive', async () => { + const contents: string = writeReservation(await getExitedPidAsync(), process.pid); + await expect(connectOrStartDaemonAsync({ ...options, startupTimeoutMs: 200 })).rejects.toThrow( + /unresolved startup handoff .*launcher PID \d+ alive/ + ); + expect(fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8')).toBe(contents); expect(fs.existsSync(path.join(folder, 'starts'))).toBe(false); }); + it.each([ + { behavior: 'cleans up after a dead owner', liveOwner: false }, + { behavior: 'leaves a live owner to release', liveOwner: true } + ])('accepts a ready daemon despite a reservation and $behavior', async ({ liveOwner }) => { + const first = await connectOrStartDaemonAsync(options); + const { pid } = await first.status; + await first.closeAsync(); + const contents: string = writeReservation(liveOwner ? process.pid : await getExitedPidAsync()); + const started: number = Date.now(); + const client = await connectOrStartDaemonAsync({ ...options, startCommand: undefined }); + expect((await client.status).pid).toBe(pid); + await client.closeAsync(); + expect(Date.now() - started).toBeLessThan(options.startupTimeoutMs!); + if (liveOwner) { + expect(fs.readFileSync(getDaemonStartupFilePath(paths), 'utf8')).toBe(contents); + } else { + expect(fs.existsSync(getDaemonStartupFilePath(paths))).toBe(false); + } + expect(fs.readFileSync(path.join(folder, 'starts'), 'utf8').trim().split('\n')).toHaveLength(1); + }); + it.each([false, true])( 'preserves an explicit launcher and environment (relative cwd: %s)', async (relative) => { diff --git a/libraries/rush-daemon/src/RushEnvironmentValidation.ts b/libraries/rush-daemon/src/RushEnvironmentValidation.ts new file mode 100644 index 0000000000..73197a1e9f --- /dev/null +++ b/libraries/rush-daemon/src/RushEnvironmentValidation.ts @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + EnvironmentConfiguration, + EnvironmentVariableNames, + resolveDaemonConfiguration +} from '@microsoft/rush-lib'; + +const BOOLEAN_VARIABLES: ReadonlySet = new Set([ + EnvironmentVariableNames.RUSH_ABSOLUTE_SYMLINKS, + EnvironmentVariableNames.RUSH_ALLOW_WARNINGS_IN_SUCCESSFUL_BUILD, + EnvironmentVariableNames.RUSH_BUILD_CACHE_ENABLED, + EnvironmentVariableNames.RUSH_BUILD_CACHE_WRITE_ALLOWED, + EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED +]); + +// These also accept the legacy "true"/"false" spellings. +const LEGACY_BOOLEAN_VARIABLES: ReadonlySet = new Set([ + EnvironmentVariableNames.RUSH_ALLOW_UNSUPPORTED_NODEJS, + EnvironmentVariableNames.RUSH_QUIET_MODE +]); + +const KNOWN_VARIABLES: ReadonlySet = new Set(Object.values(EnvironmentVariableNames)); + +/** + * Applies the same checks as `EnvironmentConfiguration.validate()` to a request-scoped environment, + * without touching this process's environment or the global configuration state. + * + * @remarks + * A request whose environment differs from the daemon's plans a process restart, and the successor + * validates that environment during startup. Validating it first lets the request fail with the native + * message while the current daemon stays available, instead of replacing a healthy daemon with a + * successor that can never start. + * + * @throws An error with the same message as native Rush for the first invalid value. + */ +export function validateRequestRushEnvironment(environment: Readonly>): void { + // Native validation resolves the RUSH_DAEMON_* settings first. + resolveDaemonConfiguration({}, environment); + const unknown: string[] = []; + const present: Set = new Set(); + for (const [name, value] of Object.entries(environment)) { + if (!/^RUSH_/i.test(name)) continue; + // Environment variable names are only case-insensitive on Windows. + const normalizedName: string = process.platform === 'win32' ? name.toUpperCase() : name; + if (!KNOWN_VARIABLES.has(normalizedName)) { + unknown.push(name); + continue; + } + if (value) present.add(normalizedName); + if (BOOLEAN_VARIABLES.has(normalizedName) || LEGACY_BOOLEAN_VARIABLES.has(normalizedName)) { + if (LEGACY_BOOLEAN_VARIABLES.has(normalizedName) && (value === 'true' || value === 'false')) continue; + EnvironmentConfiguration.parseBooleanEnvironmentVariable(normalizedName, value); + } + } + if (unknown.length > 0) { + throw new Error( + 'The following environment variables were found with the "RUSH_" prefix, but they are not ' + + `recognized by this version of Rush: ${unknown.join(', ')}` + ); + } + if ( + present.has(EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH) && + present.has(EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON) + ) { + throw new Error( + `Environment variable ${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH} and ` + + `${EnvironmentVariableNames.RUSH_BUILD_CACHE_OVERRIDE_JSON} are mutually exclusive. ` + + `Only one may be specified.` + ); + } +} \ No newline at end of file diff --git a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts index 2ee815aad2..3c52ba43b3 100644 --- a/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts +++ b/libraries/rush-daemon/src/WorkspaceRequestLifecycle.ts @@ -41,6 +41,7 @@ import { } from './WorkspaceRequestAdmission'; import { WorkspaceEngineRecreationRequiredError } from './WorkspaceEngineComponentFactory'; import { getDaemonShutdownReason } from './DaemonShutdownError'; +import { validateRequestRushEnvironment } from './RushEnvironmentValidation'; import type { IWorkspaceSession } from './WorkspaceSession'; import type { WorkspaceSessionProvider } from './WorkspaceSessionProvider'; import { assertWorkspaceRequestResourcesHealthy } from './WorkspaceRequestResources'; @@ -101,6 +102,17 @@ class RestartPendingBeforeExecution extends Error { } } +/** The request's Rush environment would prevent a successor from starting; the current daemon is kept. */ +class InvalidRequestEnvironment extends Error {} + +function assertValidRequestEnvironment(envelope: IDaemonRequestEnvelope): void { + try { + validateRequestRushEnvironment(envelope.environment); + } catch (error) { + throw new InvalidRequestEnvironment(error instanceof Error ? error.message : String(error)); + } +} + /** * Generation admission composes the existing request schedulers, native locks and session provider. * It never releases a resolved request onto a different session, and never replays scheduled work. @@ -261,6 +273,11 @@ export class WorkspaceRequestLifecycle implements IDaemonRequestLifecycle { }); return; } + if (error instanceof InvalidRequestEnvironment && !state.began && !state.terminalAttempted) { + await client.interactiveSession.finishAsync(); + await client.writeResultAsync(preExecutionFailure(envelope.requestId, error)); + return; + } if (error instanceof RequestSchedulerError && !state.began && !state.terminalAttempted) { await client.interactiveSession.finishAsync(); await client.writeResultAsync({ @@ -387,6 +404,7 @@ export class WorkspaceRequestLifecycle implements IDaemonRequestLifecycle { } let fingerprint: IWorkspaceInputFingerprint = await this.#captureAsync(session, envelope); let tier: WorkspaceInputChangeTier = this.#classify(fingerprint, isMutation(envelope)); + if (tier === WorkspaceInputChangeTier.Restart) assertValidRequestEnvironment(envelope); let commandIdentity: string | undefined; let projectFingerprint: string | undefined; if (tier !== WorkspaceInputChangeTier.Restart && !isMutation(envelope)) { @@ -445,6 +463,7 @@ export class WorkspaceRequestLifecycle implements IDaemonRequestLifecycle { fingerprint = await this.#captureAsync(session, envelope); tier = this.#classify(fingerprint, isMutation(envelope)); if (tier === WorkspaceInputChangeTier.Restart) { + assertValidRequestEnvironment(envelope); await this.#quiesceWarmSetAsync(session); const workspaceLease: IRequestLease = await admission.acquireAsync( getWorkspaceRequestScheduler(session), diff --git a/libraries/rush-daemon/src/test/RushEnvironmentValidation.test.ts b/libraries/rush-daemon/src/test/RushEnvironmentValidation.test.ts new file mode 100644 index 0000000000..6ce7300ad4 --- /dev/null +++ b/libraries/rush-daemon/src/test/RushEnvironmentValidation.test.ts @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { validateRequestRushEnvironment } from '../RushEnvironmentValidation'; + +describe(validateRequestRushEnvironment.name, () => { + it('accepts valid and unrelated values', () => { + expect(() => + validateRequestRushEnvironment({ + PATH: '/usr/bin', + RUSH_ALLOW_WARNINGS_IN_SUCCESSFUL_BUILD: '1', + RUSH_BUILD_CACHE_ENABLED: '0', + RUSH_QUIET_MODE: 'true', + RUSH_ALLOW_UNSUPPORTED_NODEJS: 'false', + RUSH_ABSOLUTE_SYMLINKS: '', + RUSH_PARALLELISM: 'max' + }) + ).not.toThrow(); + }); + + it('rejects an invalid boolean with the native message', () => { + expect(() => + validateRequestRushEnvironment({ RUSH_ALLOW_WARNINGS_IN_SUCCESSFUL_BUILD: 'yes' }) + ).toThrow( + 'Invalid value "yes" for the environment variable RUSH_ALLOW_WARNINGS_IN_SUCCESSFUL_BUILD. Valid choices are 0 or 1.' + ); + }); + + it('rejects unknown RUSH_ variables', () => { + expect(() => validateRequestRushEnvironment({ RUSH_NOT_A_SETTING: '1' })).toThrow( + 'not recognized by this version of Rush: RUSH_NOT_A_SETTING' + ); + }); + + it('rejects invalid daemon settings before a restart is planned', () => { + expect(() => validateRequestRushEnvironment({ RUSH_DAEMON_AUTO_START: 'yes' })).toThrow( + 'RUSH_DAEMON_AUTO_START must be 0 or 1.' + ); + expect(() => validateRequestRushEnvironment({ RUSH_DAEMON_AUTO_START: '1' })).not.toThrow(); + }); + + it('rejects mutually exclusive build cache overrides', () => { + expect(() => + validateRequestRushEnvironment({ + RUSH_BUILD_CACHE_OVERRIDE_JSON: '{}', + RUSH_BUILD_CACHE_OVERRIDE_JSON_FILE_PATH: 'cache.json' + }) + ).toThrow('are mutually exclusive'); + }); +}); \ No newline at end of file diff --git a/libraries/rush-daemon/src/test/WorkspaceReloadTierStatus.test.ts b/libraries/rush-daemon/src/test/WorkspaceReloadTierStatus.test.ts index afc7bc6a13..26fc5f204b 100644 --- a/libraries/rush-daemon/src/test/WorkspaceReloadTierStatus.test.ts +++ b/libraries/rush-daemon/src/test/WorkspaceReloadTierStatus.test.ts @@ -138,3 +138,32 @@ it('retains the requested restart tier on the old host while a real successor st } } }); + +it('fails an invalid request-scoped Rush environment before planning a restart and stays usable', async () => { + const getSuccessorLaunchAsync = jest.fn(getInstalledWorkspaceSuccessorLaunchAsync); + const fixture = await DaemonGraphTestFixture.createAsync((created) => { + setDaemonPolicy(created, {}); + created.getSuccessorLaunchAsync = getSuccessorLaunchAsync; + }); + try { + expect((await fixture.buildAsync()).terminal).toMatchObject({ payload: { exitCode: 0 } }); + const before = await pongAsync(fixture); + const result = await fixture.runAsync(['build', '--to', 'b', '--parallelism', '3'], { + environment: { ...fixture.environment, RUSH_ALLOW_WARNINGS_IN_SUCCESSFUL_BUILD: 'yes' } + }); + expect(result.terminal).toMatchObject({ + kind: 'requestResult', + payload: { + exitCode: 1, + errorMessage: + 'Invalid value "yes" for the environment variable RUSH_ALLOW_WARNINGS_IN_SUCCESSFUL_BUILD. Valid choices are 0 or 1.' + } + }); + expect(result.terminal).not.toHaveProperty('payload.retryAfterRestart'); + expect(getSuccessorLaunchAsync).not.toHaveBeenCalled(); + expect((await fixture.buildAsync()).terminal).toMatchObject({ payload: { exitCode: 0 } }); + expect((await pongAsync(fixture)).pid).toBe(before.pid); + } finally { + await fixture[Symbol.asyncDispose](); + } +}); \ No newline at end of file