From 203aebbae2b4958e6fb594c8b22699b21915f218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 21:39:51 +0200 Subject: [PATCH 01/22] fix(daemon): wait for a slow daemon startup and fail with typed reasons A client killed a healthy daemon that was still starting: the local startup used one 15 s wall-clock deadline, and a timeout never retried. On an overloaded host that is a failed run before any work starts (#3782, failure class #3342). - startLocalDaemon now waits while the spawned daemon stays alive, up to a 120 s ceiling; AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS overrides it for the CLI, MCP, and Node client alike. - The daemon records startup phase markers (modules-loaded, lock-acquired, listening, registered) with timestamps in daemon-startup-phases.jsonl beside its registration. A live daemon that publishes no marker for 60 s is retired as stalled; one that keeps progressing is waited for to the ceiling. - Failure details carry reason (startup-exited, startup-stalled, startup-ceiling), the observed phase timings, and the effective startupTimeoutMs. Each reason has its own hint. - help commands documents the variable and the three reasons. --- .../src/daemon-registration-owner.ts | 79 +++++ .../daemon/src/server/http/daemon-runtime.ts | 10 + src/commands/schema/cli-help.ts | 5 + .../daemon-client-startup-budget.test.ts | 324 ++++++++++++++++++ .../daemon-client-startup-race.test.ts | 7 + src/daemon-client/daemon-client-lifecycle.ts | 211 ++++++++++-- src/daemon.ts | 18 +- website/docs/docs/configuration.md | 2 +- 8 files changed, 630 insertions(+), 26 deletions(-) create mode 100644 src/daemon-client/__tests__/daemon-client-startup-budget.test.ts diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 5e7090da51..e58dab8dad 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -46,6 +46,85 @@ import { } from './daemon-shutdown-report.ts'; export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); + +/** + * Startup phase markers (#3782): the record a daemon leaves beside its registration of how far its + * startup got. The client waiting for registration reads this record to tell "slow but progressing" + * from "stuck". Each marker is one JSONL line appended to a dedicated state-directory file, so no + * `daemon.log` reader has to learn a new line shape and `daemon.log` keeps its existing ownership. + * The record is append-only across startups: a waiting client scopes it to its own launch by the + * marker timestamps, never by deleting what an earlier startup left. + */ +const DAEMON_STARTUP_PHASES = Object.freeze([ + 'modules-loaded', + 'lock-acquired', + 'listening', + 'registered', +] as const); + +export type DaemonStartupPhase = (typeof DAEMON_STARTUP_PHASES)[number]; + +const DAEMON_STARTUP_PHASE_SET: ReadonlySet = new Set(DAEMON_STARTUP_PHASES); + +const DAEMON_STARTUP_PHASES_FILE = 'daemon-startup-phases.jsonl'; + +type DaemonStartupMarkerRecord = { + data?: { startupPhase?: unknown; timestampMs?: unknown }; +}; + +export function resolveDaemonStartupPhasesPath(stateDir: string): string { + return path.join(stateDir, DAEMON_STARTUP_PHASES_FILE); +} + +export function recordDaemonStartupMarker( + startupPhasesPath: string, + phase: DaemonStartupPhase, +): void { + try { + const timestampMs = Date.now(); + fs.mkdirSync(path.dirname(startupPhasesPath), { recursive: true }); + fs.appendFileSync( + startupPhasesPath, + `${JSON.stringify({ data: { startupPhase: phase, timestampMs } })}\n`, + ); + } catch { + // A marker is progress evidence, never a startup requirement: a host that refuses the append + // must not lose a daemon that can still register. + } +} + +/** + * The phases observed in a startup marker tail, keyed by phase, holding each phase's earliest + * occurrence. Unparsable and unknown lines are ignored: the protocol is additive, and a client must + * never fail a startup over a line shape it does not know. + */ +export function readDaemonStartupPhaseMarkers( + markerTail: string, +): Partial> { + const phases: Partial> = {}; + for (const line of markerTail.split('\n')) { + if (!line.includes('"startupPhase"')) continue; + let record: DaemonStartupMarkerRecord; + try { + record = JSON.parse(line) as DaemonStartupMarkerRecord; + } catch { + continue; + } + const phase = record.data?.startupPhase; + const timestampMs = record.data?.timestampMs; + if ( + typeof phase !== 'string' || + !DAEMON_STARTUP_PHASE_SET.has(phase) || + typeof timestampMs !== 'number' || + !Number.isFinite(timestampMs) + ) { + continue; + } + const seen = phases[phase as DaemonStartupPhase]; + if (seen === undefined || timestampMs < seen) phases[phase as DaemonStartupPhase] = timestampMs; + } + return phases; +} export type DaemonRegistrationFields = Readonly<{ socketPort?: number; httpPort?: number; diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index eae6f66584..744baf062b 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -88,6 +88,8 @@ import { import { tryAcquireDaemonRegistration, DAEMON_STARTUP_EXIT_CODES, + recordDaemonStartupMarker, + resolveDaemonStartupPhasesPath, type DaemonRegistrationOwner, } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { watchDaemonMetadataLoss } from './daemon-metadata-loss.ts'; @@ -660,6 +662,10 @@ async function startDaemonRuntimeOnHost( rollback.onRollback(async () => { await finishDaemonRegistration({ registration, infoPath, logPath }); }); + // #3782: the client waiting on this state dir reads these markers to tell a slow daemon from a + // stuck one. + recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(baseDir), 'lock-acquired'); + rollback.onRollback(() => { stopMetadataLossWatch(); }); @@ -752,7 +758,11 @@ async function startDaemonRuntimeOnHost( }, rollback, ); + // #3782: the last two progress markers. A client that polls the registration and the markers + // together can tell "listening but slow to answer" from "never progressed". + recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(baseDir), 'listening'); publishDaemonInfo(socketPort, httpPort); + recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(baseDir), 'registered'); stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); for (const plugin of deps.pluginInventory.filter(({ state }) => state === 'failed')) { diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 2619c82eb5..65ce3e4fe6 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -41,6 +41,10 @@ const ENVIRONMENT_LINES = [ label: 'AGENT_DEVICE_DAEMON_AUTH_TOKEN', description: 'Remote daemon service/API token', }, + { + label: 'AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', + description: 'Local daemon startup ceiling (default 120000)', + }, { label: 'AGENT_DEVICE_CLOUD_BASE_URL', description: 'Bridge/control-plane API origin for cloud auth and /api-keys', @@ -508,6 +512,7 @@ Runner and daemon lifecycle (applies to simulators too): No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. + A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive but no startup progress for 60 s), or startup-ceiling (progressing, not registered before the ceiling). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts new file mode 100644 index 0000000000..4eefcae913 --- /dev/null +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -0,0 +1,324 @@ +// #3782: the local-daemon startup budget. A live daemon that starts slowly is waited for; a +// daemon that exits is failed fast; a live daemon that publishes no startup progress is retired at +// the stall window, not at the ceiling; and the ceiling itself is operator-settable. These cases +// pin each outcome at the `sendToDaemon` transport boundary, the route CLI, MCP, and Node share. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { ExecDetachedExit } from '@agent-device/host-kit/command'; +import { + DAEMON_STARTUP_EXIT_CODES, + recordDaemonStartupMarker, + resolveDaemonStartupPhasesPath, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { + resolveDaemonPaths, + type DaemonPaths, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import { + closeLoopbackServer, + mkdtempForTestSync, + supportsLoopbackBind, +} from '@agent-device/test-kit'; +import { isProcessAlive } from '@agent-device/host-kit/process'; +import { readVersion } from '@agent-device/host-kit/version'; +import { + spawnRegisteredDaemonFixture, + finishRegisteredDaemonFixtures, +} from '../../__tests__/test-utils/registered-daemon-fixture.ts'; +import { + startHttpDaemonFixture, + currentDaemonCodeSignature, +} from '../../__tests__/test-utils/daemon-http-fixture.ts'; + +vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({ + ...(await importOriginal()), + runCmdDetachedMonitored: vi.fn(), +})); +vi.mock('@agent-device/host-kit/retry', async (importOriginal) => ({ + ...(await importOriginal()), + sleep: vi.fn(), +})); +import { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; +import { sleep } from '@agent-device/host-kit/retry'; +import { sendToDaemon } from '../daemon-client.ts'; + +const actualRetry = await vi.importActual( + '@agent-device/host-kit/retry', +); +const spawn = vi.mocked(runCmdDetachedMonitored); +const pause = vi.mocked(sleep); + +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await finishRegisteredDaemonFixtures(); + spawn.mockReset(); + pause.mockReset(); +}); + +function request(paths: DaemonPaths, command = 'devices') { + return { + session: 'default', + command, + positionals: [], + flags: { stateDir: paths.baseDir, daemonTransport: 'http' as const }, + }; +} + +function fields(httpPort: number) { + return { + httpPort, + token: 'secret', + version: readVersion(), + codeOrigin: 'checkout' as const, + codeSignature: currentDaemonCodeSignature(), + }; +} + +/** + * A registered-fixture daemon with deferred publication: it acquires the registration, publishes + * no daemon.json, and never exits on its own — the startup the client must judge from process + * liveness and startup markers alone. Its command matches the daemon identity rules, so the + * client's retirement signals it and it exits, keeping each test's fixtures disposable. + */ +function spawnUnregisteredDaemonChild( + paths: DaemonPaths, + options: Parameters[2], +): ReturnType { + fs.mkdirSync(paths.baseDir, { recursive: true }); + fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); + return spawnRegisteredDaemonFixture(paths, fields(1), options); +} + +async function expectStartupFailure(pending: Promise): Promise { + let thrown: AppError | undefined; + try { + await pending; + } catch (error) { + if (!(error instanceof AppError)) throw error; + thrown = error; + } + assert.ok(thrown, 'expected the startup to reject'); + assert.equal(thrown.message, 'Failed to start daemon'); + assert.equal(thrown.details?.kind, 'daemon_startup_failed'); + return thrown; +} + +function startupPhases(failure: AppError): Record { + assert.ok( + failure.details?.phases && typeof failure.details.phases === 'object', + 'startup failure must record the phase timings it observed', + ); + return failure.details.phases as Record; +} + +/** Waits (on the unmocked clock) for one line of the client's own sleep seam. */ +async function realPause(ms = 5) { + await actualRetry.sleep(ms); +} + +test('a daemon slower than the retired 15 s budget starts successfully while it keeps progressing', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const http = await startHttpDaemonFixture({ devices: [] }); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-slow-success-')); + const markerPath = resolveDaemonStartupPhasesPath(paths.baseDir); + // Virtual time: the client waits ~20 s of startup — past the retired 15 s deadline — in real + // milliseconds. The clock stands still while the real fixture child is still booting (marker + // timestamps must come from this launch), and a progress marker every ~6 virtual seconds keeps + // the wait outside the 60 s stall window. + const startedAt = Date.now(); + const now = { value: startedAt }; + vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const spawned = { at: 0, child: undefined as { pid: number } | undefined }; + let booted = false; + spawn.mockImplementation((_command, _args, options) => { + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + spawned.at = now.value; + spawned.child = child; + return child; + }); + fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); + let polls = 0; + pause.mockImplementation(async () => { + polls += 1; + if (!booted) { + booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); + if (!booted) return await realPause(); + } + if (now.value - spawned.at < 20_000) { + now.value += 2_000; + if (polls % 3 === 0) recordDaemonStartupMarker(markerPath, 'lock-acquired'); + if (now.value - spawned.at >= 20_000) + fs.rmSync(path.join(paths.baseDir, 'defer-publication'), { force: true }); + return await realPause(); + } + // The client has waited past the retired 15 s budget; hold the virtual clock while the real + // child notices the lifted barrier and publishes, polling at full real speed. + await realPause(20); + }); + try { + const response = await sendToDaemon(request(paths)); + assert.equal(response.ok, true); + assert.ok( + now.value - spawned.at >= 20_000, + 'the startup this test proves must exceed the retired 15 s budget', + ); + assert.equal(spawn.mock.calls.length, 1); + assert.equal(http.rpcRequests.length, 1); + assert.ok(spawned.child && isProcessAlive(spawned.child.pid), 'the client must not retire it'); + } finally { + await closeLoopbackServer(http.server); + } +}); + +test('a daemon that exits during startup fails with startup-exited within one second of the exit', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-exit-fast-')); + let resolveFirstExit: ((exit: ExecDetachedExit) => void) | undefined; + let exitObservedAt = 0; + spawn.mockImplementation(() => { + // Fabricated pid: the retirement path proves daemon identity through the process table, finds + // no such process, and signals nothing. + const pid = 998_231 + spawn.mock.calls.length; + if (spawn.mock.calls.length === 1) { + // The exit arrives ~400 ms of real time after the spawn: the client must observe the + // monitor and report within its one-second detection budget. + const exited = new Promise((resolve) => { + resolveFirstExit = resolve; + }); + void exited.then(() => { + exitObservedAt = Date.now(); + }); + setTimeout(() => resolveFirstExit?.({ pid, exitCode: 1 }), 400); + return { pid, exited }; + } + return { pid, exited: Promise.resolve({ pid, exitCode: 1 }) }; + }); + pause.mockImplementation(async (ms) => { + await realPause(Math.min(Number(ms), 50)); + }); + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-exited'); + assert.ok(exitObservedAt > 0, 'the fixture must have delivered the exit'); + assert.ok(Date.now() - exitObservedAt <= 1_000, 'the failure must follow the exit within 1 s'); +}); + +test('a live daemon that publishes no startup marker is retired as stalled before the ceiling', async () => { + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '120000'); + const started = Date.now(); + const now = { value: started }; + vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-no-marker-')); + let child: ReturnType | undefined; + spawn.mockImplementation((_command, _args, options) => { + child = spawnUnregisteredDaemonChild(paths, options); + return child; + }); + let booted = false; + pause.mockImplementation(async () => { + if (!booted) { + booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); + if (!booted) return await realPause(); + } + // Advance the virtual clock to the stall window; freeze it afterward so the real-time + // retirement below cannot stretch the measured detection past it. + if (now.value - started < 70_000) now.value += 5_000; + await realPause(); + }); + try { + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + const elapsed = now.value - started; + assert.equal(failure.details?.reason, 'startup-stalled'); + assert.ok( + elapsed >= 60_000 && elapsed < 75_000, + `a stall must retire the daemon at the stall window, not the ceiling (${elapsed}ms)`, + ); + assert.equal(failure.details?.startupTimeoutMs, 120_000); + assert.equal(startupPhases(failure).spawned, 0); + assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); + assert.equal(spawn.mock.calls.length, 1, 'a stalled daemon is not retried'); + } finally { + await (child?.exited ?? Promise.resolve()); + } +}); + +test('a live daemon that keeps progressing but never registers fails at the env ceiling with phases', async () => { + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '30000'); + const started = Date.now(); + const now = { value: started }; + vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-ceiling-')); + const markerPath = resolveDaemonStartupPhasesPath(paths.baseDir); + let child: ReturnType | undefined; + let booted = false; + const spawned = { at: 0 }; + spawn.mockImplementation((_command, _args, options) => { + child = spawnUnregisteredDaemonChild(paths, options); + spawned.at = now.value; + return child; + }); + let polls = 0; + pause.mockImplementation(async () => { + polls += 1; + if (!booted) { + booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); + if (!booted) return await realPause(); + } + if (now.value - started < 30_000) { + now.value += 1_000; + // A progress marker every 10 virtual seconds: always inside the 60 s stall window, so only + // the ceiling can stop this wait. + if (polls % 10 === 0) recordDaemonStartupMarker(markerPath, 'lock-acquired'); + } + await realPause(); + }); + try { + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-ceiling'); + assert.equal(failure.details?.startupTimeoutMs, 30_000); + const phases = startupPhases(failure); + assert.equal(phases.spawned, 0); + assert.ok( + typeof phases['lock-acquired'] === 'number' && + phases['lock-acquired'] >= 0 && + phases['lock-acquired'] < 30_000, + 'the observed marker must be reported relative to the spawn', + ); + assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); + } finally { + await (child?.exited ?? Promise.resolve()); + } +}); + +test('an unproven registration exit reports startup-exited even under a short env ceiling', async () => { + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-exit-under-ceiling-')); + spawn.mockImplementation(() => ({ + pid: 998_444, + exited: Promise.resolve({ pid: 998_444, exitCode: DAEMON_STARTUP_EXIT_CODES.unproven }), + })); + pause.mockImplementation(async () => { + await realPause(); + }); + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-exited'); + assert.equal(failure.details?.startupTimeoutMs, 2_000); +}); + +test('a startup ceiling that is not a positive integer is rejected before any daemon spawns', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-bad-ceiling-')); + for (const raw of ['0', '-5', 'abc', '12.5']) { + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', raw); + await assert.rejects( + sendToDaemon(request(paths)), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message.includes('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS'), + ); + } + assert.equal(spawn.mock.calls.length, 0); +}); diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index 726c45031a..5871945144 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -205,6 +205,7 @@ test.for(['early exit', 'timeout'] as const)( let child: ReturnType | undefined; let now = Date.now(); let advanced = false; + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '15000'); vi.spyOn(Date, 'now').mockImplementation(() => now); spawn.mockImplementation((_command, _args, options) => { paths = resolveDaemonPaths(String(options?.env?.AGENT_DEVICE_STATE_DIR)); @@ -426,6 +427,9 @@ test('a joined busy contender waits for a published winner to become ready witho for (const held of [true, false]) { test(`startup uses one deadline when the claim is ${held ? 'held' : 'released for relaunch'}`, async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-budget-')); + // The one-deadline behavior is budget-independent; pin the ceiling so the test keeps its + // original 15 s shape while the production default is the #3782 ceiling. + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '15000'); const claim = tryAcquireProcessLock({ lockDirPath: paths.lockPath, owner: { ...readCurrentOwnerIdentity(), acquiredAtMs: Date.now() }, @@ -484,6 +488,9 @@ test.for([ ])('an older winner is replaced only with enough startup time ($budget)', async (expected, t) => { if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-older-winner-')); + // The replacement-window behavior is budget-independent; pin the ceiling so this test keeps its + // original 15 s shape while the production default is the #3782 ceiling. + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '15000'); const http = await startHttpDaemonFixture({ devices: [] }); const deferred = path.join(paths.baseDir, 'defer-publication'); fs.writeFileSync(deferred, 'wait'); diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 47ccda9d53..d6567bf434 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -22,9 +22,12 @@ import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/ho import type { findUnrecoveredRepairCommitFailure } from '@agent-device/daemon-contracts/session-repair-tombstone'; import { DAEMON_STARTUP_EXIT_CODES, + readDaemonStartupPhaseMarkers, createOwnedReplayStateDir, recoverAbandonedDaemonRegistration, + resolveDaemonStartupPhasesPath, type DaemonRetirementResult, + type DaemonStartupPhase, launchDaemonProcess, stopAndRetireDaemon, type OwnedReplayStateDir, @@ -85,14 +88,59 @@ type DaemonStartupWaitResult = | { kind: 'ready'; daemon: EnsuredDaemon } | { kind: 'early_exit'; exit: ExecDetachedExit } | { kind: 'unproven'; error: AppError } - | { kind: 'retry' | 'timeout' }; + | { kind: 'retry' | 'timeout' } + /** The process stayed alive but published no new startup phase within the stall window. */ + | { kind: 'stalled' } + /** The process stayed alive and kept progressing, but did not register before the ceiling. */ + | { kind: 'ceiling' }; -const DAEMON_STARTUP_TIMEOUT_MS = 15_000; +/** Startup phase markers one launch published, as absolute host-clock timestamps. */ +type StartupPhaseObservation = { + spawnedAtMs: number; + markers: Partial>; +}; + +/** + * #3782: the ceiling on one local daemon startup, shared by the CLI, MCP, and Node client, because + * all three reach the daemon through `startLocalDaemon`. A slow host makes startup slower, not + * failed: while the spawned daemon stays alive and keeps publishing phase markers, the client keeps + * waiting. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides the ceiling for a caller that must + * fail sooner than the host allows. + */ +const DAEMON_STARTUP_CEILING_MS = 120_000; +/** + * How long a live daemon may publish no new startup phase marker before the client stops waiting. + * Kept beside the ceiling as one pair: a stalled daemon is retired at the stall window, a + * progressing one is waited for up to the ceiling. An env ceiling below this window makes the + * ceiling the only stop; the stall never fires past the ceiling, only earlier than it. + */ +const DAEMON_STARTUP_STALL_WINDOW_MS = 60_000; const MINIMUM_DAEMON_TAKEOVER_BUDGET_MS = 5_000; + +const DAEMON_STARTUP_TIMEOUT_ENV = 'AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS'; + +/** Why a startup that never reached readiness failed, in terms a caller can branch on (#3782). */ +export type DaemonStartupFailureReason = 'startup-exited' | 'startup-stalled' | 'startup-ceiling'; + +function resolveDaemonStartupTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env[DAEMON_STARTUP_TIMEOUT_ENV]?.trim(); + if (raw === undefined || raw === '') return DAEMON_STARTUP_CEILING_MS; + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new AppError( + 'INVALID_ARGS', + `${DAEMON_STARTUP_TIMEOUT_ENV} must be a positive integer number of milliseconds`, + { value: raw }, + ); + } + return parsed; +} + const LIVE_DAEMON_PROBE_RETRIES = 3; const LIVE_DAEMON_PROBE_RETRY_DELAY_MS = 200; const DAEMON_STARTUP_ATTEMPTS = 2; const DAEMON_STARTUP_LOG_TAIL_BYTES = 64_000; +const DAEMON_STARTUP_MARKER_TAIL_BYTES = 16_000; const LOOPBACK_BLOCK_LIST = new net.BlockList(); LOOPBACK_BLOCK_LIST.addSubnet('127.0.0.0', 8, 'ipv4'); LOOPBACK_BLOCK_LIST.addAddress('::1', 'ipv6'); @@ -501,10 +549,14 @@ type FailedDaemonStartup = { startupError?: NormalizedError; daemonProcess?: ExecDetachedExit | { pid: number }; retry: boolean; + /** Set when the wait ended on startup progress rather than on an exit or a race. */ + reason?: DaemonStartupFailureReason; + phases?: StartupPhaseObservation; }; export async function startLocalDaemon(settings: DaemonClientSettings): Promise { - const deadline = Date.now() + DAEMON_STARTUP_TIMEOUT_MS; + const startupTimeoutMs = resolveDaemonStartupTimeoutMs(); + const deadline = Date.now() + startupTimeoutMs; const cleanupResults: DaemonRetirementResult[] = []; let failure: FailedDaemonStartup | undefined; let attempts = 0; @@ -517,24 +569,31 @@ export async function startLocalDaemon(settings: DaemonClientSettings): Promise< if (!result.retry) break; await sleep(Math.min(150, Math.max(0, deadline - Date.now()))); } - const { startError, startupError, daemonProcess }: Partial = failure ?? {}; + const { startError, startupError, daemonProcess, reason, phases }: Partial = + failure ?? {}; + const effectiveReason: DaemonStartupFailureReason = reason ?? 'startup-exited'; const state = getDaemonMetadataState(settings.paths); const daemonLogTail = readRecentLogTail(settings.paths.logPath); throw new AppError('COMMAND_FAILED', 'Failed to start daemon', { kind: 'daemon_startup_failed', + reason: effectiveReason, stateDir: settings.paths.baseDir, infoPath: settings.paths.infoPath, lockPath: settings.paths.lockPath, logPath: settings.paths.logPath, - startupTimeoutMs: DAEMON_STARTUP_TIMEOUT_MS, + startupTimeoutMs, startupAttempts: attempts, cleanupResults, startError, startupError, daemonProcess, + ...(phases ? { phases: describeStartupPhaseTimings(phases) } : {}), ...(daemonLogTail ? { daemonLogTail } : {}), metadataState: state, - hint: resolveDaemonStartupHint(state, settings.paths), + hint: + reason === 'startup-stalled' || reason === 'startup-ceiling' + ? resolveStartupProgressHint(reason, startupTimeoutMs) + : resolveDaemonStartupHint(state, settings.paths), }); } @@ -543,14 +602,17 @@ async function attemptLocalDaemonStartup( deadline: number, ): Promise<{ daemon: EnsuredDaemon } | FailedDaemonStartup> { let launch: DaemonStartupLaunch; + const spawnedAtMs = Date.now(); try { launch = startDaemon(settings); } catch (error) { return await recoverFailedDaemonLaunch(settings, error); } - const startup = await waitForDaemonStartup(deadline, settings, launch); + const observation: StartupPhaseObservation = { spawnedAtMs, markers: {} }; + const startup = await waitForDaemonStartup(deadline, settings, launch, observation); if (startup.kind === 'ready') return { daemon: startup.daemon }; - if (startup.kind === 'retry') return { retry: true }; + refreshStartupPhaseMarkers(settings, observation); + if (startup.kind === 'retry') return { retry: true, phases: observation }; if (startup.kind === 'unproven') { const startupError = normalizeError(startup.error); return { @@ -558,15 +620,24 @@ async function attemptLocalDaemonStartup( startError: startupError.message, startupError, daemonProcess: { pid: launch.pid }, + phases: observation, }; } const { cleanup, joined } = await retireStartupAttempt(settings, launch, deadline); + const reason: DaemonStartupFailureReason = + startup.kind === 'stalled' + ? 'startup-stalled' + : startup.kind === 'ceiling' + ? 'startup-ceiling' + : 'startup-exited'; const available = isRegistrationAvailable(inspectProcessLock(settings.paths.lockPath)); return { cleanup, retry: joined && startup.kind === 'early_exit' && available, + reason, startError: startup.kind === 'early_exit' ? describeDaemonEarlyExit(startup.exit) : undefined, daemonProcess: startup.kind === 'early_exit' ? startup.exit : { pid: launch.pid }, + phases: observation, }; } @@ -846,26 +917,114 @@ async function waitForDaemonStartup( deadline: number, settings: DaemonClientSettings, launch: DaemonStartupLaunch, + observation: StartupPhaseObservation, ): Promise { let earlyExit: ExecDetachedExit | undefined; - void launch.exited.then((exit) => { - earlyExit = exit; - }); + void launch.exited + .then((exit) => { + earlyExit = exit; + }) + // The exit is observed on the next poll; a rejected monitor promise leaves the liveness and + // stall paths to retire the launch, so it must not surface as an unhandled rejection. + .catch(() => {}); while (Date.now() < deadline) { if (earlyExit) { - const kind = classifyDaemonStartupExit(earlyExit); - if (kind === 'unproven') - return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings) }; - if (kind === 'failed') return { kind: 'early_exit', exit: earlyExit }; - const contender = await observeContendingDaemon(settings, deadline); - if (contender) return contender; + const settled = await judgeExitedDaemonStartup(settings, earlyExit, deadline); + if (settled) return settled; } else { - const info = await readReadyLaunchedDaemon(settings, launch, deadline); - if (info && !earlyExit) return { kind: 'ready', daemon: { info, startedByClient: true } }; + const settled = await observeLiveDaemonStartup(settings, launch, deadline, observation); + // An exit observed while this iteration was in flight outranks a `ready` read: the process + // that published is gone, and its classification owns the next step. + if (settled && !(settled.kind === 'ready' && earlyExit)) return settled; } await sleep(Math.min(100, remainingStartupBudget(deadline))); } - return { kind: 'timeout' }; + if (earlyExit || !isProcessAlive(launch.pid)) return { kind: 'timeout' }; + // A live daemon at the ceiling registered late, not never: only its retirement is this deadline's + // failure, and the reason says the host, not the daemon, ran out of room. + return { kind: 'ceiling' }; +} + +async function judgeExitedDaemonStartup( + settings: DaemonClientSettings, + exit: ExecDetachedExit, + deadline: number, +): Promise { + const kind = classifyDaemonStartupExit(exit); + if (kind === 'unproven') + return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings) }; + if (kind === 'failed') return { kind: 'early_exit', exit }; + return await observeContendingDaemon(settings, deadline); +} + +async function observeLiveDaemonStartup( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + deadline: number, + observation: StartupPhaseObservation, +): Promise { + const info = await readReadyLaunchedDaemon(settings, launch, deadline); + if (info) return { kind: 'ready', daemon: { info, startedByClient: true } }; + // #3782: a live daemon that keeps publishing startup markers is slow, not stuck, and is + // waited for up to the ceiling. Only silence for a full stall window retires it early. + refreshStartupPhaseMarkers(settings, observation); + if ( + isProcessAlive(launch.pid) && + Date.now() - lastStartupProgressMs(observation) >= DAEMON_STARTUP_STALL_WINDOW_MS + ) { + return { kind: 'stalled' }; + } + return null; +} + +/** + * #3782: folds this launch's startup phase markers into `observation`. The record is append-only + * across startups, so markers older than this launch are ignored by timestamp, never by deletion. + */ +function refreshStartupPhaseMarkers( + settings: DaemonClientSettings, + observation: StartupPhaseObservation, +): void { + const tail = readRecentFileTail( + resolveDaemonStartupPhasesPath(settings.paths.baseDir), + DAEMON_STARTUP_MARKER_TAIL_BYTES, + ); + if (!tail) return; + for (const [phase, timestampMs] of Object.entries(readDaemonStartupPhaseMarkers(tail)) as Array< + [DaemonStartupPhase, number] + >) { + if (timestampMs >= observation.spawnedAtMs) observation.markers[phase] = timestampMs; + } +} + +function lastStartupProgressMs(observation: StartupPhaseObservation): number { + return Object.values(observation.markers).reduce( + (latest, timestampMs) => Math.max(latest, timestampMs), + observation.spawnedAtMs, + ); +} + +function describeStartupPhaseTimings( + observation: StartupPhaseObservation, +): Record & { spawned: 0 } { + const timings: Record = { spawned: 0 }; + for (const [phase, timestampMs] of Object.entries(observation.markers)) { + timings[phase] = Math.max(0, timestampMs - observation.spawnedAtMs); + } + return timings as Record & { spawned: 0 }; +} + +function resolveStartupProgressHint(reason: DaemonStartupFailureReason, ceilingMs: number): string { + if (reason === 'startup-stalled') { + return ( + `The daemon is alive but made no startup progress for ${DAEMON_STARTUP_STALL_WINDOW_MS / 1000} s. ` + + `The host can be overloaded. Retry, or raise ${DAEMON_STARTUP_TIMEOUT_ENV}.` + ); + } + return ( + `The daemon kept making startup progress but did not register within ${Math.round(ceilingMs / 1000)} s. ` + + `The host can be overloaded. Retry, or raise ${DAEMON_STARTUP_TIMEOUT_ENV}.` + ); } function classifyDaemonStartupExit(exit: ExecDetachedExit): 'busy' | 'unproven' | 'failed' { @@ -966,12 +1125,16 @@ function describeDaemonEarlyExit(exit: ExecDetachedExit): string { } function readRecentLogTail(logPath: string): string | undefined { + return readRecentFileTail(logPath, DAEMON_STARTUP_LOG_TAIL_BYTES); +} + +function readRecentFileTail(filePath: string, maxBytes: number): string | undefined { try { - if (!fs.existsSync(logPath)) return undefined; - const stats = fs.statSync(logPath); + if (!fs.existsSync(filePath)) return undefined; + const stats = fs.statSync(filePath); if (stats.size <= 0) return undefined; - const length = Math.min(stats.size, DAEMON_STARTUP_LOG_TAIL_BYTES); - const fd = fs.openSync(logPath, 'r'); + const length = Math.min(stats.size, maxBytes); + const fd = fs.openSync(filePath, 'r'); try { const buffer = Buffer.alloc(length); fs.readSync(fd, buffer, 0, length, stats.size - length); diff --git a/src/daemon.ts b/src/daemon.ts index 5f34d5f54e..66972b90a6 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,6 +1,22 @@ import { asAppError } from '@agent-device/kernel/errors'; -import { daemonStartupExitCode, reportDaemonClose, startDaemonRuntime } from '@agent-device/daemon'; +import { + daemonStartupExitCode, + reportDaemonClose, + startDaemonRuntime, +} from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; +import { + recordDaemonStartupMarker, + resolveDaemonStartupPhasesPath, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; + +// #3782: the first marker the waiting client can read. It says this entry's module graph evaluated, +// which on an overloaded host is the slowest thing a daemon start does before it can prove itself. +recordDaemonStartupMarker( + resolveDaemonStartupPhasesPath(resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir), + 'modules-loaded', +); // The executable owns the process: signals, fatal-error handlers and the exit code. The runtime // only reports when it has closed, whoever closed it. diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index eb2896d7b0..ea2128ca76 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -122,7 +122,7 @@ Only the variables below, and those documented on command pages, are supported. | Category | Env vars | Decision | | --- | --- | --- | -| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public | +| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (whole-integer milliseconds, default `120000`) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for 60 s), or `startup-ceiling` (progressing, not registered before the ceiling). Branch on `details.reason`, never on the message. | | Device scoping | `AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST`, `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` | Public. `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` is read by the daemon: every iOS simulator request that names no `--ios-simulator-device-set` uses that set, so iOS physical devices are not listed, and a request naming a different path string fails with `INVALID_ARGS` (`details.reason: "ios_simulator_device_set_outside_daemon_set"`). macOS and non-iOS requests ignore it. A daemon started by the CLI inherits the CLI's value; restart the daemon after changing it. | | Android test IME | `AGENT_DEVICE_TEST_IME` | Public. Same setting as `--test-ime` / `--no-test-ime` on `open`, `test`, and `replay`; see known limitations. | | Local daemon storage | `AGENT_DEVICE_STATE_DIR` | Public | From a672ee2c276614257583ac0ad45d59f5a28c7e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 23:23:06 +0200 Subject: [PATCH 02/22] fix(daemon): give each startup its own marker file and retire it at its owner The integration leak oracle failed because every daemon appended its startup phase markers to one shared `daemon-startup-phases.jsonl` beside the registration and nothing retired that file, so any completed launch left state-dir residue the strict oracle names as a stray entry. Ownership now follows the file name: each launch writes only to `daemon-startup-phases-.jsonl` through `resolveDaemonStartupPhasesPath(stateDir, pid)`. The writing daemon removes its file once it has registered; the launching client removes it once that process's exit is joined (in `retireStartupAttempt`, after the exit is observed, never before); and each new registration-lock owner sweeps the files of provably dead pids next to the shutdown-report removal. A live pid's file is always kept, so a concurrent or replaced launch can neither add to, hide behind, nor delete another launch's evidence, and the client no longer needs `spawnedAtMs` to attribute markers to its own launch. Along the protocol boundary: - `recordDaemonStartupMarker` takes an optional `reportFailure` callback so producers surface a refused append on stderr (the daemon log tail the waiting client already ships as failure evidence) instead of swallowing it; the append stays non-fatal and a crash-truncated final line is separated so it cannot glue two records together. - The phase list gains real completed boundaries, `composition-loaded` and `recovery-finished`, and no heartbeats, so silence still means a stalled startup and progress extends the stall window exactly as far as the work took. - The launch-exit join drops its redundant `.catch(() => {})`: the `runCmdDetachedMonitored` owner resolves the promise on error and exit, so it never rejects. - Startup code splits out of `daemon-client-lifecycle.ts` into `daemon-client-startup.ts` (budget, wait loop, marker refresh, hints), keeping both modules under the 1000-line tripwire; importers attach and ensure point at the owning module directly. Regression coverage: the budget suite now pins per-launch attribution, including a fresh daemon producing its own markers while an earlier launch's leftover file is swept at the new registration, and a new protocol suite pins the per-pid paths, the dead-pid sweep, removal idempotence, and the write-failure seam. Refs #3782, #3791 --- ...registration-owner-startup-markers.test.ts | 145 ++++ .../src/daemon-registration-owner.ts | 109 ++- .../daemon/src/server/http/daemon-runtime.ts | 24 +- .../test-utils/registered-daemon-fixture.ts | 6 +- .../daemon-client-startup-budget.test.ts | 122 ++- src/daemon-client/daemon-client-attach.ts | 10 +- src/daemon-client/daemon-client-ensure.ts | 9 +- src/daemon-client/daemon-client-lifecycle.ts | 741 +----------------- src/daemon-client/daemon-client-startup.ts | 741 ++++++++++++++++++ src/daemon.ts | 31 +- 10 files changed, 1162 insertions(+), 776 deletions(-) create mode 100644 packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts create mode 100644 src/daemon-client/daemon-client-startup.ts diff --git a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts new file mode 100644 index 0000000000..fafbdd532b --- /dev/null +++ b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts @@ -0,0 +1,145 @@ +import { appendFile, readFile, writeFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + readDaemonStartupPhaseMarkers, + recordDaemonStartupMarker, + removeDaemonStartupPhasesFile, + resolveDaemonStartupPhasesPath, + tryAcquireDaemonRegistration, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { mkdtempForTest } from '@agent-device/test-kit'; + +describe('per-launch startup marker files', () => { + it('scopes marker files to the writing pid so a fresh launch never reads an older launch phases', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const first = resolveDaemonStartupPhasesPath(stateDir, 1111); + const second = resolveDaemonStartupPhasesPath(stateDir, 2222); + + recordDaemonStartupMarker(first, 'lock-acquired'); + + expect(second).not.toBe(first); + expect(existsSync(second)).toBe(false); + + recordDaemonStartupMarker(second, 'modules-loaded'); + const firstPhases = readDaemonStartupPhaseMarkers(readFileSync(first, 'utf8')); + const secondPhases = readDaemonStartupPhaseMarkers(readFileSync(second, 'utf8')); + expect(Object.keys(firstPhases)).toEqual(['lock-acquired']); + expect(Object.keys(secondPhases)).toEqual(['modules-loaded']); + }); + + it('keeps the earliest record when a replayed phase is appended after a later one', () => { + const text = [ + '{"data":{"startupPhase":"listening","timestampMs":500}}', + '{"data":{"startupPhase":"lock-acquired","timestampMs":20}}', + '{"data":{"startupPhase":"lock-acquired","timestampMs":35}}', + ].join('\n'); + const phases = readDaemonStartupPhaseMarkers(text); + expect(phases['lock-acquired']).toBe(20); + expect(phases.listening).toBe(500); + }); + + it('ignores unknown phases and unparsable lines so the protocol stays additive', () => { + const text = [ + '{"data":{"startupPhase":"heartbeat","timestampMs":10}}', + '{"data":{"startupPhase":"registered","timestampMs":"7"}}', + 'not json', + '{"data":{"startupPhase":"listening","timestampMs":30}}', + ].join('\n'); + expect(readDaemonStartupPhaseMarkers(text)).toEqual({ listening: 30 }); + }); +}); + +describe('registration-time sweep', () => { + it('retires marker files of provably-dead pids and keeps files of live pids', async () => { + const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); + const deadPid = spawnDeadPid(); + const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); + const dead = resolveDaemonStartupPhasesPath(paths.baseDir, deadPid); + const legacy = path.join(paths.baseDir, 'daemon-startup-phases.jsonl'); + await Promise.all([ + writeFile(own, 'x\n', 'utf8'), + writeFile(dead, 'x\n', 'utf8'), + writeFile(legacy, 'x\n', 'utf8'), + ]); + + const attempt = await tryAcquireDaemonRegistration(paths); + expect(attempt.status).toBe('acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + + expect(existsSync(dead)).toBe(false); + expect(existsSync(own)).toBe(true); + // The sweep retires only its own kind: an unknown entry stays as evidence. + expect(existsSync(legacy)).toBe(true); + await attempt.owner.finish(); + }); + + it('retires files whose pid is unparseable', async () => { + const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); + const stale = path.join(paths.baseDir, 'daemon-startup-phases-.jsonl'); + await writeFile(stale, 'x\n', 'utf8'); + + const attempt = await tryAcquireDaemonRegistration(paths); + expect(attempt.status).toBe('acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + + expect(existsSync(stale)).toBe(false); + await attempt.owner.finish(); + }); +}); + +describe('removeDaemonStartupPhasesFile', () => { + it('removes a marker file and tolerates a file already gone', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const file = resolveDaemonStartupPhasesPath(stateDir, 4242); + await writeFile(file, 'x\n', 'utf8'); + + removeDaemonStartupPhasesFile(file); + expect(existsSync(file)).toBe(false); + + expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow(); + }); +}); + +describe('recordDaemonStartupMarker failure seam', () => { + it('reports a write failure through the callback instead of throwing, keeping startup running', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + // A directory in place of the marker file makes every append fail. + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); + mkdirSync(markerPath, { recursive: true }); + + const reported: unknown[] = []; + expect(() => + recordDaemonStartupMarker(markerPath, 'modules-loaded', (error) => { + reported.push(error); + }), + ).not.toThrow(); + + expect(reported).toHaveLength(1); + expect(reported[0]).toHaveProperty('message'); + }); + + it('keeps appending valid JSONL when an incomplete line precedes a crash-replay append', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999); + await appendFile(markerPath, '{"data":{"startupPhase":"modules-lo', 'utf8'); + + recordDaemonStartupMarker(markerPath, 'composition-loaded'); + + const tail = await readFile(markerPath, 'utf8'); + expect(Object.keys(readDaemonStartupPhaseMarkers(tail))).toEqual(['composition-loaded']); + }); +}); + +function spawnDeadPid(): number { + // A process that has already exited proves the liveness check rejects it, + // the same way the daemon-process takeover guards prove it. + const child = spawnSync(process.execPath, ['-e', ''], { encoding: 'utf8' }); + if (child.error) throw child.error; + const pid = child.pid; + if (pid === undefined) throw new Error('spawnSync returned no pid'); + return pid; +} diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index e58dab8dad..c4bb64e0af 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -10,6 +10,7 @@ import { type DaemonTerminationResult, } from './daemon-process.ts'; import { + isProcessAlive, readCurrentOwnerIdentity, readProcessStartTime, type OwnerIdentity, @@ -48,16 +49,24 @@ import { export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); /** - * Startup phase markers (#3782): the record a daemon leaves beside its registration of how far its + * Startup phase markers (#3782): the record a daemon writes beside its registration of how far its * startup got. The client waiting for registration reads this record to tell "slow but progressing" - * from "stuck". Each marker is one JSONL line appended to a dedicated state-directory file, so no - * `daemon.log` reader has to learn a new line shape and `daemon.log` keeps its existing ownership. - * The record is append-only across startups: a waiting client scopes it to its own launch by the - * marker timestamps, never by deleting what an earlier startup left. + * from "stuck". One launch owns exactly one file, named for the pid that wrote it: an earlier or a + * concurrent launch cannot add to, replace, or hide another launch's markers, and neither the file + * nor a waiting client ever needs a timestamp to decide which startup a marker belongs to. A phase + * is recorded when the work it names has finished; the record never carries a heartbeat, so silence + * stays what a stalled startup looks like. + * + * Ownership, so the record never becomes residue: the writer removes the file once its startup + * completes; the client that launched a process removes it once that process's exit is joined; and + * each new registration owner sweeps the files of provably dead pids when it takes the lock. Each + * marker is one JSONL line appended to the launch's own file. */ const DAEMON_STARTUP_PHASES = Object.freeze([ 'modules-loaded', + 'composition-loaded', 'lock-acquired', + 'recovery-finished', 'listening', 'registered', ] as const); @@ -66,35 +75,96 @@ export type DaemonStartupPhase = (typeof DAEMON_STARTUP_PHASES)[number]; const DAEMON_STARTUP_PHASE_SET: ReadonlySet = new Set(DAEMON_STARTUP_PHASES); -const DAEMON_STARTUP_PHASES_FILE = 'daemon-startup-phases.jsonl'; +const DAEMON_STARTUP_PHASES_ENTRY = /^daemon-startup-phases-(\d*)\.jsonl$/; -type DaemonStartupMarkerRecord = { - data?: { startupPhase?: unknown; timestampMs?: unknown }; -}; - -export function resolveDaemonStartupPhasesPath(stateDir: string): string { - return path.join(stateDir, DAEMON_STARTUP_PHASES_FILE); +/** The marker file the daemon process `pid` writes in `stateDir`; an absent file is no evidence. */ +export function resolveDaemonStartupPhasesPath(stateDir: string, pid: number): string { + return path.join(stateDir, `daemon-startup-phases-${pid}.jsonl`); } +/** + * Appends one phase marker for the launch owning `startupPhasesPath`. A marker is progress + * evidence, never a startup requirement: a host that refuses the append must not lose a daemon + * that can still register. Producers that own a diagnostics channel pass `reportFailure` so the + * refusal is observable where the waiting client already reads failure evidence (the daemon log + * tail in the startup failure details); a client never receives a false marker from a broken + * channel — it reads a missing record as no evidence. + */ export function recordDaemonStartupMarker( startupPhasesPath: string, phase: DaemonStartupPhase, + reportFailure?: (error: NormalizedError) => void, ): void { try { const timestampMs = Date.now(); fs.mkdirSync(path.dirname(startupPhasesPath), { recursive: true }); + // A crash mid-append can leave the file without a trailing newline; without a separator the + // next append would glue into one unparsable line and lose both records. + const separator = endsWithoutNewline(startupPhasesPath) ? '\n' : ''; fs.appendFileSync( startupPhasesPath, - `${JSON.stringify({ data: { startupPhase: phase, timestampMs } })}\n`, + `${separator}${JSON.stringify({ data: { startupPhase: phase, timestampMs } })}\n`, ); + } catch (error) { + reportFailure?.(normalizeError(error)); + } +} + +function endsWithoutNewline(file: string): boolean { + let fd: number | undefined; + try { + const stats = fs.statSync(file); + if (stats.size === 0) return false; + const buffer = Buffer.alloc(1); + fd = fs.openSync(file, 'r'); + if (fs.readSync(fd, buffer, 0, 1, stats.size - 1) !== 1) return false; + return buffer[0] !== 0x0a; } catch { - // A marker is progress evidence, never a startup requirement: a host that refuses the append - // must not lose a daemon that can still register. + return false; + } finally { + if (fd !== undefined) fs.closeSync(fd); } } /** - * The phases observed in a startup marker tail, keyed by phase, holding each phase's earliest + * Removes one startup's marker file. The writer calls it once its own startup completes; the client + * that launched a process calls it once that process's exit is joined. Both callers can prove no + * other writer can still append to this file, which is what the pid-specific name encodes. + */ +export function removeDaemonStartupPhasesFile(startupPhasesPath: string): void { + try { + fs.rmSync(startupPhasesPath, { force: true }); + } catch { + // Evidence removal never fails a daemon that is already up or a client that already joined. + } +} + +/** + * Sweeps marker files left by provably dead launches, from the position of a process that holds the + * registration lock. A live pid's file is kept: its launch may still be between phases, and only + * that launch's own completion or a joined exit may remove it. A file whose name carries no usable + * pid cannot be owned by any launch, so this owner retires it too. + */ +function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { + let entries: string[]; + try { + entries = fs.readdirSync(stateDir); + } catch { + return; + } + for (const entry of entries) { + const match = DAEMON_STARTUP_PHASES_ENTRY.exec(entry); + if (match === null) continue; + const deadPid = Number(match[1]); + if (deadPid === process.pid) continue; + if (!Number.isSafeInteger(deadPid) || deadPid <= 0 || !isProcessAlive(deadPid)) { + removeDaemonStartupPhasesFile(path.join(stateDir, entry)); + } + } +} + +/** + * The phases observed in one launch's marker tail, keyed by phase, holding each phase's earliest * occurrence. Unparsable and unknown lines are ignored: the protocol is additive, and a client must * never fail a startup over a line shape it does not know. */ @@ -125,6 +195,10 @@ export function readDaemonStartupPhaseMarkers( } return phases; } + +type DaemonStartupMarkerRecord = { + data?: { startupPhase?: unknown; timestampMs?: unknown }; +}; export type DaemonRegistrationFields = Readonly<{ socketPort?: number; httpPort?: number; @@ -164,6 +238,9 @@ export async function tryAcquireDaemonRegistration( try { acquisition.assertHeld(); fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true }); + // A dead launch's markers are this owner's to sweep: only the launch itself or a client that + // joined its exit may remove a live launch's file, and neither can still reach this one. + sweepDeadDaemonStartupPhasesFiles(boundPaths.baseDir); } catch (error) { await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 744baf062b..1e62ad18a0 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -89,6 +89,7 @@ import { tryAcquireDaemonRegistration, DAEMON_STARTUP_EXIT_CODES, recordDaemonStartupMarker, + removeDaemonStartupPhasesFile, resolveDaemonStartupPhasesPath, type DaemonRegistrationOwner, } from '@agent-device/daemon-contracts/daemon-registration-owner'; @@ -663,8 +664,16 @@ async function startDaemonRuntimeOnHost( await finishDaemonRegistration({ registration, infoPath, logPath }); }); // #3782: the client waiting on this state dir reads these markers to tell a slow daemon from a - // stuck one. - recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(baseDir), 'lock-acquired'); + // stuck one. Each marker names a phase whose work has finished; this launch's file is named for + // its own pid, so no other launch's evidence can add to or hide it. + const startupPhasesPath = resolveDaemonStartupPhasesPath(baseDir, process.pid); + const recordStartupPhase = (phase: Parameters[1]): void => + recordDaemonStartupMarker(startupPhasesPath, phase, (error) => { + // Same contract as the entry's markers: the refusal goes where the waiting client already + // reads failure evidence (the daemon log), and startup keeps going. + stderr.write(`Daemon startup marker write failed: ${error.message}\n`); + }); + recordStartupPhase('lock-acquired'); rollback.onRollback(() => { stopMetadataLossWatch(); @@ -719,6 +728,10 @@ async function startDaemonRuntimeOnHost( data: { error: error instanceof Error ? error.message : String(error) }, }); }); + // #3782: the recovery chain above (owner-lifecycle configuration, legacy-marker and app-log + // recovery, owned-process reaping, web-orphan cleanup) is real completed work: its boundary + // extends the stall window exactly as much as it took, never more. + recordStartupPhase('recovery-finished'); const { openDaemonServers, reconcileDeviceClaimsForDaemonStartup } = await import('./server-startup.ts'); const { servers, socketPort, httpPort } = await openDaemonServers( @@ -760,9 +773,12 @@ async function startDaemonRuntimeOnHost( ); // #3782: the last two progress markers. A client that polls the registration and the markers // together can tell "listening but slow to answer" from "never progressed". - recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(baseDir), 'listening'); + recordStartupPhase('listening'); publishDaemonInfo(socketPort, httpPort); - recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(baseDir), 'registered'); + recordStartupPhase('registered'); + // This startup's evidence has done its job: no waiting client needs it anymore, and this + // process is the only writer of the file, so this is the removal its owner contract allows. + removeDaemonStartupPhasesFile(startupPhasesPath); stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); for (const plugin of deps.pluginInventory.filter(({ state }) => state === 'failed')) { diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index a4240ce539..250a91032d 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -34,7 +34,7 @@ export function registeredDaemonFixtureArgs( entry, `import fs from 'node:fs'; import path from 'node:path'; -import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; +import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; const paths = ${JSON.stringify(paths)}; const barrier = ${JSON.stringify(acquisitionBarrier)}; if (barrier) { @@ -43,6 +43,10 @@ if (barrier) { } const acquired = await tryAcquireDaemonRegistration(paths); if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); +// A suite that wants this child to behave like the real entry's marker producer drops this flag +// file before spawning; the marker is written through the real protocol, to this pid's own file. +if (fs.existsSync(path.join(paths.baseDir, 'emit-startup-markers'))) + recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(paths.baseDir, process.pid), 'lock-acquired'); process.on('SIGTERM', async () => { if (fs.existsSync(path.join(paths.baseDir, 'ignore-sigterm'))) return; const deferred = path.join(paths.baseDir, 'repair-on-shutdown.json'); diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 4eefcae913..8834f897a5 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -88,9 +88,13 @@ function fields(httpPort: number) { function spawnUnregisteredDaemonChild( paths: DaemonPaths, options: Parameters[2], + emitStartupMarkers = false, ): ReturnType { fs.mkdirSync(paths.baseDir, { recursive: true }); fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); + // When asked, the child records its phases through the real protocol, into the file named for + // its own pid, exactly as the daemon entry does. + if (emitStartupMarkers) fs.writeFileSync(path.join(paths.baseDir, 'emit-startup-markers'), 'on'); return spawnRegisteredDaemonFixture(paths, fields(1), options); } @@ -121,15 +125,22 @@ async function realPause(ms = 5) { await actualRetry.sleep(ms); } +async function awaitFile(file: string) { + for (let attempt = 0; !fs.existsSync(file); attempt += 1) { + assert.ok(attempt < 400, `fixture did not publish ${file}`); + await actualRetry.sleep(10); + } +} + test('a daemon slower than the retired 15 s budget starts successfully while it keeps progressing', async (t) => { if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const http = await startHttpDaemonFixture({ devices: [] }); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-slow-success-')); - const markerPath = resolveDaemonStartupPhasesPath(paths.baseDir); // Virtual time: the client waits ~20 s of startup — past the retired 15 s deadline — in real // milliseconds. The clock stands still while the real fixture child is still booting (marker // timestamps must come from this launch), and a progress marker every ~6 virtual seconds keeps - // the wait outside the 60 s stall window. + // the wait outside the 60 s stall window. Markers go to this launch's own pid-named file, which + // is where a real daemon writes them. const startedAt = Date.now(); const now = { value: startedAt }; vi.spyOn(Date, 'now').mockImplementation(() => now.value); @@ -151,7 +162,11 @@ test('a daemon slower than the retired 15 s budget starts successfully while it } if (now.value - spawned.at < 20_000) { now.value += 2_000; - if (polls % 3 === 0) recordDaemonStartupMarker(markerPath, 'lock-acquired'); + if (polls % 3 === 0 && spawned.child) + recordDaemonStartupMarker( + resolveDaemonStartupPhasesPath(paths.baseDir, spawned.child.pid), + 'lock-acquired', + ); if (now.value - spawned.at >= 20_000) fs.rmSync(path.join(paths.baseDir, 'defer-publication'), { force: true }); return await realPause(); @@ -240,6 +255,15 @@ test('a live daemon that publishes no startup marker is retired as stalled befor assert.equal(startupPhases(failure).spawned, 0); assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); assert.equal(spawn.mock.calls.length, 1, 'a stalled daemon is not retried'); + // #3782 residue rule: the retired launch's exit was joined, so its marker file is removed — + // a startup the client owned must leave no evidence behind. The prefix check is name-agnostic + // so no marker-file shape can pass by accident. + await (child?.exited ?? Promise.resolve()); + assert.deepEqual( + fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + [], + 'a joined startup must sweep its own phase-marker file', + ); } finally { await (child?.exited ?? Promise.resolve()); } @@ -251,12 +275,11 @@ test('a live daemon that keeps progressing but never registers fails at the env const now = { value: started }; vi.spyOn(Date, 'now').mockImplementation(() => now.value); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-ceiling-')); - const markerPath = resolveDaemonStartupPhasesPath(paths.baseDir); let child: ReturnType | undefined; let booted = false; const spawned = { at: 0 }; spawn.mockImplementation((_command, _args, options) => { - child = spawnUnregisteredDaemonChild(paths, options); + child = spawnUnregisteredDaemonChild(paths, options, true); spawned.at = now.value; return child; }); @@ -267,12 +290,10 @@ test('a live daemon that keeps progressing but never registers fails at the env booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); if (!booted) return await realPause(); } - if (now.value - started < 30_000) { - now.value += 1_000; - // A progress marker every 10 virtual seconds: always inside the 60 s stall window, so only - // the ceiling can stop this wait. - if (polls % 10 === 0) recordDaemonStartupMarker(markerPath, 'lock-acquired'); - } + // The real child has published its own `lock-acquired` marker: progress this wait keeps + // extending is the daemon's, not the test's. Virtual time advances 1 s per poll; the 30 s + // env ceiling stops the wait well before the 60 s stall window can. + if (now.value - started < 30_000) now.value += 1_000; await realPause(); }); try { @@ -293,6 +314,85 @@ test('a live daemon that keeps progressing but never registers fails at the env } }); +test("a startup progressed by a fresh launch is not judged stalled by an earlier launch's leftover markers", async () => { + // F1 (#3782): a reused state dir still holds the phase record a dead earlier startup wrote. The + // fresh launch records its own progress through the real protocol, from its own process. The + // shared-record reader kept the earliest occurrence per phase, so the older record hid the fresh + // launch's `lock-acquired` from the failure details entirely — the exact loss of attribution that + // turns a progressing startup into an empty observation and a false stall. The state dir must + // also sweep what its own launches leave behind. + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '30000'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-stale-history-')); + fs.mkdirSync(paths.baseDir, { recursive: true }); + // The leftover launch: a real daemon-shaped child that records its phase, is SIGTERMed before + // publishing, and leaves its record behind exactly as a crashed startup would. + fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); + fs.writeFileSync(path.join(paths.baseDir, 'emit-startup-markers'), 'on'); + const leftover = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }); + await awaitFile(path.join(paths.baseDir, 'registration-held')); + process.kill(leftover.pid, 'SIGTERM'); + await leftover.exited; + const leftoverRecords = () => + fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')); + assert.ok( + leftoverRecords().length > 0, + 'the leftover launch must have left a real daemon-written marker record', + ); + + const started = Date.now(); + const now = { value: started }; + vi.spyOn(Date, 'now').mockImplementation(() => now.value); + let child: ReturnType | undefined; + let booted = false; + fs.rmSync(path.join(paths.baseDir, 'registration-held'), { force: true }); + spawn.mockImplementation((_command, _args, options) => { + child = spawnUnregisteredDaemonChild(paths, options, true); + return child; + }); + pause.mockImplementation(async () => { + if (!booted) { + booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); + if (!booted) return await realPause(); + } + // The 30 s env ceiling stops this wait before the 60 s stall window can; the fresh child's own + // `lock-acquired` marker is the only progress evidence and must reach the failure record. + if (now.value - started < 30_000) now.value += 1_000; + await realPause(); + }); + try { + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-ceiling'); + const phases = startupPhases(failure); + assert.equal( + typeof phases['lock-acquired'], + 'number', + 'the fresh launch must observe its own marker despite the older record', + ); + await (child?.exited ?? Promise.resolve()); + assert.ok( + !leftoverRecords().some((entry) => entry.includes(String(child?.pid))), + "the joined exit must sweep this launch's own marker file", + ); + // A new registration owner sweeps the leftover record of a provably dead launch. + const cleaner = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }); + try { + await awaitFile(path.join(paths.baseDir, 'registration-held')); + assert.deepEqual( + leftoverRecords(), + [], + "acquiring the registration sweeps every dead launch's marker record", + ); + } finally { + process.kill(cleaner.pid, 'SIGTERM'); + await cleaner.exited; + } + } finally { + // The client's own retirement (or the sweep test's SIGTERM) already stopped this child; the + // hermetic signal guard refuses any signal the worker did not need to send. + await (child?.exited ?? Promise.resolve()); + } +}); + test('an unproven registration exit reports startup-exited even under a short env ceiling', async () => { vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-exit-under-ceiling-')); diff --git a/src/daemon-client/daemon-client-attach.ts b/src/daemon-client/daemon-client-attach.ts index 9597a83f1a..975c80eb95 100644 --- a/src/daemon-client/daemon-client-attach.ts +++ b/src/daemon-client/daemon-client-attach.ts @@ -1,10 +1,6 @@ -import { - assertDaemonPolicyMatches, - ensureManagedDaemon, - observeLocalDaemon, - type DaemonClientSettings, - type EnsuredDaemon, -} from './daemon-client-lifecycle.ts'; +import { ensureManagedDaemon } from './daemon-client-lifecycle.ts'; +import { assertDaemonPolicyMatches, observeLocalDaemon } from './daemon-client-startup.ts'; +import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; import { daemonIncompatibleError, daemonNotRunningError } from './daemon-lifecycle-errors.ts'; /** diff --git a/src/daemon-client/daemon-client-ensure.ts b/src/daemon-client/daemon-client-ensure.ts index d2d6696b81..3db48b3902 100644 --- a/src/daemon-client/daemon-client-ensure.ts +++ b/src/daemon-client/daemon-client-ensure.ts @@ -9,16 +9,15 @@ import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process import { recoverAbandonedDaemonRegistration } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { sleep } from '@agent-device/host-kit/retry'; import { AppError } from '@agent-device/kernel/errors'; +import { resolveLocalDaemonSettings } from './daemon-client-lifecycle.ts'; import { assertDaemonPolicyMatches, observeLocalDaemon, - resolveLocalDaemonSettings, retireDaemonForTakeover, - stopDaemonForTakeover, startLocalDaemon, - type DaemonClientSettings, - type EnsuredDaemon, -} from './daemon-client-lifecycle.ts'; + stopDaemonForTakeover, +} from './daemon-client-startup.ts'; +import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; import { requestDaemonQuiesce } from './daemon-client-quiesce.ts'; import { probeDaemonResponsive } from './daemon-client-liveness-probe.ts'; import type { DaemonInfo } from './daemon-client-metadata.ts'; diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index d6567bf434..9d433fdf85 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -1,37 +1,19 @@ +// The daemon a request reaches: which settings resolve to which daemon (local, remote, or a +// private one-shot directory), the managed ensure that defers to the startup module, and the +// per-request cleanup that owns one-shot replay lifetimes. Starting and judging a daemon lives in +// `daemon-client-startup.ts`. import type { DaemonWireRequest } from '@agent-device/contracts/command'; import type { DaemonLifecycle, EnsureDaemonOptions } from '@agent-device/contracts/client'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; -import { - daemonContendedError, - daemonRetirementUnconfirmedError, - daemonStopCommand, -} from './daemon-lifecycle-errors.ts'; -import { attachActiveSessionAddressHint } from './daemon-client-address-hints.ts'; -import fs from 'node:fs'; -import { processLockHoldsDaemonIdentity } from '@agent-device/daemon-contracts/daemon-registration'; import net from 'node:net'; import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; -import { readReplayDivergenceResume } from '@agent-device/ad-replay/divergence'; -import { type ExecDetachedExit } from '@agent-device/host-kit/command'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; +import { readReplayDivergenceResume } from '@agent-device/ad-replay/divergence'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { isProcessAlive } from '@agent-device/host-kit/process'; -import { sleep } from '@agent-device/host-kit/retry'; -import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/host-kit/file'; - -import type { findUnrecoveredRepairCommitFailure } from '@agent-device/daemon-contracts/session-repair-tombstone'; import { - DAEMON_STARTUP_EXIT_CODES, - readDaemonStartupPhaseMarkers, createOwnedReplayStateDir, - recoverAbandonedDaemonRegistration, - resolveDaemonStartupPhasesPath, - type DaemonRetirementResult, - type DaemonStartupPhase, - launchDaemonProcess, stopAndRetireDaemon, type OwnedReplayStateDir, - type DaemonStartupLaunch, } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { resolveDaemonPaths, @@ -41,25 +23,18 @@ import { type DaemonServerMode, type DaemonTransportPreference, } from '@agent-device/daemon-contracts/daemon-resolution'; -import { - resolveDaemonLaunchSpec, - resolveDaemonTakeover, - type DaemonTakeoverDecision, -} from './daemon-launch-spec.ts'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { findUnrecoveredRepairCommitFailure } from '@agent-device/daemon-contracts/session-repair-tombstone'; +import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { attachActiveSessionAddressHint } from './daemon-client-address-hints.ts'; +import { isRemoteDaemon } from './daemon-client-metadata.ts'; +import { cachedRemoteDaemonHealth } from './daemon-client-transport.ts'; import { - getDaemonMetadataState, - isRemoteDaemon, - readDaemonInfo, - resolveDaemonStartupHint, - type DaemonInfo, -} from './daemon-client-metadata.ts'; -import { - canConnect, - cachedRemoteDaemonHealth, - isDaemonTransportUnavailableError, -} from './daemon-client-transport.ts'; + assertDaemonPolicyMatches, + readReusableLocalDaemon, + startLocalDaemon, +} from './daemon-client-startup.ts'; export type DaemonClientSettings = { /** The environment and working directory this client reads, for every setting below and the launch. */ @@ -84,63 +59,6 @@ export type EnsuredDaemon = { startedByClient: boolean; }; -type DaemonStartupWaitResult = - | { kind: 'ready'; daemon: EnsuredDaemon } - | { kind: 'early_exit'; exit: ExecDetachedExit } - | { kind: 'unproven'; error: AppError } - | { kind: 'retry' | 'timeout' } - /** The process stayed alive but published no new startup phase within the stall window. */ - | { kind: 'stalled' } - /** The process stayed alive and kept progressing, but did not register before the ceiling. */ - | { kind: 'ceiling' }; - -/** Startup phase markers one launch published, as absolute host-clock timestamps. */ -type StartupPhaseObservation = { - spawnedAtMs: number; - markers: Partial>; -}; - -/** - * #3782: the ceiling on one local daemon startup, shared by the CLI, MCP, and Node client, because - * all three reach the daemon through `startLocalDaemon`. A slow host makes startup slower, not - * failed: while the spawned daemon stays alive and keeps publishing phase markers, the client keeps - * waiting. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides the ceiling for a caller that must - * fail sooner than the host allows. - */ -const DAEMON_STARTUP_CEILING_MS = 120_000; -/** - * How long a live daemon may publish no new startup phase marker before the client stops waiting. - * Kept beside the ceiling as one pair: a stalled daemon is retired at the stall window, a - * progressing one is waited for up to the ceiling. An env ceiling below this window makes the - * ceiling the only stop; the stall never fires past the ceiling, only earlier than it. - */ -const DAEMON_STARTUP_STALL_WINDOW_MS = 60_000; -const MINIMUM_DAEMON_TAKEOVER_BUDGET_MS = 5_000; - -const DAEMON_STARTUP_TIMEOUT_ENV = 'AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS'; - -/** Why a startup that never reached readiness failed, in terms a caller can branch on (#3782). */ -export type DaemonStartupFailureReason = 'startup-exited' | 'startup-stalled' | 'startup-ceiling'; - -function resolveDaemonStartupTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { - const raw = env[DAEMON_STARTUP_TIMEOUT_ENV]?.trim(); - if (raw === undefined || raw === '') return DAEMON_STARTUP_CEILING_MS; - const parsed = Number(raw); - if (!Number.isSafeInteger(parsed) || parsed <= 0) { - throw new AppError( - 'INVALID_ARGS', - `${DAEMON_STARTUP_TIMEOUT_ENV} must be a positive integer number of milliseconds`, - { value: raw }, - ); - } - return parsed; -} - -const LIVE_DAEMON_PROBE_RETRIES = 3; -const LIVE_DAEMON_PROBE_RETRY_DELAY_MS = 200; -const DAEMON_STARTUP_ATTEMPTS = 2; -const DAEMON_STARTUP_LOG_TAIL_BYTES = 64_000; -const DAEMON_STARTUP_MARKER_TAIL_BYTES = 16_000; const LOOPBACK_BLOCK_LIST = new net.BlockList(); LOOPBACK_BLOCK_LIST.addSubnet('127.0.0.0', 8, 'ipv4'); LOOPBACK_BLOCK_LIST.addAddress('::1', 'ipv6'); @@ -314,402 +232,6 @@ async function ensureRemoteDaemon(settings: DaemonClientSettings): Promise { - const { deadline } = options; - const inspection = inspectProcessLock(settings.paths.lockPath); - if (inspection.state === 'unproven') throw daemonRegistrationUnprovenError(settings, inspection); - const existing = readDaemonInfo(settings.paths.infoPath); - if (!existing) return null; - if (!registrationAllowsDaemonObservation(inspection, existing)) return null; - if (options.waitForLiveStartup && (await isAwaitingDaemonStartup(existing, deadline))) - return null; - - const decision = await resolveDaemonTakeover(existing, { - onClientTransport: () => - canReachReusableDaemon(existing, settings.transportPreference, deadline), - onAnyAdvertisedTransport: () => canReachReusableDaemon(existing, 'auto', deadline), - }); - return { info: existing, decision: await refineReplacement(existing, decision, options) }; -} - -/** - * Tells apart what the ladder lumps together, for callers that must not kill a daemon on a guess: a - * registration whose process is gone is unreachable whatever its version, and one that answers on - * another transport is alive. - */ -async function refineReplacement( - info: DaemonInfo, - decision: DaemonTakeoverDecision, - options: { deadline?: number; refineUnreachable?: boolean }, -): Promise { - if (!options.refineUnreachable || decision.kind !== 'replace') return decision; - if (!isProcessAlive(info.pid)) return { ...decision, cause: 'unreachable' }; - if ( - decision.cause === 'unreachable' && - (await canReachReusableDaemon(info, 'auto', options.deadline)) - ) { - return { ...decision, cause: 'transport' }; - } - return decision; -} - -async function readReusableLocalDaemon( - settings: DaemonClientSettings, - options: { deadline?: number; waitForLiveStartup?: boolean } = {}, -): Promise { - const { deadline } = options; - const observed = await observeLocalDaemon(settings, options); - if (!observed) return null; - const { info: existing, decision } = observed; - if (decision.kind === 'reuse') return existing; - if (decision.kind === 'refuseNewer') { - throw newerDaemonRefusedError(existing, decision, settings.paths.baseDir); - } - - if (!settings.retireIncompatibleContender) throw daemonContendedError(); - if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; - emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); - await retireDaemonForTakeover(existing, settings.paths); - return null; -} - -async function isAwaitingDaemonStartup( - info: DaemonInfo, - deadline: number | undefined, -): Promise { - return ( - isProcessAlive(info.pid) && !(await canConnect(info, 'auto', remainingStartupBudget(deadline))) - ); -} - -function registrationAllowsDaemonObservation( - inspection: ProcessLockInspection, - info: DaemonInfo, -): boolean { - return ( - inspection.state === 'absent' || - processLockHoldsDaemonIdentity(inspection, { - pid: info.pid, - startTime: info.processStartTime ?? null, - }) - ); -} - -function daemonRegistrationUnprovenError( - settings: DaemonClientSettings, - inspection?: ProcessLockInspection, -): AppError { - return new AppError('COMMAND_FAILED', 'Daemon registration ownership could not be verified.', { - reason: 'daemon_registration_unproven', - inspection, - stateDir: settings.paths.baseDir, - hint: resolveDaemonStartupHint(getDaemonMetadataState(settings.paths), settings.paths), - }); -} - -export async function stopDaemonForTakeover( - existing: DaemonInfo, - paths: DaemonPaths, - mode: 'graceful' | 'force' = 'graceful', -): Promise { - return await stopAndRetireDaemon({ - paths: paths, - observed: { pid: existing.pid, startTime: existing.processStartTime ?? null }, - mode, - }); -} - -export async function retireDaemonForTakeover( - existing: DaemonInfo, - paths: DaemonPaths, - mode: 'graceful' | 'force' = 'graceful', -): Promise { - const retirement = await stopDaemonForTakeover(existing, paths, mode); - if (retirement.status === 'retained') throw daemonRetirementUnconfirmedError(retirement, paths); -} - -/** - * A daemon whose pid is still alive is probed again before it can be judged unreachable. A probe's - * budget is wall-clock time on this client's event loop, so a client that stalls past it (a large - * synchronous parse, a GC pause on a loaded host) reads a listening daemon as unreachable, and - * replacing it ends every session the daemon holds. Liveness is the signal-0 check, not the `ps` - * identity read: under the load that stalls the probe, `ps` misses its deadline too, and the - * takeover still proves identity before it signals anything. - */ -async function canReachReusableDaemon( - info: DaemonInfo, - preference: DaemonTransportPreference, - deadline?: number, -): Promise { - if (await canConnectReusableDaemon(info, preference, deadline)) return true; - for (let retry = 1; retry <= LIVE_DAEMON_PROBE_RETRIES; retry += 1) { - if (!isProcessAlive(info.pid) || (deadline !== undefined && Date.now() >= deadline)) - return false; - await sleep(Math.min(LIVE_DAEMON_PROBE_RETRY_DELAY_MS, remainingStartupBudget(deadline))); - if (await canConnectReusableDaemon(info, preference, deadline)) { - emitDiagnostic({ - level: 'warn', - phase: 'daemon_probe_recovered', - data: { pid: info.pid, retry }, - }); - return true; - } - } - return false; -} - -/** - * ADR 0029: a caller that names a daemon policy must not silently use a daemon that enforces a - * different one (or none). A caller that names no policy uses whatever the daemon enforces. - */ -export async function assertDaemonPolicyMatches( - existing: DaemonInfo, - stateDir: string, - host: HostEnvironment, -): Promise { - const { env } = host; - if (!env.AGENT_DEVICE_DAEMON_POLICY?.trim()) return; - const { loadDaemonPolicy } = await import('@agent-device/daemon-contracts/daemon-policy-file'); - const expected = loadDaemonPolicy(env, host.cwd)?.digest; - if (expected === existing.policyDigest) return; - throw new AppError( - 'COMMAND_FAILED', - 'The running daemon does not enforce the daemon policy named by AGENT_DEVICE_DAEMON_POLICY.', - { - reason: 'DAEMON_POLICY_MISMATCH', - expectedPolicyDigest: expected, - daemonPolicyDigest: existing.policyDigest ?? null, - hint: `Stop the running daemon (${daemonStopCommand(stateDir)}), then retry so a daemon starts with this policy.`, - }, - ); -} - -async function canConnectReusableDaemon( - info: DaemonInfo, - preference: DaemonTransportPreference, - deadline?: number, -): Promise { - try { - return await canConnect(info, preference, remainingStartupBudget(deadline)); - } catch (error) { - if (isDaemonTransportUnavailableError(error)) return false; - throw error; - } -} - -function newerDaemonRefusedError( - info: DaemonInfo, - decision: Extract, - stateDir: string, -): AppError { - const { daemonVersion, clientVersion } = decision; - return new AppError( - 'COMMAND_FAILED', - `Daemon (pid ${info.pid}, v${daemonVersion}) is newer than this client (v${clientVersion}); refusing to replace it.`, - { - daemonPid: info.pid, - daemonVersion, - clientVersion, - hint: `Use the agent-device v${daemonVersion} CLI that started it, or stop it deliberately: ${daemonStopCommand(stateDir)}`, - }, - ); -} - -function emitDaemonTakeoverNotice(info: DaemonInfo, reason: string, stateDir: string): void { - try { - const identity = info.version ? `pid ${info.pid}, v${info.version}` : `pid ${info.pid}`; - process.stderr.write(`Replacing daemon (${identity}) in ${stateDir}: ${reason}\n`); - } catch { - // The takeover notice is best effort; never fail the command on stderr issues. - } -} - -type FailedDaemonStartup = { - cleanup?: DaemonRetirementResult; - startError?: string; - startupError?: NormalizedError; - daemonProcess?: ExecDetachedExit | { pid: number }; - retry: boolean; - /** Set when the wait ended on startup progress rather than on an exit or a race. */ - reason?: DaemonStartupFailureReason; - phases?: StartupPhaseObservation; -}; - -export async function startLocalDaemon(settings: DaemonClientSettings): Promise { - const startupTimeoutMs = resolveDaemonStartupTimeoutMs(); - const deadline = Date.now() + startupTimeoutMs; - const cleanupResults: DaemonRetirementResult[] = []; - let failure: FailedDaemonStartup | undefined; - let attempts = 0; - while (attempts < DAEMON_STARTUP_ATTEMPTS && Date.now() < deadline) { - attempts += 1; - const result = await attemptLocalDaemonStartup(settings, deadline); - if ('daemon' in result) return result.daemon; - failure = result; - if (result.cleanup) cleanupResults.push(result.cleanup); - if (!result.retry) break; - await sleep(Math.min(150, Math.max(0, deadline - Date.now()))); - } - const { startError, startupError, daemonProcess, reason, phases }: Partial = - failure ?? {}; - const effectiveReason: DaemonStartupFailureReason = reason ?? 'startup-exited'; - const state = getDaemonMetadataState(settings.paths); - const daemonLogTail = readRecentLogTail(settings.paths.logPath); - throw new AppError('COMMAND_FAILED', 'Failed to start daemon', { - kind: 'daemon_startup_failed', - reason: effectiveReason, - stateDir: settings.paths.baseDir, - infoPath: settings.paths.infoPath, - lockPath: settings.paths.lockPath, - logPath: settings.paths.logPath, - startupTimeoutMs, - startupAttempts: attempts, - cleanupResults, - startError, - startupError, - daemonProcess, - ...(phases ? { phases: describeStartupPhaseTimings(phases) } : {}), - ...(daemonLogTail ? { daemonLogTail } : {}), - metadataState: state, - hint: - reason === 'startup-stalled' || reason === 'startup-ceiling' - ? resolveStartupProgressHint(reason, startupTimeoutMs) - : resolveDaemonStartupHint(state, settings.paths), - }); -} - -async function attemptLocalDaemonStartup( - settings: DaemonClientSettings, - deadline: number, -): Promise<{ daemon: EnsuredDaemon } | FailedDaemonStartup> { - let launch: DaemonStartupLaunch; - const spawnedAtMs = Date.now(); - try { - launch = startDaemon(settings); - } catch (error) { - return await recoverFailedDaemonLaunch(settings, error); - } - const observation: StartupPhaseObservation = { spawnedAtMs, markers: {} }; - const startup = await waitForDaemonStartup(deadline, settings, launch, observation); - if (startup.kind === 'ready') return { daemon: startup.daemon }; - refreshStartupPhaseMarkers(settings, observation); - if (startup.kind === 'retry') return { retry: true, phases: observation }; - if (startup.kind === 'unproven') { - const startupError = normalizeError(startup.error); - return { - retry: false, - startError: startupError.message, - startupError, - daemonProcess: { pid: launch.pid }, - phases: observation, - }; - } - const { cleanup, joined } = await retireStartupAttempt(settings, launch, deadline); - const reason: DaemonStartupFailureReason = - startup.kind === 'stalled' - ? 'startup-stalled' - : startup.kind === 'ceiling' - ? 'startup-ceiling' - : 'startup-exited'; - const available = isRegistrationAvailable(inspectProcessLock(settings.paths.lockPath)); - return { - cleanup, - retry: joined && startup.kind === 'early_exit' && available, - reason, - startError: startup.kind === 'early_exit' ? describeDaemonEarlyExit(startup.exit) : undefined, - daemonProcess: startup.kind === 'early_exit' ? startup.exit : { pid: launch.pid }, - phases: observation, - }; -} - -async function recoverFailedDaemonLaunch( - settings: DaemonClientSettings, - error: unknown, -): Promise { - const cleanup = settings.ownedStateDir - ? await stopAndRetireDaemon({ - paths: settings.paths, - observed: null, - mode: 'graceful', - ownedStateDir: settings.ownedStateDir, - lockTimeoutMs: 0, - }) - : await recoverAbandonedDaemonRegistration({ - paths: settings.paths, - observed: null, - lockTimeoutMs: 0, - }); - return { - cleanup, - startError: normalizeError(error).message, - retry: !settings.ownedStateDir && cleanup.status !== 'retained', - }; -} - -async function retireStartupAttempt( - settings: DaemonClientSettings, - launch: DaemonStartupLaunch, - deadline: number, -): Promise<{ cleanup: DaemonRetirementResult; joined: boolean }> { - const cleanup = await stopAndRetireDaemon({ - paths: settings.paths, - observed: launch.readIdentity(), - mode: 'graceful', - ownedStateDir: settings.ownedStateDir, - termTimeoutMs: Math.min(3_000, remainingStartupBudget(deadline)), - killTimeoutMs: 1_000, - lockTimeoutMs: 0, - }); - const joined = await joinStartup(launch); - return { cleanup, joined }; -} - -async function joinStartup(launch: DaemonStartupLaunch): Promise { - let timer: ReturnType | undefined; - try { - return await Promise.race([ - launch.exited.then(() => true), - new Promise((resolve) => { - timer = setTimeout(() => resolve(false), 1_000); - }), - ]); - } finally { - clearTimeout(timer); - } -} - -function remainingStartupBudget(deadline?: number): number { - return deadline === undefined ? Number.POSITIVE_INFINITY : Math.max(0, deadline - Date.now()); -} - -function isRegistrationAvailable(inspection: ProcessLockInspection): boolean { - return ( - inspection.state === 'absent' || - (inspection.state === 'held' && - (inspection.liveness === 'owner-process-dead' || - inspection.liveness === 'owner-process-reused')) - ); -} - /** * ADR 0012 decision 6 (BLOCKER 2, third follow-up): a one-shot repair * (`replay --save-script`) that COMPLETES without diverging returns SUCCESS @@ -913,241 +435,6 @@ function isActiveReplaySessionResponse( return response.data?.sessionActive === true; } -async function waitForDaemonStartup( - deadline: number, - settings: DaemonClientSettings, - launch: DaemonStartupLaunch, - observation: StartupPhaseObservation, -): Promise { - let earlyExit: ExecDetachedExit | undefined; - void launch.exited - .then((exit) => { - earlyExit = exit; - }) - // The exit is observed on the next poll; a rejected monitor promise leaves the liveness and - // stall paths to retire the launch, so it must not surface as an unhandled rejection. - .catch(() => {}); - while (Date.now() < deadline) { - if (earlyExit) { - const settled = await judgeExitedDaemonStartup(settings, earlyExit, deadline); - if (settled) return settled; - } else { - const settled = await observeLiveDaemonStartup(settings, launch, deadline, observation); - // An exit observed while this iteration was in flight outranks a `ready` read: the process - // that published is gone, and its classification owns the next step. - if (settled && !(settled.kind === 'ready' && earlyExit)) return settled; - } - await sleep(Math.min(100, remainingStartupBudget(deadline))); - } - if (earlyExit || !isProcessAlive(launch.pid)) return { kind: 'timeout' }; - // A live daemon at the ceiling registered late, not never: only its retirement is this deadline's - // failure, and the reason says the host, not the daemon, ran out of room. - return { kind: 'ceiling' }; -} - -async function judgeExitedDaemonStartup( - settings: DaemonClientSettings, - exit: ExecDetachedExit, - deadline: number, -): Promise { - const kind = classifyDaemonStartupExit(exit); - if (kind === 'unproven') - return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings) }; - if (kind === 'failed') return { kind: 'early_exit', exit }; - return await observeContendingDaemon(settings, deadline); -} - -async function observeLiveDaemonStartup( - settings: DaemonClientSettings, - launch: DaemonStartupLaunch, - deadline: number, - observation: StartupPhaseObservation, -): Promise { - const info = await readReadyLaunchedDaemon(settings, launch, deadline); - if (info) return { kind: 'ready', daemon: { info, startedByClient: true } }; - // #3782: a live daemon that keeps publishing startup markers is slow, not stuck, and is - // waited for up to the ceiling. Only silence for a full stall window retires it early. - refreshStartupPhaseMarkers(settings, observation); - if ( - isProcessAlive(launch.pid) && - Date.now() - lastStartupProgressMs(observation) >= DAEMON_STARTUP_STALL_WINDOW_MS - ) { - return { kind: 'stalled' }; - } - return null; -} - -/** - * #3782: folds this launch's startup phase markers into `observation`. The record is append-only - * across startups, so markers older than this launch are ignored by timestamp, never by deletion. - */ -function refreshStartupPhaseMarkers( - settings: DaemonClientSettings, - observation: StartupPhaseObservation, -): void { - const tail = readRecentFileTail( - resolveDaemonStartupPhasesPath(settings.paths.baseDir), - DAEMON_STARTUP_MARKER_TAIL_BYTES, - ); - if (!tail) return; - for (const [phase, timestampMs] of Object.entries(readDaemonStartupPhaseMarkers(tail)) as Array< - [DaemonStartupPhase, number] - >) { - if (timestampMs >= observation.spawnedAtMs) observation.markers[phase] = timestampMs; - } -} - -function lastStartupProgressMs(observation: StartupPhaseObservation): number { - return Object.values(observation.markers).reduce( - (latest, timestampMs) => Math.max(latest, timestampMs), - observation.spawnedAtMs, - ); -} - -function describeStartupPhaseTimings( - observation: StartupPhaseObservation, -): Record & { spawned: 0 } { - const timings: Record = { spawned: 0 }; - for (const [phase, timestampMs] of Object.entries(observation.markers)) { - timings[phase] = Math.max(0, timestampMs - observation.spawnedAtMs); - } - return timings as Record & { spawned: 0 }; -} - -function resolveStartupProgressHint(reason: DaemonStartupFailureReason, ceilingMs: number): string { - if (reason === 'startup-stalled') { - return ( - `The daemon is alive but made no startup progress for ${DAEMON_STARTUP_STALL_WINDOW_MS / 1000} s. ` + - `The host can be overloaded. Retry, or raise ${DAEMON_STARTUP_TIMEOUT_ENV}.` - ); - } - return ( - `The daemon kept making startup progress but did not register within ${Math.round(ceilingMs / 1000)} s. ` + - `The host can be overloaded. Retry, or raise ${DAEMON_STARTUP_TIMEOUT_ENV}.` - ); -} - -function classifyDaemonStartupExit(exit: ExecDetachedExit): 'busy' | 'unproven' | 'failed' { - if (exit.error || exit.signal) return 'failed'; - switch (exit.exitCode) { - case DAEMON_STARTUP_EXIT_CODES.busy: - return 'busy'; - case DAEMON_STARTUP_EXIT_CODES.unproven: - return 'unproven'; - default: - return 'failed'; - } -} - -async function observeContendingDaemon( - settings: DaemonClientSettings, - deadline: number, -): Promise { - let winner: DaemonInfo | null; - try { - winner = await readReusableLocalDaemon(settings, { deadline, waitForLiveStartup: true }); - } catch (error) { - if (error instanceof AppError && error.details?.reason === 'daemon_registration_unproven') - return { kind: 'unproven', error }; - throw error; - } - if (Date.now() >= deadline) return null; - if (winner) return { kind: 'ready', daemon: { info: winner, startedByClient: false } }; - const inspection = inspectProcessLock(settings.paths.lockPath); - if (inspection.state === 'unproven') - return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings, inspection) }; - return isRegistrationAvailable(inspection) ? { kind: 'retry' } : null; -} - -async function readReadyLaunchedDaemon( - settings: DaemonClientSettings, - launch: DaemonStartupLaunch, - deadline: number, -): Promise { - const info = readDaemonInfo(settings.paths.infoPath); - if (!info || !isLaunchedDaemon(info, launch)) return null; - try { - return (await canConnect( - info, - settings.transportPreference, - remainingStartupBudget(deadline), - )) && Date.now() < deadline - ? info - : null; - } catch (error) { - const { cleanup, joined } = await retireStartupAttempt(settings, launch, deadline); - emitDiagnostic({ - level: 'warn', - phase: 'daemon_startup_observation_failed', - data: { stateDir: settings.paths.baseDir, cleanup, joined, error: normalizeError(error) }, - }); - if (error instanceof AppError) { - error.details = { - ...error.details, - stateDir: settings.paths.baseDir, - cleanupResults: [cleanup], - startupJoined: joined, - }; - } - throw error; - } -} - -/** Whether `info` names the daemon process this client launched: same pid and start time. */ -function isLaunchedDaemon(info: DaemonInfo, launch: DaemonStartupLaunch): boolean { - const identity = launch.readIdentity(); - return ( - info.pid === identity.pid && - identity.startTime !== null && - info.processStartTime === identity.startTime - ); -} - -function startDaemon(settings: DaemonClientSettings): DaemonStartupLaunch { - const launchSpec = resolveDaemonLaunchSpec(); - return launchDaemonProcess({ - paths: settings.paths, - serverMode: settings.serverMode, - env: settings.host.env, - cwd: settings.host.cwd, - ownedStateDir: settings.ownedStateDir, - args: launchSpec.useSrc - ? ['--experimental-strip-types', launchSpec.srcPath] - : [launchSpec.distPath], - }); -} - -function describeDaemonEarlyExit(exit: ExecDetachedExit): string { - if (exit.error) return `daemon process ${exit.pid} failed to start: ${exit.error}`; - if (exit.signal) - return `daemon process ${exit.pid} exited before readiness with signal ${exit.signal}`; - return `daemon process ${exit.pid} exited before readiness with code ${exit.exitCode ?? 0}`; -} - -function readRecentLogTail(logPath: string): string | undefined { - return readRecentFileTail(logPath, DAEMON_STARTUP_LOG_TAIL_BYTES); -} - -function readRecentFileTail(filePath: string, maxBytes: number): string | undefined { - try { - if (!fs.existsSync(filePath)) return undefined; - const stats = fs.statSync(filePath); - if (stats.size <= 0) return undefined; - const length = Math.min(stats.size, maxBytes); - const fd = fs.openSync(filePath, 'r'); - try { - const buffer = Buffer.alloc(length); - fs.readSync(fd, buffer, 0, length, stats.size - length); - const text = buffer.toString('utf8').trim(); - return text.length > 0 ? text : undefined; - } finally { - fs.closeSync(fd); - } - } catch { - return undefined; - } -} - function resolveRemoteDaemonBaseUrl(raw: string | undefined): string | undefined { if (!raw) return undefined; let parsed: URL; diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts new file mode 100644 index 0000000000..9e2a6b1ed8 --- /dev/null +++ b/src/daemon-client/daemon-client-startup.ts @@ -0,0 +1,741 @@ +// The local daemon this client starts and observes: reading what a state dir already registered, +// judging it against this client, and driving a launched daemon to readiness under the #3782 +// startup budget (ceiling, stall window, per-launch phase markers, typed failure reasons). +import fs from 'node:fs'; +import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; +import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; +import { processLockHoldsDaemonIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/host-kit/file'; +import { isProcessAlive } from '@agent-device/host-kit/process'; +import { sleep } from '@agent-device/host-kit/retry'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { type ExecDetachedExit } from '@agent-device/host-kit/command'; +import { + DAEMON_STARTUP_EXIT_CODES, + readDaemonStartupPhaseMarkers, + recoverAbandonedDaemonRegistration, + removeDaemonStartupPhasesFile, + resolveDaemonStartupPhasesPath, + type DaemonRetirementResult, + type DaemonStartupPhase, + launchDaemonProcess, + stopAndRetireDaemon, + type DaemonStartupLaunch, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import type { + DaemonPaths, + DaemonTransportPreference, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import { + resolveDaemonLaunchSpec, + resolveDaemonTakeover, + type DaemonTakeoverDecision, +} from './daemon-launch-spec.ts'; +import { + getDaemonMetadataState, + readDaemonInfo, + resolveDaemonStartupHint, + type DaemonInfo, +} from './daemon-client-metadata.ts'; +import { canConnect, isDaemonTransportUnavailableError } from './daemon-client-transport.ts'; +import { + daemonContendedError, + daemonRetirementUnconfirmedError, + daemonStopCommand, +} from './daemon-lifecycle-errors.ts'; +import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; + +/** Whether the daemon a state dir registered is reusable by this client, and what to do if not. */ +type LocalDaemonObservation = { + info: DaemonInfo; + decision: DaemonTakeoverDecision; +}; + +type DaemonStartupWaitResult = + | { kind: 'ready'; daemon: EnsuredDaemon } + | { kind: 'early_exit'; exit: ExecDetachedExit } + | { kind: 'unproven'; error: AppError } + | { kind: 'retry' | 'timeout' } + /** The process stayed alive but published no new startup phase within the stall window. */ + | { kind: 'stalled' } + /** The process stayed alive and kept progressing, but did not register before the ceiling. */ + | { kind: 'ceiling' }; + +/** Startup phase markers one launch published, as absolute host-clock timestamps. */ +type StartupPhaseObservation = { + spawnedAtMs: number; + markers: Partial>; +}; + +/** + * #3782: the ceiling on one local daemon startup, shared by the CLI, MCP, and Node client, because + * all three reach the daemon through `startLocalDaemon`. A slow host makes startup slower, not + * failed: while the spawned daemon stays alive and keeps publishing phase markers, the client keeps + * waiting. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides the ceiling for a caller that must + * fail sooner than the host allows. + */ +const DAEMON_STARTUP_CEILING_MS = 120_000; +/** + * How long a live daemon may publish no new startup phase marker before the client stops waiting. + * Kept beside the ceiling as one pair: a stalled daemon is retired at the stall window, a + * progressing one is waited for up to the ceiling. An env ceiling below this window makes the + * ceiling the only stop; the stall never fires past the ceiling, only earlier than it. + */ +const DAEMON_STARTUP_STALL_WINDOW_MS = 60_000; +const MINIMUM_DAEMON_TAKEOVER_BUDGET_MS = 5_000; + +const DAEMON_STARTUP_TIMEOUT_ENV = 'AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS'; + +/** Why a startup that never reached readiness failed, in terms a caller can branch on (#3782). */ +type DaemonStartupFailureReason = 'startup-exited' | 'startup-stalled' | 'startup-ceiling'; + +function resolveDaemonStartupTimeoutMs(env: Readonly): number { + const raw = env[DAEMON_STARTUP_TIMEOUT_ENV]?.trim(); + if (raw === undefined || raw === '') return DAEMON_STARTUP_CEILING_MS; + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new AppError( + 'INVALID_ARGS', + `${DAEMON_STARTUP_TIMEOUT_ENV} must be a positive integer number of milliseconds`, + { value: raw }, + ); + } + return parsed; +} + +const LIVE_DAEMON_PROBE_RETRIES = 3; +const LIVE_DAEMON_PROBE_RETRY_DELAY_MS = 200; +const DAEMON_STARTUP_ATTEMPTS = 2; +const DAEMON_STARTUP_LOG_TAIL_BYTES = 64_000; +const DAEMON_STARTUP_MARKER_TAIL_BYTES = 16_000; + +/** + * Reads the registered daemon and judges it against this client, changing nothing. Whether the + * judgement is acted on belongs to the caller's lifecycle: a managed client replaces, an attach + * client reports, `ensureDaemon` replaces only through a quiesce. + */ +export async function observeLocalDaemon( + settings: DaemonClientSettings, + options: { + deadline?: number; + waitForLiveStartup?: boolean; + /** Tell a live daemon on another transport from one that answers nowhere. */ + refineUnreachable?: boolean; + } = {}, +): Promise { + const { deadline } = options; + const inspection = inspectProcessLock(settings.paths.lockPath); + if (inspection.state === 'unproven') throw daemonRegistrationUnprovenError(settings, inspection); + const existing = readDaemonInfo(settings.paths.infoPath); + if (!existing) return null; + if (!registrationAllowsDaemonObservation(inspection, existing)) return null; + if (options.waitForLiveStartup && (await isAwaitingDaemonStartup(existing, deadline))) + return null; + + const decision = await resolveDaemonTakeover(existing, { + onClientTransport: () => + canReachReusableDaemon(existing, settings.transportPreference, deadline), + onAnyAdvertisedTransport: () => canReachReusableDaemon(existing, 'auto', deadline), + }); + return { info: existing, decision: await refineReplacement(existing, decision, options) }; +} + +/** + * Tells apart what the ladder lumps together, for callers that must not kill a daemon on a guess: a + * registration whose process is gone is unreachable whatever its version, and one that answers on + * another transport is alive. + */ +async function refineReplacement( + info: DaemonInfo, + decision: DaemonTakeoverDecision, + options: { deadline?: number; refineUnreachable?: boolean }, +): Promise { + if (!options.refineUnreachable || decision.kind !== 'replace') return decision; + if (!isProcessAlive(info.pid)) return { ...decision, cause: 'unreachable' }; + if ( + decision.cause === 'unreachable' && + (await canReachReusableDaemon(info, 'auto', options.deadline)) + ) { + return { ...decision, cause: 'transport' }; + } + return decision; +} + +export async function readReusableLocalDaemon( + settings: DaemonClientSettings, + options: { deadline?: number; waitForLiveStartup?: boolean } = {}, +): Promise { + const { deadline } = options; + const observed = await observeLocalDaemon(settings, options); + if (!observed) return null; + const { info: existing, decision } = observed; + if (decision.kind === 'reuse') return existing; + if (decision.kind === 'refuseNewer') { + throw newerDaemonRefusedError(existing, decision, settings.paths.baseDir); + } + + if (!settings.retireIncompatibleContender) throw daemonContendedError(); + if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; + emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); + await retireDaemonForTakeover(existing, settings.paths); + return null; +} + +async function isAwaitingDaemonStartup( + info: DaemonInfo, + deadline: number | undefined, +): Promise { + return ( + isProcessAlive(info.pid) && !(await canConnect(info, 'auto', remainingStartupBudget(deadline))) + ); +} + +function registrationAllowsDaemonObservation( + inspection: ProcessLockInspection, + info: DaemonInfo, +): boolean { + return ( + inspection.state === 'absent' || + processLockHoldsDaemonIdentity(inspection, { + pid: info.pid, + startTime: info.processStartTime ?? null, + }) + ); +} + +function daemonRegistrationUnprovenError( + settings: DaemonClientSettings, + inspection?: ProcessLockInspection, +): AppError { + return new AppError('COMMAND_FAILED', 'Daemon registration ownership could not be verified.', { + reason: 'daemon_registration_unproven', + inspection, + stateDir: settings.paths.baseDir, + hint: resolveDaemonStartupHint(getDaemonMetadataState(settings.paths), settings.paths), + }); +} + +export async function stopDaemonForTakeover( + existing: DaemonInfo, + paths: DaemonPaths, + mode: 'graceful' | 'force' = 'graceful', +): Promise { + return await stopAndRetireDaemon({ + paths: paths, + observed: { pid: existing.pid, startTime: existing.processStartTime ?? null }, + mode, + }); +} + +export async function retireDaemonForTakeover( + existing: DaemonInfo, + paths: DaemonPaths, + mode: 'graceful' | 'force' = 'graceful', +): Promise { + const retirement = await stopDaemonForTakeover(existing, paths, mode); + if (retirement.status === 'retained') throw daemonRetirementUnconfirmedError(retirement, paths); +} + +/** + * A daemon whose pid is still alive is probed again before it can be judged unreachable. A probe's + * budget is wall-clock time on this client's event loop, so a client that stalls past it (a large + * synchronous parse, a GC pause on a loaded host) reads a listening daemon as unreachable, and + * replacing it ends every session the daemon holds. Liveness is the signal-0 check, not the `ps` + * identity read: under the load that stalls the probe, `ps` misses its deadline too, and the + * takeover still proves identity before it signals anything. + */ +async function canReachReusableDaemon( + info: DaemonInfo, + preference: DaemonTransportPreference, + deadline?: number, +): Promise { + if (await canConnectReusableDaemon(info, preference, deadline)) return true; + for (let retry = 1; retry <= LIVE_DAEMON_PROBE_RETRIES; retry += 1) { + if (!isProcessAlive(info.pid) || (deadline !== undefined && Date.now() >= deadline)) + return false; + await sleep(Math.min(LIVE_DAEMON_PROBE_RETRY_DELAY_MS, remainingStartupBudget(deadline))); + if (await canConnectReusableDaemon(info, preference, deadline)) { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_probe_recovered', + data: { pid: info.pid, retry }, + }); + return true; + } + } + return false; +} + +/** + * ADR 0029: a caller that names a daemon policy must not silently use a daemon that enforces a + * different one (or none). A caller that names no policy uses whatever the daemon enforces. + */ +export async function assertDaemonPolicyMatches( + existing: DaemonInfo, + stateDir: string, + host: HostEnvironment, +): Promise { + const { env } = host; + if (!env.AGENT_DEVICE_DAEMON_POLICY?.trim()) return; + const { loadDaemonPolicy } = await import('@agent-device/daemon-contracts/daemon-policy-file'); + const expected = loadDaemonPolicy(env, host.cwd)?.digest; + if (expected === existing.policyDigest) return; + throw new AppError( + 'COMMAND_FAILED', + 'The running daemon does not enforce the daemon policy named by AGENT_DEVICE_DAEMON_POLICY.', + { + reason: 'DAEMON_POLICY_MISMATCH', + expectedPolicyDigest: expected, + daemonPolicyDigest: existing.policyDigest ?? null, + hint: `Stop the running daemon (${daemonStopCommand(stateDir)}), then retry so a daemon starts with this policy.`, + }, + ); +} + +async function canConnectReusableDaemon( + info: DaemonInfo, + preference: DaemonTransportPreference, + deadline?: number, +): Promise { + try { + return await canConnect(info, preference, remainingStartupBudget(deadline)); + } catch (error) { + if (isDaemonTransportUnavailableError(error)) return false; + throw error; + } +} + +function newerDaemonRefusedError( + info: DaemonInfo, + decision: Extract, + stateDir: string, +): AppError { + const { daemonVersion, clientVersion } = decision; + return new AppError( + 'COMMAND_FAILED', + `Daemon (pid ${info.pid}, v${daemonVersion}) is newer than this client (v${clientVersion}); refusing to replace it.`, + { + daemonPid: info.pid, + daemonVersion, + clientVersion, + hint: `Use the agent-device v${daemonVersion} CLI that started it, or stop it deliberately: ${daemonStopCommand(stateDir)}`, + }, + ); +} + +function emitDaemonTakeoverNotice(info: DaemonInfo, reason: string, stateDir: string): void { + try { + const identity = info.version ? `pid ${info.pid}, v${info.version}` : `pid ${info.pid}`; + process.stderr.write(`Replacing daemon (${identity}) in ${stateDir}: ${reason}\n`); + } catch { + // The takeover notice is best effort; never fail the command on stderr issues. + } +} + +type FailedDaemonStartup = { + cleanup?: DaemonRetirementResult; + startError?: string; + startupError?: NormalizedError; + daemonProcess?: ExecDetachedExit | { pid: number }; + retry: boolean; + /** Set when the wait ended on startup progress rather than on an exit or a race. */ + reason?: DaemonStartupFailureReason; + phases?: StartupPhaseObservation; +}; + +export async function startLocalDaemon(settings: DaemonClientSettings): Promise { + const startupTimeoutMs = resolveDaemonStartupTimeoutMs(settings.host.env); + const deadline = Date.now() + startupTimeoutMs; + const cleanupResults: DaemonRetirementResult[] = []; + let failure: FailedDaemonStartup | undefined; + let attempts = 0; + while (attempts < DAEMON_STARTUP_ATTEMPTS && Date.now() < deadline) { + attempts += 1; + const result = await attemptLocalDaemonStartup(settings, deadline); + if ('daemon' in result) return result.daemon; + failure = result; + if (result.cleanup) cleanupResults.push(result.cleanup); + if (!result.retry) break; + await sleep(Math.min(150, Math.max(0, deadline - Date.now()))); + } + const { startError, startupError, daemonProcess, reason, phases }: Partial = + failure ?? {}; + const effectiveReason: DaemonStartupFailureReason = reason ?? 'startup-exited'; + const state = getDaemonMetadataState(settings.paths); + const daemonLogTail = readRecentLogTail(settings.paths.logPath); + throw new AppError('COMMAND_FAILED', 'Failed to start daemon', { + kind: 'daemon_startup_failed', + reason: effectiveReason, + stateDir: settings.paths.baseDir, + infoPath: settings.paths.infoPath, + lockPath: settings.paths.lockPath, + logPath: settings.paths.logPath, + startupTimeoutMs, + startupAttempts: attempts, + cleanupResults, + startError, + startupError, + daemonProcess, + ...(phases ? { phases: describeStartupPhaseTimings(phases) } : {}), + ...(daemonLogTail ? { daemonLogTail } : {}), + metadataState: state, + hint: + reason === 'startup-stalled' || reason === 'startup-ceiling' + ? resolveStartupProgressHint(reason, startupTimeoutMs) + : resolveDaemonStartupHint(state, settings.paths), + }); +} + +async function attemptLocalDaemonStartup( + settings: DaemonClientSettings, + deadline: number, +): Promise<{ daemon: EnsuredDaemon } | FailedDaemonStartup> { + let launch: DaemonStartupLaunch; + const spawnedAtMs = Date.now(); + try { + launch = startDaemon(settings); + } catch (error) { + return await recoverFailedDaemonLaunch(settings, error); + } + const observation: StartupPhaseObservation = { spawnedAtMs, markers: {} }; + const startup = await waitForDaemonStartup(deadline, settings, launch, observation); + if (startup.kind === 'ready') return { daemon: startup.daemon }; + refreshStartupPhaseMarkers(settings, launch.pid, observation); + if (startup.kind === 'retry') return { retry: true, phases: observation }; + if (startup.kind === 'unproven') { + const startupError = normalizeError(startup.error); + return { + retry: false, + startError: startupError.message, + startupError, + daemonProcess: { pid: launch.pid }, + phases: observation, + }; + } + const { cleanup, joined } = await retireStartupAttempt(settings, launch, deadline); + const reason: DaemonStartupFailureReason = + startup.kind === 'stalled' + ? 'startup-stalled' + : startup.kind === 'ceiling' + ? 'startup-ceiling' + : 'startup-exited'; + const available = isRegistrationAvailable(inspectProcessLock(settings.paths.lockPath)); + return { + cleanup, + retry: joined && startup.kind === 'early_exit' && available, + reason, + startError: startup.kind === 'early_exit' ? describeDaemonEarlyExit(startup.exit) : undefined, + daemonProcess: startup.kind === 'early_exit' ? startup.exit : { pid: launch.pid }, + phases: observation, + }; +} + +async function recoverFailedDaemonLaunch( + settings: DaemonClientSettings, + error: unknown, +): Promise { + const cleanup = settings.ownedStateDir + ? await stopAndRetireDaemon({ + paths: settings.paths, + observed: null, + mode: 'graceful', + ownedStateDir: settings.ownedStateDir, + lockTimeoutMs: 0, + }) + : await recoverAbandonedDaemonRegistration({ + paths: settings.paths, + observed: null, + lockTimeoutMs: 0, + }); + return { + cleanup, + startError: normalizeError(error).message, + retry: !settings.ownedStateDir && cleanup.status !== 'retained', + }; +} + +async function retireStartupAttempt( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + deadline: number, +): Promise<{ cleanup: DaemonRetirementResult; joined: boolean }> { + const cleanup = await stopAndRetireDaemon({ + paths: settings.paths, + observed: launch.readIdentity(), + mode: 'graceful', + ownedStateDir: settings.ownedStateDir, + termTimeoutMs: Math.min(3_000, remainingStartupBudget(deadline)), + killTimeoutMs: 1_000, + lockTimeoutMs: 0, + }); + const joined = await joinStartup(launch); + // The launch's exit is joined, so nothing can append to its marker file anymore, and the file + // has served its last purpose (the failure record carries the phases). Removing it here is what + // keeps a retired startup's evidence from becoming state-dir residue. Without a joined exit the + // file stays: its writer may still be alive, and only that writer or a later joined exit owns it. + if (joined) + removeDaemonStartupPhasesFile( + resolveDaemonStartupPhasesPath(settings.paths.baseDir, launch.pid), + ); + return { cleanup, joined }; +} + +async function joinStartup(launch: DaemonStartupLaunch): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + launch.exited.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), 1_000); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +function remainingStartupBudget(deadline?: number): number { + return deadline === undefined ? Number.POSITIVE_INFINITY : Math.max(0, deadline - Date.now()); +} + +function isRegistrationAvailable(inspection: ProcessLockInspection): boolean { + return ( + inspection.state === 'absent' || + (inspection.state === 'held' && + (inspection.liveness === 'owner-process-dead' || + inspection.liveness === 'owner-process-reused')) + ); +} + +async function waitForDaemonStartup( + deadline: number, + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + observation: StartupPhaseObservation, +): Promise { + let earlyExit: ExecDetachedExit | undefined; + void launch.exited.then((exit) => { + earlyExit = exit; + }); + while (Date.now() < deadline) { + if (earlyExit) { + const settled = await judgeExitedDaemonStartup(settings, earlyExit, deadline); + if (settled) return settled; + } else { + const settled = await observeLiveDaemonStartup(settings, launch, deadline, observation); + // An exit observed while this iteration was in flight outranks a `ready` read: the process + // that published is gone, and its classification owns the next step. + if (settled && !(settled.kind === 'ready' && earlyExit)) return settled; + } + await sleep(Math.min(100, remainingStartupBudget(deadline))); + } + if (earlyExit || !isProcessAlive(launch.pid)) return { kind: 'timeout' }; + // A live daemon at the ceiling registered late, not never: only its retirement is this deadline's + // failure, and the reason says the host, not the daemon, ran out of room. + return { kind: 'ceiling' }; +} + +async function judgeExitedDaemonStartup( + settings: DaemonClientSettings, + exit: ExecDetachedExit, + deadline: number, +): Promise { + const kind = classifyDaemonStartupExit(exit); + if (kind === 'unproven') + return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings) }; + if (kind === 'failed') return { kind: 'early_exit', exit }; + return await observeContendingDaemon(settings, deadline); +} + +async function observeLiveDaemonStartup( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + deadline: number, + observation: StartupPhaseObservation, +): Promise { + const info = await readReadyLaunchedDaemon(settings, launch, deadline); + if (info) return { kind: 'ready', daemon: { info, startedByClient: true } }; + // #3782: a live daemon that keeps publishing startup markers is slow, not stuck, and is + // waited for up to the ceiling. Only silence for a full stall window retires it early. + refreshStartupPhaseMarkers(settings, launch.pid, observation); + if ( + isProcessAlive(launch.pid) && + Date.now() - lastStartupProgressMs(observation) >= DAEMON_STARTUP_STALL_WINDOW_MS + ) { + return { kind: 'stalled' }; + } + return null; +} + +/** + * #3782: folds this launch's startup phase markers into `observation`. The record is one launch's + * own file, named for the pid that wrote it, so no other launch's evidence can add to or hide it. + */ +function refreshStartupPhaseMarkers( + settings: DaemonClientSettings, + launchPid: number, + observation: StartupPhaseObservation, +): void { + const tail = readRecentFileTail( + resolveDaemonStartupPhasesPath(settings.paths.baseDir, launchPid), + DAEMON_STARTUP_MARKER_TAIL_BYTES, + ); + if (!tail) return; + for (const [phase, timestampMs] of Object.entries(readDaemonStartupPhaseMarkers(tail)) as Array< + [DaemonStartupPhase, number] + >) { + observation.markers[phase] = timestampMs; + } +} + +function lastStartupProgressMs(observation: StartupPhaseObservation): number { + return Object.values(observation.markers).reduce( + (latest, timestampMs) => Math.max(latest, timestampMs), + observation.spawnedAtMs, + ); +} + +function describeStartupPhaseTimings( + observation: StartupPhaseObservation, +): Record & { spawned: 0 } { + const timings: Record = { spawned: 0 }; + for (const [phase, timestampMs] of Object.entries(observation.markers)) { + timings[phase] = Math.max(0, timestampMs - observation.spawnedAtMs); + } + return timings as Record & { spawned: 0 }; +} + +function resolveStartupProgressHint(reason: DaemonStartupFailureReason, ceilingMs: number): string { + if (reason === 'startup-stalled') { + return ( + `The daemon is alive but made no startup progress for ${DAEMON_STARTUP_STALL_WINDOW_MS / 1000} s. ` + + `The host can be overloaded. Retry once the host load drops; ${DAEMON_STARTUP_TIMEOUT_ENV} ` + + `raises only the registration ceiling and does not change this stall window.` + ); + } + return ( + `The daemon is alive but did not register within ${Math.round(ceilingMs / 1000)} s. ` + + `The host can be overloaded. Retry, or raise ${DAEMON_STARTUP_TIMEOUT_ENV}.` + ); +} + +function classifyDaemonStartupExit(exit: ExecDetachedExit): 'busy' | 'unproven' | 'failed' { + if (exit.error || exit.signal) return 'failed'; + switch (exit.exitCode) { + case DAEMON_STARTUP_EXIT_CODES.busy: + return 'busy'; + case DAEMON_STARTUP_EXIT_CODES.unproven: + return 'unproven'; + default: + return 'failed'; + } +} + +async function observeContendingDaemon( + settings: DaemonClientSettings, + deadline: number, +): Promise { + let winner: DaemonInfo | null; + try { + winner = await readReusableLocalDaemon(settings, { deadline, waitForLiveStartup: true }); + } catch (error) { + if (error instanceof AppError && error.details?.reason === 'daemon_registration_unproven') + return { kind: 'unproven', error }; + throw error; + } + if (Date.now() >= deadline) return null; + if (winner) return { kind: 'ready', daemon: { info: winner, startedByClient: false } }; + const inspection = inspectProcessLock(settings.paths.lockPath); + if (inspection.state === 'unproven') + return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings, inspection) }; + return isRegistrationAvailable(inspection) ? { kind: 'retry' } : null; +} + +async function readReadyLaunchedDaemon( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + deadline: number, +): Promise { + const info = readDaemonInfo(settings.paths.infoPath); + if (!info || !isLaunchedDaemon(info, launch)) return null; + try { + return (await canConnect( + info, + settings.transportPreference, + remainingStartupBudget(deadline), + )) && Date.now() < deadline + ? info + : null; + } catch (error) { + const { cleanup, joined } = await retireStartupAttempt(settings, launch, deadline); + emitDiagnostic({ + level: 'warn', + phase: 'daemon_startup_observation_failed', + data: { stateDir: settings.paths.baseDir, cleanup, joined, error: normalizeError(error) }, + }); + if (error instanceof AppError) { + error.details = { + ...error.details, + stateDir: settings.paths.baseDir, + cleanupResults: [cleanup], + startupJoined: joined, + }; + } + throw error; + } +} + +/** Whether `info` names the daemon process this client launched: same pid and start time. */ +function isLaunchedDaemon(info: DaemonInfo, launch: DaemonStartupLaunch): boolean { + const identity = launch.readIdentity(); + return ( + info.pid === identity.pid && + identity.startTime !== null && + info.processStartTime === identity.startTime + ); +} + +function startDaemon(settings: DaemonClientSettings): DaemonStartupLaunch { + const launchSpec = resolveDaemonLaunchSpec(); + return launchDaemonProcess({ + paths: settings.paths, + serverMode: settings.serverMode, + env: settings.host.env, + cwd: settings.host.cwd, + ownedStateDir: settings.ownedStateDir, + args: launchSpec.useSrc + ? ['--experimental-strip-types', launchSpec.srcPath] + : [launchSpec.distPath], + }); +} + +function describeDaemonEarlyExit(exit: ExecDetachedExit): string { + if (exit.error) return `daemon process ${exit.pid} failed to start: ${exit.error}`; + if (exit.signal) + return `daemon process ${exit.pid} exited before readiness with signal ${exit.signal}`; + return `daemon process ${exit.pid} exited before readiness with code ${exit.exitCode ?? 0}`; +} + +function readRecentLogTail(logPath: string): string | undefined { + return readRecentFileTail(logPath, DAEMON_STARTUP_LOG_TAIL_BYTES); +} + +function readRecentFileTail(filePath: string, maxBytes: number): string | undefined { + try { + if (!fs.existsSync(filePath)) return undefined; + const stats = fs.statSync(filePath); + if (stats.size <= 0) return undefined; + const length = Math.min(stats.size, maxBytes); + const fd = fs.openSync(filePath, 'r'); + try { + const buffer = Buffer.alloc(length); + fs.readSync(fd, buffer, 0, length, stats.size - length); + const text = buffer.toString('utf8').trim(); + return text.length > 0 ? text : undefined; + } finally { + fs.closeSync(fd); + } + } catch { + return undefined; + } +} diff --git a/src/daemon.ts b/src/daemon.ts index 66972b90a6..c9f958b7ab 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,4 +1,4 @@ -import { asAppError } from '@agent-device/kernel/errors'; +import { asAppError, normalizeError } from '@agent-device/kernel/errors'; import { daemonStartupExitCode, reportDaemonClose, @@ -11,23 +11,44 @@ import { } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +// Marker failures go to stderr, which the launcher redirects into the daemon log — the same tail a +// startup failure already ships as evidence. A marker is evidence, not a requirement, so a refused +// append never stops a startup that can still register; it must only stay observable. +function reportMarkerFailure(error: unknown): void { + process.stderr.write(`Daemon startup marker write failed: ${normalizeError(error).message}\n`); +} + // #3782: the first marker the waiting client can read. It says this entry's module graph evaluated, // which on an overloaded host is the slowest thing a daemon start does before it can prove itself. +// The file is named for this process's pid, so the client attributes every line to this launch. recordDaemonStartupMarker( - resolveDaemonStartupPhasesPath(resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir), + resolveDaemonStartupPhasesPath( + resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, + process.pid, + ), 'modules-loaded', + reportMarkerFailure, ); // The executable owns the process: signals, fatal-error handlers and the exit code. The runtime // only reports when it has closed, whoever closed it. // The root's composition enters as a factory behind a function-scoped `await import`: the server // calls it before it acquires the registration, so startup order is unchanged, and the eager -// closure of this entry stays exactly the operational server's (ADR 0019). +// closure of this entry stays exactly the operational server's (ADR 0019). The factory records +// `composition-loaded` when that import and dep build finish: the second module-graph edge this +// startup crosses, and a real completed boundary the waiting client can attribute progress to. async function runDaemon(): Promise { const runtime = await startDaemonRuntime({ host: captureHostEnvironment(process), - composeDeps: async (input) => - (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), + composeDeps: async (input) => { + const deps = await (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input); + recordDaemonStartupMarker( + resolveDaemonStartupPhasesPath(input.baseDir, process.pid), + 'composition-loaded', + reportMarkerFailure, + ); + return deps; + }, }); let exitCode = 0; const closeAndExit = (code: number, cause?: unknown): void => { From c629be20aeebeb99d6de848e445ba787d1b5d20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 23:23:06 +0200 Subject: [PATCH 03/22] fix(help): state the startup failure reasons the code actually produces The documented reasons overclaimed. `startup-ceiling` said "progressing, not registered before the ceiling", but the ceiling is reached by a daemon that is merely alive: a progressing one is only ever stopped by a ceiling above the stall window, so the promise was unverifiable from the client's evidence. `startup-stalled` named a 60 s window without saying that `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` does not change it, so raising the ceiling looked like a fix for silence. Both the CLI help and the configuration table now describe the two reasons as what they are, and the env var is specified as a positive integer number of milliseconds. Refs #3791 --- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/configuration.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 65ce3e4fe6..e1a307286a 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -512,7 +512,7 @@ Runner and daemon lifecycle (applies to simulators too): No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. - A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive but no startup progress for 60 s), or startup-ceiling (progressing, not registered before the ceiling). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. + A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window, so a ceiling set below 60 s also ends silent startups this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index ea2128ca76..15a887bd64 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -122,7 +122,7 @@ Only the variables below, and those documented on command pages, are supported. | Category | Env vars | Decision | | --- | --- | --- | -| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (whole-integer milliseconds, default `120000`) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for 60 s), or `startup-ceiling` (progressing, not registered before the ceiling). Branch on `details.reason`, never on the message. | +| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (a positive integer number of milliseconds, default `120000`; any other value fails with `INVALID_ARGS`) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for a fixed 60 s window this variable does not change), or `startup-ceiling` (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window). Branch on `details.reason`, never on the message. | | Device scoping | `AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST`, `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` | Public. `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` is read by the daemon: every iOS simulator request that names no `--ios-simulator-device-set` uses that set, so iOS physical devices are not listed, and a request naming a different path string fails with `INVALID_ARGS` (`details.reason: "ios_simulator_device_set_outside_daemon_set"`). macOS and non-iOS requests ignore it. A daemon started by the CLI inherits the CLI's value; restart the daemon after changing it. | | Android test IME | `AGENT_DEVICE_TEST_IME` | Public. Same setting as `--test-ime` / `--no-test-ime` on `open`, `test`, and `replay`; see known limitations. | | Local daemon storage | `AGENT_DEVICE_STATE_DIR` | Public | From b6731154be1bb9bb6bee5054050e5284aa82fd3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 23:46:36 +0200 Subject: [PATCH 04/22] docs(daemon): remove narrating module headers The lifecycle and startup module headers only narrate each file's role, which AGENTS.md reserves for public APIs, tool directives, and constraints code cannot show. The split is owned by the imports and the file names. Fixes review thread PRRT_kwDORFGHaM6tkpef on #3791. --- src/daemon-client/daemon-client-lifecycle.ts | 4 ---- src/daemon-client/daemon-client-startup.ts | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 9d433fdf85..a30fd19bcc 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -1,7 +1,3 @@ -// The daemon a request reaches: which settings resolve to which daemon (local, remote, or a -// private one-shot directory), the managed ensure that defers to the startup module, and the -// per-request cleanup that owns one-shot replay lifetimes. Starting and judging a daemon lives in -// `daemon-client-startup.ts`. import type { DaemonWireRequest } from '@agent-device/contracts/command'; import type { DaemonLifecycle, EnsureDaemonOptions } from '@agent-device/contracts/client'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index 9e2a6b1ed8..a5e34c4644 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -1,6 +1,3 @@ -// The local daemon this client starts and observes: reading what a state dir already registered, -// judging it against this client, and driving a launched daemon to readiness under the #3782 -// startup budget (ceiling, stall window, per-launch phase markers, typed failure reasons). import fs from 'node:fs'; import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; From d57a0ab32a26887e3e1b97af49ae57e1218c7f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:03:09 +0200 Subject: [PATCH 05/22] test(daemon): scope fixture marker emission to the child that asked for it The shared emit-startup-markers flag file applied to every later fixture in the same state dir, so the stale-history cleaner inherited the request, wrote its own live marker, and the sweep assertion raced it. The request is now a typed per-child option bound into that child's entry script, and readiness waits use a per-pid file so a stale registration-held from an earlier child cannot shortcut the await. Fixes review thread PRRT_kwDORFGHaM6tkpeV on #3791. --- .../test-utils/registered-daemon-fixture.ts | 20 +++++++++--- .../daemon-client-startup-budget.test.ts | 31 ++++++++++++------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index 250a91032d..73b6f09846 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -17,11 +17,22 @@ const children = new Map< Array<{ launch: ReturnType; startTime: string | null }> >(); +/** Per-child behavior switches, bound into this child's entry script at write time. */ +export type RegisteredDaemonFixtureOptions = Readonly<{ + /** + * Record `lock-acquired` through the real protocol after this child acquires the registration, + * as the daemon runtime does. Scoped to this child: no later fixture in the same state dir + * inherits it. + */ + emitStartupMarker?: boolean; +}>; + /** A real registration owner, advertising the caller's HTTP fixture and joining before deletion. */ export function registeredDaemonFixtureArgs( paths: DaemonPaths, fields: DaemonRegistrationFields, acquisitionBarrier?: string, + fixtureOptions?: RegisteredDaemonFixtureOptions, ): string[] { const entry = path.join(paths.baseDir, 'dist', 'src', 'internal', 'daemon.js'); fs.mkdirSync(path.dirname(entry), { recursive: true }); @@ -37,15 +48,14 @@ import path from 'node:path'; import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; const paths = ${JSON.stringify(paths)}; const barrier = ${JSON.stringify(acquisitionBarrier)}; +const emitAfter = ${fixtureOptions?.emitStartupMarker ? 'true' : 'false'}; if (barrier) { fs.writeFileSync(barrier + '.ready-' + process.pid, 'ready'); while (!fs.existsSync(barrier)) await new Promise(resolve => setTimeout(resolve, 10)); } const acquired = await tryAcquireDaemonRegistration(paths); if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); -// A suite that wants this child to behave like the real entry's marker producer drops this flag -// file before spawning; the marker is written through the real protocol, to this pid's own file. -if (fs.existsSync(path.join(paths.baseDir, 'emit-startup-markers'))) +if (emitAfter) recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(paths.baseDir, process.pid), 'lock-acquired'); process.on('SIGTERM', async () => { if (fs.existsSync(path.join(paths.baseDir, 'ignore-sigterm'))) return; @@ -59,6 +69,7 @@ process.on('SIGTERM', async () => { process.exit(0); }); fs.writeFileSync(path.join(paths.baseDir, 'registration-held'), 'ready'); +fs.writeFileSync(path.join(paths.baseDir, 'registration-held-' + process.pid), 'ready'); while (fs.existsSync(path.join(paths.baseDir, 'defer-publication'))) await new Promise(resolve => setTimeout(resolve, 10)); acquired.owner.publish(${JSON.stringify(fields)}); setInterval(() => {}, 1000); @@ -72,10 +83,11 @@ export function spawnRegisteredDaemonFixture( fields: DaemonRegistrationFields, options: Parameters[2], acquisitionBarrier?: string, + fixtureOptions?: RegisteredDaemonFixtureOptions, ): ReturnType { const child = actualCommand.runCmdDetachedMonitored( process.execPath, - registeredDaemonFixtureArgs(paths, fields, acquisitionBarrier), + registeredDaemonFixtureArgs(paths, fields, acquisitionBarrier, fixtureOptions), options, ); const owned = children.get(paths.baseDir) ?? []; diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 8834f897a5..fc2e851830 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -88,14 +88,18 @@ function fields(httpPort: number) { function spawnUnregisteredDaemonChild( paths: DaemonPaths, options: Parameters[2], - emitStartupMarkers = false, + emitStartupMarker = false, ): ReturnType { fs.mkdirSync(paths.baseDir, { recursive: true }); fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); // When asked, the child records its phases through the real protocol, into the file named for - // its own pid, exactly as the daemon entry does. - if (emitStartupMarkers) fs.writeFileSync(path.join(paths.baseDir, 'emit-startup-markers'), 'on'); - return spawnRegisteredDaemonFixture(paths, fields(1), options); + // its own pid, exactly as the daemon entry does. The request is bound into this child's entry + // script, so no later fixture in this state dir inherits it. + return spawnRegisteredDaemonFixture(paths, fields(1), options, undefined, { emitStartupMarker }); +} + +function registrationHeldFor(paths: DaemonPaths, pid: number): string { + return path.join(paths.baseDir, `registration-held-${pid}`); } async function expectStartupFailure(pending: Promise): Promise { @@ -327,9 +331,10 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier // The leftover launch: a real daemon-shaped child that records its phase, is SIGTERMed before // publishing, and leaves its record behind exactly as a crashed startup would. fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); - fs.writeFileSync(path.join(paths.baseDir, 'emit-startup-markers'), 'on'); - const leftover = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }); - await awaitFile(path.join(paths.baseDir, 'registration-held')); + const leftover = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }, undefined, { + emitStartupMarker: true, + }); + await awaitFile(registrationHeldFor(paths, leftover.pid)); process.kill(leftover.pid, 'SIGTERM'); await leftover.exited; const leftoverRecords = () => @@ -344,14 +349,13 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier vi.spyOn(Date, 'now').mockImplementation(() => now.value); let child: ReturnType | undefined; let booted = false; - fs.rmSync(path.join(paths.baseDir, 'registration-held'), { force: true }); spawn.mockImplementation((_command, _args, options) => { child = spawnUnregisteredDaemonChild(paths, options, true); return child; }); pause.mockImplementation(async () => { - if (!booted) { - booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); + if (child && !booted) { + booted = fs.existsSync(registrationHeldFor(paths, child.pid)); if (!booted) return await realPause(); } // The 30 s env ceiling stops this wait before the 60 s stall window can; the fresh child's own @@ -373,10 +377,13 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier !leftoverRecords().some((entry) => entry.includes(String(child?.pid))), "the joined exit must sweep this launch's own marker file", ); - // A new registration owner sweeps the leftover record of a provably dead launch. + // A dead launch's record that survived every sweep so far: written through the real protocol + // against a provably dead pid after the fresh launch's own acquisition sweep, so only the next + // registration owner can remove it. + recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(paths.baseDir, 998_776), 'listening'); const cleaner = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }); try { - await awaitFile(path.join(paths.baseDir, 'registration-held')); + await awaitFile(registrationHeldFor(paths, cleaner.pid)); assert.deepEqual( leftoverRecords(), [], From 6ba4639c469b7ebc138e7e4c276a49f80d5c40de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:03:27 +0200 Subject: [PATCH 06/22] fix(daemon): sweep an exited contender's own record on every terminal exit A launch that exited busy, unproven, or lost the registration to a contending winner returned its retry, error, or winner handoff without joining its exact exited child, so the child's pid-named phase file sat in the state dir until some later registration sweep. Every terminal child outcome now joins the child first and removes only that child's own file, the same ownership rule a retired startup follows; the current winner's registration and record are untouched. Regression tests write markers at the real pre-acquisition producer seam (new emitStartupMarkerBeforeAcquisition fixture option) and pin the residue rule on the retry, winner-attach, and unproven-exit paths: red at the prior head, green with the fix. Fixes review thread PRRT_kwDORFGHaM6tkpec on #3791. --- .../test-utils/registered-daemon-fixture.ts | 8 ++ .../daemon-client-startup-race.test.ts | 101 ++++++++++++++++++ src/daemon-client/daemon-client-startup.ts | 31 +++++- 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index 73b6f09846..f808133a73 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -25,6 +25,11 @@ export type RegisteredDaemonFixtureOptions = Readonly<{ * inherits it. */ emitStartupMarker?: boolean; + /** + * Record `modules-loaded` through the real protocol before acquisition attempts, as the daemon + * entry does, so a child that exits busy or unproven leaves a real pre-lock marker behind. + */ + emitStartupMarkerBeforeAcquisition?: boolean; }>; /** A real registration owner, advertising the caller's HTTP fixture and joining before deletion. */ @@ -48,11 +53,14 @@ import path from 'node:path'; import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; const paths = ${JSON.stringify(paths)}; const barrier = ${JSON.stringify(acquisitionBarrier)}; +const emitBefore = ${fixtureOptions?.emitStartupMarkerBeforeAcquisition ? 'true' : 'false'}; const emitAfter = ${fixtureOptions?.emitStartupMarker ? 'true' : 'false'}; if (barrier) { fs.writeFileSync(barrier + '.ready-' + process.pid, 'ready'); while (!fs.existsSync(barrier)) await new Promise(resolve => setTimeout(resolve, 10)); } +if (emitBefore) + recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(paths.baseDir, process.pid), 'modules-loaded'); const acquired = await tryAcquireDaemonRegistration(paths); if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); if (emitAfter) diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index 5871945144..b0f6727c70 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -385,6 +385,107 @@ for (const exit of [ }); } +test('a busy contender that exits sweeps its own pre-lock marker before the retried attempt', async (t) => { + // #3782 residue rule on the retry path: the real daemon entry records `modules-loaded` before it + // acquires the registration, so a contender that exits busy has a record of its own. The client + // that launched it joins the exact exited child and removes only that child's file at the retry + // boundary; the second launch here is refused at the spawn seam, so nothing else can sweep the + // state dir and the assertion below can only pass through the client's own cleanup. + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-busy-retry-marker-')); + const http = await startHttpDaemonFixture({ devices: [] }); + const claim = tryAcquireProcessLock({ + lockDirPath: paths.lockPath, + owner: { ...readCurrentOwnerIdentity(), acquiredAtMs: Date.now() }, + }); + assert.equal(claim.status, 'acquired'); + if (claim.status !== 'acquired') throw new Error('fixture claim refused'); + let released = false; + let contenderPid = 0; + spawn.mockImplementation((_command, _args, options) => { + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options, undefined, { + emitStartupMarkerBeforeAcquisition: true, + }); + if (spawn.mock.calls.length === 1) { + contenderPid = child.pid; + void child.exited.then((exit) => { + assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + if (!released) { + // The lock only frees once the real contender has exited busy, so this child always + // loses the registration and always leaves its pre-lock record behind. + void claim.acquisition.release().then(() => { + released = true; + }); + } + }); + return child; + } + throw new Error('no second launch: the retry boundary must be observable without a sweeper'); + }); + pause.mockImplementation((ms) => actualRetry.sleep(Math.min(Number(ms), 50))); + try { + await assert.rejects( + sendToDaemon(request(paths)), + (error: unknown) => + error instanceof AppError && error.details?.kind === 'daemon_startup_failed', + ); + assert.ok(contenderPid > 0); + assert.deepEqual( + fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + [], + "the retry result must not leave the joined busy contender's record behind", + ); + assert.equal(released, true, 'the contender must have exited busy against the held claim'); + } finally { + if (!released) await claim.acquisition.release(); + await closeLoopbackServer(http.server); + } +}); + +test('attaching to a contending winner sweeps the busy contender it exited as', async (t) => { + // #3782 residue rule on the winner-attach path: `ready` with startedByClient false is reached + // only after this client's launch exited busy. That record belongs to the client to remove, and + // the live winner never sweeps it (sweeps run at acquisition, and the winner acquired first). + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-winner-attach-marker-')); + const http = await startHttpDaemonFixture({ devices: [] }); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const winner = spawnRegisteredDaemonFixture(paths, fields(http.port), { stdio: 'ignore' }); + await awaitFile(path.join(paths.baseDir, 'registration-held')); + let contenderExit: ExecDetachedExit | undefined; + spawn.mockImplementation((_command, _args, options) => { + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options, undefined, { + emitStartupMarkerBeforeAcquisition: true, + }); + void child.exited.then((exit) => { + contenderExit = exit; + }); + return child; + }); + pause.mockImplementation(async () => { + // The contender is only refused while the winner still publishes nothing; its busy exit then + // releases the winner to publish through the unmocked clock. + if (contenderExit) fs.rmSync(deferred, { force: true }); + await actualRetry.sleep(10); + }); + try { + const response = await sendToDaemon(request(paths)); + assert.equal(response.ok, true); + assert.equal(contenderExit?.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + assert.equal(spawn.mock.calls.length, 1); + assert.equal(http.rpcRequests.length, 1); + assert.equal(isProcessAlive(winner.pid), true); + assert.deepEqual( + fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + [], + "the winner handoff must not leave the exited contender's record behind", + ); + } finally { + await closeLoopbackServer(http.server); + } +}); + test('a joined busy contender waits for a published winner to become ready without signaling it', async (t) => { if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-delayed-ready-')); diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index a5e34c4644..0fffea9369 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -396,10 +396,19 @@ async function attemptLocalDaemonStartup( } const observation: StartupPhaseObservation = { spawnedAtMs, markers: {} }; const startup = await waitForDaemonStartup(deadline, settings, launch, observation); - if (startup.kind === 'ready') return { daemon: startup.daemon }; + if (startup.kind === 'ready') { + // A contending winner's readiness (startedByClient false) is only reachable after this launch + // exited busy: its own record is this client's to sweep before the handoff. + if (!startup.daemon.startedByClient) await discardExitedLaunchRecord(settings, launch); + return { daemon: startup.daemon }; + } refreshStartupPhaseMarkers(settings, launch.pid, observation); - if (startup.kind === 'retry') return { retry: true, phases: observation }; + if (startup.kind === 'retry') { + await discardExitedLaunchRecord(settings, launch); + return { retry: true, phases: observation }; + } if (startup.kind === 'unproven') { + await discardExitedLaunchRecord(settings, launch); const startupError = normalizeError(startup.error); return { retry: false, @@ -491,6 +500,24 @@ async function joinStartup(launch: DaemonStartupLaunch): Promise { } } +/** + * #3782: sweeps the record of a launch whose exit this client just proved — a busy or unproven + * exit, or a launch that lost the registration to a contending winner. It joins this exact child + * first and removes only this child's own pid-named file, the same ownership rule a retired + * startup follows; an unjoined exit keeps its record for its writer or a later joined exit. The + * current winner's registration and record are untouched. + */ +async function discardExitedLaunchRecord( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, +): Promise { + if (await joinStartup(launch)) { + removeDaemonStartupPhasesFile( + resolveDaemonStartupPhasesPath(settings.paths.baseDir, launch.pid), + ); + } +} + function remainingStartupBudget(deadline?: number): number { return deadline === undefined ? Number.POSITIVE_INFINITY : Math.max(0, deadline - Date.now()); } From b5ff316578a378cb3f27ad00188ed85d4df8c166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:03:33 +0200 Subject: [PATCH 07/22] fix(daemon): let each launch claim its pid's marker record The per-pid file names its writer, not its lifetime: if client and daemon of an old launch both died before either cleanup ran and the host reused the pid, the old record survived under the new writer's name, the sweep skipped it as live, and the earliest-per-phase reader mixed two lifetimes into one observation. The producer now claims its own record (claimDaemonStartupPhasesFile) before its first marker, so stale same-pid content can neither mask nor extend the fresh startup's progress. A refused claim follows the marker contract: reported, never fatal. The test child repeats the real producer sequence against its own pid, which no outside test can arrange. Addresses review finding R2-1 on #3791. --- ...registration-owner-startup-markers.test.ts | 52 ++++++++++++++++++- .../src/daemon-registration-owner.ts | 20 +++++++ .../test-utils/registered-daemon-fixture.ts | 9 ++-- src/daemon.ts | 11 ++++ 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts index fafbdd532b..ec85ae5ec0 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts @@ -1,9 +1,10 @@ import { appendFile, readFile, writeFile } from 'node:fs/promises'; import { spawnSync } from 'node:child_process'; -import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { + claimDaemonStartupPhasesFile, readDaemonStartupPhaseMarkers, recordDaemonStartupMarker, removeDaemonStartupPhasesFile, @@ -53,6 +54,55 @@ describe('per-launch startup marker files', () => { }); }); +describe('producer lifetime claim', () => { + it('empties a same-pid leftover so only the fresh lifetime markers survive', async () => { + // The scenario the per-pid name alone cannot cover: both the client and the daemon of an old + // launch died before either cleanup ran, the host reused the pid, and the new writer finds the + // old lifetime's record under its own name. The entry claims its record before its first + // marker; this child repeats that producer sequence against its own pid, which no test can + // arrange from the outside. + const stateDir = await mkdtempForTest('agent-device-markers-'); + writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}'); + const entry = path.join(stateDir, 'claim-producer.js'); + const registrationUrl = new URL('./daemon-registration-owner.ts', import.meta.url).href; + writeFileSync( + entry, + `import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; +const file = resolveDaemonStartupPhasesPath(${JSON.stringify(stateDir)}, process.pid); +recordDaemonStartupMarker(file, 'listening'); +recordDaemonStartupMarker(file, 'registered'); +claimDaemonStartupPhasesFile(file); +recordDaemonStartupMarker(file, 'modules-loaded'); +`, + ); + const child = spawnSync(process.execPath, ['--experimental-strip-types', entry], { + encoding: 'utf8', + }); + if (child.status !== 0) throw new Error(child.stderr); + const phases = readDaemonStartupPhaseMarkers( + readFileSync(resolveDaemonStartupPhasesPath(stateDir, child.pid!), 'utf8'), + ); + expect(phases).toEqual({ 'modules-loaded': expect.any(Number) }); + }); + + it('reports a refused reset through the callback instead of throwing', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + // A directory in place of the marker file makes the unlink fail. + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 778); + mkdirSync(markerPath, { recursive: true }); + + const reported: unknown[] = []; + expect(() => + claimDaemonStartupPhasesFile(markerPath, (error) => { + reported.push(error); + }), + ).not.toThrow(); + + expect(reported).toHaveLength(1); + expect(reported[0]).toHaveProperty('message'); + }); +}); + describe('registration-time sweep', () => { it('retires marker files of provably-dead pids and keeps files of live pids', async () => { const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index c4bb64e0af..9734bfde77 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -126,6 +126,26 @@ function endsWithoutNewline(file: string): boolean { } } +/** + * Establishes a fresh record for this launch at `startupPhasesPath`, before the producer's first + * marker. A per-pid file names its writer, not its lifetime: if both the client and the daemon of + * an old launch died before either cleanup ran and the host later reuses that pid, the old record + * is still on disk, and only this new writer can tell a same-name file apart from its own record. + * It therefore empties its own pid's record before appending; no caller claims or resets a file + * named for another pid. A refused reset follows the marker contract: reported, never fatal, and + * a later append cannot silently mix lifetimes because the reset ran first when it could run. + */ +export function claimDaemonStartupPhasesFile( + startupPhasesPath: string, + reportFailure?: (error: NormalizedError) => void, +): void { + try { + fs.rmSync(startupPhasesPath, { force: true }); + } catch (error) { + reportFailure?.(normalizeError(error)); + } +} + /** * Removes one startup's marker file. The writer calls it once its own startup completes; the client * that launched a process calls it once that process's exit is joined. Both callers can prove no diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index f808133a73..66668f7e0a 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -50,7 +50,7 @@ export function registeredDaemonFixtureArgs( entry, `import fs from 'node:fs'; import path from 'node:path'; -import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; +import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration, claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; const paths = ${JSON.stringify(paths)}; const barrier = ${JSON.stringify(acquisitionBarrier)}; const emitBefore = ${fixtureOptions?.emitStartupMarkerBeforeAcquisition ? 'true' : 'false'}; @@ -59,8 +59,11 @@ if (barrier) { fs.writeFileSync(barrier + '.ready-' + process.pid, 'ready'); while (!fs.existsSync(barrier)) await new Promise(resolve => setTimeout(resolve, 10)); } -if (emitBefore) - recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(paths.baseDir, process.pid), 'modules-loaded'); +if (emitBefore) { + const ownRecord = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); + claimDaemonStartupPhasesFile(ownRecord); + recordDaemonStartupMarker(ownRecord, 'modules-loaded'); +} const acquired = await tryAcquireDaemonRegistration(paths); if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); if (emitAfter) diff --git a/src/daemon.ts b/src/daemon.ts index c9f958b7ab..c3773a6922 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -6,6 +6,7 @@ import { } from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import { + claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath, } from '@agent-device/daemon-contracts/daemon-registration-owner'; @@ -21,6 +22,16 @@ function reportMarkerFailure(error: unknown): void { // #3782: the first marker the waiting client can read. It says this entry's module graph evaluated, // which on an overloaded host is the slowest thing a daemon start does before it can prove itself. // The file is named for this process's pid, so the client attributes every line to this launch. +// The claim first: a host that reused this pid after an old launch died before either cleanup ran +// would otherwise leave that lifetime's phases in this same file, and this launch's timings would +// read as older and farther-progressed than they are. +claimDaemonStartupPhasesFile( + resolveDaemonStartupPhasesPath( + resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, + process.pid, + ), + reportMarkerFailure, +); recordDaemonStartupMarker( resolveDaemonStartupPhasesPath( resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, From 09ae8ae62ba2f9b92cb87ada5f7d487f9a2ad303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:03:38 +0200 Subject: [PATCH 08/22] refactor(daemon): move startup diagnostics and registration completion out of the runtime Code ownership allows no exception: behavior was added to a module over 1000 lines, so the module is split at its existing boundary first. flushDaemonStartupDiagnostics, emitDaemonDiagnostic, finishDaemonRegistration, and armDaemonMetadataLossWatch form the daemon-log diagnostic and registration-completion edge the runtime already treats as one unit; they move with their callers updated, no behavior change. daemon-runtime.ts is back under the limit. --- .../__tests__/daemon-runtime-app-log.test.ts | 2 +- .../daemon/src/server/http/daemon-runtime.ts | 88 ++----------------- .../server/http/daemon-startup-diagnostics.ts | 83 +++++++++++++++++ 3 files changed, 91 insertions(+), 82 deletions(-) create mode 100644 packages/daemon/src/server/http/daemon-startup-diagnostics.ts diff --git a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts index e6230f3a9d..18f5f902cb 100644 --- a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts +++ b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts @@ -10,7 +10,7 @@ import type { ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import { createDurableResourceEnvelope } from '@agent-device/capture-kit'; import { createTestAppLogLiveHandle, mkdtempForTestSync } from '@agent-device/test-kit'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; -import { flushDaemonStartupDiagnostics } from '../http/daemon-runtime.ts'; +import { flushDaemonStartupDiagnostics } from '../http/daemon-startup-diagnostics.ts'; import type { SessionState } from '../../core/session-state.ts'; import { unavailableDeviceRuntimeGateway } from '../../__tests__/support/unavailable-device-runtime-gateway.ts'; diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 1e62ad18a0..4ab04644dd 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -6,7 +6,7 @@ import { type ClaimRecoveryGatewayFactory, } from '@agent-device/daemon-contracts/device-claim-recovery'; import crypto from 'node:crypto'; -import { asAppError, AppError, normalizeError } from '@agent-device/kernel/errors'; +import { asAppError, AppError } from '@agent-device/kernel/errors'; import { SessionStore } from '../../core/session-store.ts'; import { DAEMON_CAPABILITIES, @@ -76,7 +76,6 @@ import { readCurrentOwnerIdentity, reapOwnedProcessRecordsAtStartup, type OwnedProcessRecordStore, - type OwnerIdentity, } from '@agent-device/host-kit/process'; import { isEnvTruthy, sleep } from '@agent-device/host-kit/retry'; import { @@ -91,9 +90,13 @@ import { recordDaemonStartupMarker, removeDaemonStartupPhasesFile, resolveDaemonStartupPhasesPath, - type DaemonRegistrationOwner, } from '@agent-device/daemon-contracts/daemon-registration-owner'; -import { watchDaemonMetadataLoss } from './daemon-metadata-loss.ts'; +import { + armDaemonMetadataLossWatch, + emitDaemonDiagnostic, + finishDaemonRegistration, + flushDaemonStartupDiagnostics, +} from './daemon-startup-diagnostics.ts'; import { retainPngWorker } from '@agent-device/capture-kit/png-worker-client'; import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; @@ -245,83 +248,6 @@ export function daemonStartupExitCode(error: unknown): number { return typeof exitCode === 'number' ? exitCode : 1; } -export async function flushDaemonStartupDiagnostics( - logPath: string, - diagnostics: readonly ResourceDiagnostic[], -): Promise { - if (diagnostics.length === 0) return; - await withDaemonDiagnosticsScope({ logPath, command: 'daemon-startup', debug: false }, () => { - for (const diagnostic of diagnostics) { - emitDiagnostic({ - level: 'warn', - phase: diagnostic.phase, - data: { resourcePath: diagnostic.resourcePath, ...diagnostic.data }, - }); - } - }); -} - -/** - * Records one daemon-level event. These run outside any request, so there is no request scope and no - * resolved debug level to inherit; debug is forced on for the same reason the #2681 handoff forces it — - * the event is the point of the record and must not be dropped by a level that was never set for it. - */ -async function emitDaemonDiagnostic( - logPath: string, - phase: string, - data: Record, -): Promise { - await withDaemonDiagnosticsScope({ logPath }, () => { - emitDiagnostic({ level: 'warn', phase, data }); - }); -} - -async function finishDaemonRegistration(params: { - registration: DaemonRegistrationOwner; - infoPath: string; - logPath: string; - outcome?: Parameters[0]; -}): Promise { - try { - const removal = await params.registration.finish(params.outcome); - if (removal.state !== 'removed' && removal.state !== 'absent') { - await emitDaemonDiagnostic(params.logPath, 'daemon_info_removal_declined', { - infoPath: params.infoPath, - removed: false, - reason: removal.state, - ...(removal.state === 'replaced' ? { registeredPid: removal.identity.pid } : {}), - }); - } - } catch (error) { - await emitDaemonDiagnostic(params.logPath, 'daemon_registration_finish_failed', { - error: normalizeError(error), - }); - } -} - -/** - * Starts the watch that reports this daemon's registration being taken over, and returns the handle - * that stops it. It is armed only once this process has published its own record: before publication - * the file legitimately describes a predecessor, and losing that is not this daemon's event. - */ -function armDaemonMetadataLossWatch( - stateDir: string, - infoPath: string, - logPath: string, - owner: OwnerIdentity, -): () => void { - return watchDaemonMetadataLoss({ - infoPath, - stateDir, - owner, - onLoss: (loss) => - void emitDaemonDiagnostic(logPath, 'daemon_metadata_lost', { - infoPath, - ...loss, - }).catch(() => {}), - }); -} - export async function startDaemonRuntime( options: DaemonRuntimeOptions, ): Promise { diff --git a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts new file mode 100644 index 0000000000..3d8c75a3b7 --- /dev/null +++ b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts @@ -0,0 +1,83 @@ +import { normalizeError } from '@agent-device/kernel/errors'; +import { emitDiagnostic, type ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; +import type { OwnerIdentity } from '@agent-device/host-kit/process'; +import type { DaemonRegistrationOwner } from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { withDaemonDiagnosticsScope } from '@agent-device/daemon-contracts/daemon-diagnostics-scope'; +import { watchDaemonMetadataLoss } from './daemon-metadata-loss.ts'; + +export async function flushDaemonStartupDiagnostics( + logPath: string, + diagnostics: readonly ResourceDiagnostic[], +): Promise { + if (diagnostics.length === 0) return; + await withDaemonDiagnosticsScope({ logPath, command: 'daemon-startup', debug: false }, () => { + for (const diagnostic of diagnostics) { + emitDiagnostic({ + level: 'warn', + phase: diagnostic.phase, + data: { resourcePath: diagnostic.resourcePath, ...diagnostic.data }, + }); + } + }); +} + +/** + * Records one daemon-level event. These run outside any request, so there is no request scope and no + * resolved debug level to inherit; debug is forced on for the same reason the #2681 handoff forces it — + * the event is the point of the record and must not be dropped by a level that was never set for it. + */ +export async function emitDaemonDiagnostic( + logPath: string, + phase: string, + data: Record, +): Promise { + await withDaemonDiagnosticsScope({ logPath }, () => { + emitDiagnostic({ level: 'warn', phase, data }); + }); +} + +export async function finishDaemonRegistration(params: { + registration: DaemonRegistrationOwner; + infoPath: string; + logPath: string; + outcome?: Parameters[0]; +}): Promise { + try { + const removal = await params.registration.finish(params.outcome); + if (removal.state !== 'removed' && removal.state !== 'absent') { + await emitDaemonDiagnostic(params.logPath, 'daemon_info_removal_declined', { + infoPath: params.infoPath, + removed: false, + reason: removal.state, + ...(removal.state === 'replaced' ? { registeredPid: removal.identity.pid } : {}), + }); + } + } catch (error) { + await emitDaemonDiagnostic(params.logPath, 'daemon_registration_finish_failed', { + error: normalizeError(error), + }); + } +} + +/** + * Starts the watch that reports this daemon's registration being taken over, and returns the handle + * that stops it. It is armed only once this process has published its own record: before publication + * the file legitimately describes a predecessor, and losing that is not this daemon's event. + */ +export function armDaemonMetadataLossWatch( + stateDir: string, + infoPath: string, + logPath: string, + owner: OwnerIdentity, +): () => void { + return watchDaemonMetadataLoss({ + infoPath, + stateDir, + owner, + onLoss: (loss) => + void emitDaemonDiagnostic(logPath, 'daemon_metadata_lost', { + infoPath, + ...loss, + }).catch(() => {}), + }); +} From 2abb7e4d68befe95cd5db991fa89cc6e6328723e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:03:42 +0200 Subject: [PATCH 09/22] refactor(daemon): drop the phase-timings return cast describeStartupPhaseTimings builds a Record that always starts at spawned: 0; the intersection cast asserted what the literal already showed. No caller branched on it. Addresses review finding R2-2 on #3791. --- src/daemon-client/daemon-client-startup.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index 0fffea9369..c4d4584919 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -619,14 +619,12 @@ function lastStartupProgressMs(observation: StartupPhaseObservation): number { ); } -function describeStartupPhaseTimings( - observation: StartupPhaseObservation, -): Record & { spawned: 0 } { +function describeStartupPhaseTimings(observation: StartupPhaseObservation): Record { const timings: Record = { spawned: 0 }; for (const [phase, timestampMs] of Object.entries(observation.markers)) { timings[phase] = Math.max(0, timestampMs - observation.spawnedAtMs); } - return timings as Record & { spawned: 0 }; + return timings; } function resolveStartupProgressHint(reason: DaemonStartupFailureReason, ceilingMs: number): string { From 7d6d8b70e91dd9ffedb16e90593ee88282de1599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:03:46 +0200 Subject: [PATCH 10/22] ci(ios): capture the cold daemon-startup timer in the preflight step The #3782 startup-budget evidence needs one measured successful cold startup: after clean:daemon, the prepare command's daemon_startup diagnostic now wraps a fresh managed startup and lands in the step log. Behavior and JSON output are unchanged. Records the evidence obligation for #3791. --- .github/workflows/ios.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 2841edd259..2fcdf915fc 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -169,7 +169,9 @@ jobs: - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + # --debug keeps the daemon_startup diagnostic, which #3791 needs as the measured + # successful cold-startup evidence for the startup budget (#3782). JSON stays on stdout. + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --debug --json # The replay reuses the prepared runner in the same daemon. --debug keeps every step's request # log, not only the failing one, so a stale tap can be traced to its pre-tap capture (#3367). From 242338aae982f5da9e8541378ab75dcbb4a47573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:16:26 +0200 Subject: [PATCH 11/22] test(daemon): give every fixture launch its own entry script registeredDaemonFixtureArgs rewrote one shared dist/src/internal/ daemon.js per construction, so a second build could replace the script before an earlier launched child loaded it, silently applying the wrong per-child marker switches and registration fields. Each construction now writes a unique fixture-entries//dist/src/internal/daemon.js: the per-launch directory keeps the daemon entry path shape the retirement identity checks require, and cleanup still removes the state dir. A regression test builds both arg sets before launching either child and proves child 1 executes its own switches while child 2 exits busy: red on the shared path, green per-launch. Fixes review thread PRRT_kwDORFGHaM6tk7ed on #3791. --- .../registered-daemon-fixture.test.ts | 54 +++++++++++++++++++ .../test-utils/registered-daemon-fixture.ts | 30 ++++++++--- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/src/__tests__/test-utils/registered-daemon-fixture.test.ts b/src/__tests__/test-utils/registered-daemon-fixture.test.ts index 53e4105d83..ced6b449d0 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.test.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.test.ts @@ -1,15 +1,69 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'vitest'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { stopDaemonProcess } from '@agent-device/daemon-contracts/daemon-process'; +import { + DAEMON_STARTUP_EXIT_CODES, + resolveDaemonStartupPhasesPath, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { + launchRegisteredDaemonFixtureArgs, + registeredDaemonFixtureArgs, spawnRegisteredDaemonFixture, waitForRegisteredDaemonFixture, finishRegisteredDaemonFixture, } from './registered-daemon-fixture.ts'; +test('args built before any launch keep their own entry script under a concurrent second build', async () => { + // #3791 review: a second registeredDaemonFixtureArgs call must not rewrite the script file an + // earlier call already generated. Both arg sets are built before either child starts (that build + // order is the overwrite window); child 1 then runs its own switches and child 2 exits busy. + // Under a shared entry path, child 1 would execute child 2's script and its own record would + // carry child 2's marker switch instead of its own. + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-fixture-entry-isolation-')); + fs.mkdirSync(paths.baseDir, { recursive: true }); + fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); + const fields = { + httpPort: 4211, + token: 'fixture', + version: 'test', + codeOrigin: 'checkout' as const, + codeSignature: 'fixture', + }; + const firstArgs = registeredDaemonFixtureArgs(paths, fields, undefined, { + emitStartupMarkerBeforeAcquisition: true, + }); + const secondArgs = registeredDaemonFixtureArgs(paths, fields, undefined, { + emitStartupMarker: true, + }); + assert.notEqual(firstArgs[1], secondArgs[1]); + const first = launchRegisteredDaemonFixtureArgs(paths, firstArgs, { stdio: 'ignore' }); + try { + // Child 1 holds the registration before child 2 starts, so child 2's busy exit is ordered, + // not scheduled. + const held = path.join(paths.baseDir, `registration-held-${first.pid}`); + for (let attempt = 0; !fs.existsSync(held); attempt += 1) { + assert.ok(attempt < 400, 'child 1 never acquired the registration'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const second = launchRegisteredDaemonFixtureArgs(paths, secondArgs, { stdio: 'ignore' }); + assert.equal((await second.exited).exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + const ownRecord = resolveDaemonStartupPhasesPath(paths.baseDir, first.pid); + const text = fs.readFileSync(ownRecord, 'utf8'); + assert.match(text, /modules-loaded/); + assert.doesNotMatch( + text, + /lock-acquired/, + "child 1 must run the script built for it, not the second build's switches", + ); + } finally { + await finishRegisteredDaemonFixture(paths.baseDir); + } +}); + test('a joined fixture exit refuses its remaining registration metadata', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-fixture-exited-publication-')); const child = spawnRegisteredDaemonFixture( diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index 66668f7e0a..c865280101 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { vi } from 'vitest'; @@ -39,9 +40,14 @@ export function registeredDaemonFixtureArgs( acquisitionBarrier?: string, fixtureOptions?: RegisteredDaemonFixtureOptions, ): string[] { - const entry = path.join(paths.baseDir, 'dist', 'src', 'internal', 'daemon.js'); + // One directory per construction: a later call must never overwrite a script an earlier + // launched child has not loaded yet. The per-launch directory still ends in the real daemon + // entry path so the daemon-command identity checks classify this child exactly as they + // classify a real daemon. + const launchDir = path.join(paths.baseDir, 'fixture-entries', crypto.randomUUID()); + const entry = path.join(launchDir, 'dist', 'src', 'internal', 'daemon.js'); fs.mkdirSync(path.dirname(entry), { recursive: true }); - fs.writeFileSync(path.join(paths.baseDir, 'package.json'), '{"type":"module"}'); + fs.writeFileSync(path.join(launchDir, 'package.json'), '{"type":"module"}'); const registrationUrl = new URL( '../../../packages/daemon-contracts/src/daemon-registration-owner.ts', import.meta.url, @@ -89,6 +95,18 @@ setInterval(() => {}, 1000); return ['--experimental-strip-types', entry]; } +export function launchRegisteredDaemonFixtureArgs( + paths: DaemonPaths, + args: string[], + options: Parameters[2], +): ReturnType { + const child = actualCommand.runCmdDetachedMonitored(process.execPath, args, options); + const owned = children.get(paths.baseDir) ?? []; + owned.push({ launch: child, startTime: readProcessStartTime(child.pid) }); + children.set(paths.baseDir, owned); + return child; +} + export function spawnRegisteredDaemonFixture( paths: DaemonPaths, fields: DaemonRegistrationFields, @@ -96,15 +114,11 @@ export function spawnRegisteredDaemonFixture( acquisitionBarrier?: string, fixtureOptions?: RegisteredDaemonFixtureOptions, ): ReturnType { - const child = actualCommand.runCmdDetachedMonitored( - process.execPath, + return launchRegisteredDaemonFixtureArgs( + paths, registeredDaemonFixtureArgs(paths, fields, acquisitionBarrier, fixtureOptions), options, ); - const owned = children.get(paths.baseDir) ?? []; - owned.push({ launch: child, startTime: readProcessStartTime(child.pid) }); - children.set(paths.baseDir, owned); - return child; } export async function waitForRegisteredDaemonFixture( From 3a6a33559fec02ad48ae6add0fdbcad1925b561d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:29:17 +0200 Subject: [PATCH 12/22] perf(daemon): keep the startup-diagnostics edge out of the eager closure The split moved the helpers next to the runtime but left a static edge, so both daemon entries evaluated one more module and 12-15 more top-level statements than the merge-base (Coverage lane red: 3 eager-closure-budgets failures at OUTPUT_ECONOMY_BASE=4a8abed7b9). startDaemonRuntime now loads them through a function-scoped await import at its first use point, the loading shape the gate prescribes; composition, publication, flush, and shutdown order are unchanged and the source-order guards still read them from this module. Red at 62a09bd298 (3 failures), green here: 846/846. Addresses the eager-closure failure reported at 62a09bd298 on #3791. --- .../daemon/src/server/http/daemon-runtime.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 4ab04644dd..416a072994 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -91,12 +91,6 @@ import { removeDaemonStartupPhasesFile, resolveDaemonStartupPhasesPath, } from '@agent-device/daemon-contracts/daemon-registration-owner'; -import { - armDaemonMetadataLossWatch, - emitDaemonDiagnostic, - finishDaemonRegistration, - flushDaemonStartupDiagnostics, -} from './daemon-startup-diagnostics.ts'; import { retainPngWorker } from '@agent-device/capture-kit/png-worker-client'; import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; @@ -276,6 +270,16 @@ async function startDaemonRuntimeOnHost( const { env } = host; const stdout = options.stdout ?? process.stdout; const stderr = options.stderr ?? process.stderr; + // The daemon-log diagnostics and registration-completion helpers enter behind a + // function-scoped `await import`: they are startup/lifecycle evidence work this server + // performs after composition, and the daemon entry's eager closure stays exactly the + // operational server's (ADR 0019). This is their first possible use point. + const { + armDaemonMetadataLossWatch, + emitDaemonDiagnostic, + finishDaemonRegistration, + flushDaemonStartupDiagnostics, + } = await import('./daemon-startup-diagnostics.ts'); const daemonPaths = resolveDaemonPaths(env.AGENT_DEVICE_STATE_DIR, { env, cwd: host.cwd }); const { baseDir, infoPath, logPath, sessionsDir } = daemonPaths; const daemonServerMode = resolveDaemonServerMode(env.AGENT_DEVICE_DAEMON_SERVER_MODE); From 0d5194f1b2b84cd11b040fa439c5f8979c52613c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 00:40:56 +0200 Subject: [PATCH 13/22] ci(ios): measure the cold daemon start at the public SDK ensure seam The prepare step's daemon_startup diagnostic is not a cold-startup measurement: the CLI's ensured transport finishes ensureDaemon before sendToDaemon's timer wraps the attach connect, so the timer only sees an attach. Time the public ensureDaemon() directly after clean:daemon and record phase, durationMs, started, and pid, so the record counts as cold only when started is true. The prepare command keeps its original semantics and attaches to the same daemon. Corrects the measurement assumption behind 62a09bd298 on #3791; the cold-startup CI evidence for #3782 is still missing until a run reports this record with started:true. --- .github/workflows/ios.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 2fcdf915fc..427774d8fc 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -169,9 +169,24 @@ jobs: - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - # --debug keeps the daemon_startup diagnostic, which #3791 needs as the measured - # successful cold-startup evidence for the startup budget (#3782). JSON stays on stdout. - node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --debug --json + # Time the cold managed startup at the public SDK seam. The clean state dir above makes + # this ensure the cold start, and `started` says whether it truly started one. The CLI + # `daemon_startup` timer cannot be this evidence: it wraps only attach, which runs after + # the ensured transport has already finished starting the daemon. + node --input-type=module -e ' + import { performance } from "node:perf_hooks"; + const { ensureDaemon } = await import("./dist/src/index.js"); + const stateDir = process.env.AGENT_DEVICE_STATE_DIR; + const startedAt = performance.now(); + const connection = await ensureDaemon({ stateDir }); + console.log(JSON.stringify({ + phase: "daemon_cold_startup", + durationMs: Math.round(performance.now() - startedAt), + started: connection.started, + pid: connection.daemon.pid, + })); + ' + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json # The replay reuses the prepared runner in the same daemon. --debug keeps every step's request # log, not only the failing one, so a stale tap can be traced to its pre-tap capture (#3367). From 60d09e3320a2c3419a6b54a328240e6f5bd7579f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 02:13:50 +0200 Subject: [PATCH 14/22] fix(daemon): report startup-ceiling when any live startup is pending at the deadline When the launched contender exited busy while another live daemon held the registration and deferred publication, the wait loop ended in an unlabeled 'timeout' result that mapped to startup-exited with the generic hint. Any deadline with a live startup still pending, this client's launch or a verified live lock owner, now reports startup-ceiling with the effective ceiling and the overload hint; startup-exited stays reserved for a real exit or no live startup. Remove the 'timeout' wait kind so the unlabeled state cannot exist and add the sendToDaemon-boundary regressions for both outcomes. Also compress this PR's narration and duplicate test setup to keep the gross diff within one coherent change, and add the Unreleased changelog entry for the startup budget, the stall window, and the three reasons. --- .github/workflows/ios.yml | 6 +- CHANGELOG.md | 2 + ...registration-owner-startup-markers.test.ts | 263 ++++++++---------- .../src/daemon-registration-owner.ts | 61 +--- .../daemon/src/server/http/daemon-runtime.ts | 17 +- .../server/http/daemon-startup-diagnostics.ts | 12 +- .../registered-daemon-fixture.test.ts | 9 +- .../test-utils/registered-daemon-fixture.ts | 17 +- .../daemon-client-startup-budget.test.ts | 134 ++++----- .../daemon-client-startup-race.test.ts | 15 +- src/daemon-client/daemon-client-startup.ts | 93 +++---- src/daemon.ts | 32 +-- 12 files changed, 253 insertions(+), 408 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 427774d8fc..36323451fd 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -169,10 +169,8 @@ jobs: - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - # Time the cold managed startup at the public SDK seam. The clean state dir above makes - # this ensure the cold start, and `started` says whether it truly started one. The CLI - # `daemon_startup` timer cannot be this evidence: it wraps only attach, which runs after - # the ensured transport has already finished starting the daemon. + # The public-SDK timing record is the cold-startup evidence; the CLI `daemon_startup` + # timer only wraps attach, after the ensured transport already started the daemon. node --input-type=module -e ' import { performance } from "node:perf_hooks"; const { ensureDaemon } = await import("./dist/src/index.js"); diff --git a/CHANGELOG.md b/CHANGELOG.md index c0b3ecaa40..d30b483e9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - The PNG worker no longer calls `process.emitWarning` when it is unavailable, and it starts with an empty environment (`env: {}`). The daemon registers a sink when it pre-warms the worker, so every transition to unavailable (spawn failure, missing entry, load error, exit) leaves exactly one `png_worker_unavailable` event, with a typed `reason` (`entry-not-found`, `spawn-failed`, `worker-error`, `worker-exited`) and its `detail`, in `daemon.log`, with or without `--debug`. Requests then continue on the in-process fallback without repeating the event. - Behavior change: `resolveHubAppReference`, shipped through `agent-device/plugins/webdriver`, now needs `cwd` to resolve a relative app path. Without it the call fails with `INVALID_ARGS` and `details.reason: provider_app_cwd_missing`; it used to resolve against the process's working directory. Absolute paths, URLs and provider references are unaffected. The daemon fills it from the client's working directory (or its host's) for every provider lease request, so `connect` and `lease_allocate` are unchanged. +- A client starting a local daemon now waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower instead of failing it. The wait stops at a ceiling of 120 s by default; set `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` to a positive integer number of milliseconds to override it for the CLI, MCP, and Node clients alike. A failed startup names the outcome in `error.details.reason`, and you should branch on it, never on the message: `startup-exited` (the process ended before registering), `startup-stalled` (alive but no startup progress for a fixed 60 s window, which the ceiling variable does not change; the daemon is stopped), or `startup-ceiling` (a startup was still live — this client's daemon or another client's — when the ceiling passed; the client stops its own launch and never signals another client's live daemon). `details.phases` carries the startup phase timings the client observed and `details.startupTimeoutMs` the effective ceiling. + - The `daemon-not-running` hint now always names `--state-dir `, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`. - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir `. - `@agent-device/testmu` is now a version 2 provider plugin. `import testmu from '@agent-device/testmu'` gives a factory, `testmu({ credentials: { username, accessKey } })`, that reads those credentials instead of `LT_USERNAME` and `LT_ACCESS_KEY`; without options it reads the variables as before. The package still declares the version 1 plugin entry, so `agent-device plugins add @agent-device/testmu` keeps working on hosts that predate the version 2 contract. diff --git a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts index ec85ae5ec0..783e927e36 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts @@ -2,7 +2,7 @@ import { appendFile, readFile, writeFile } from 'node:fs/promises'; import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { expect, test } from 'vitest'; import { claimDaemonStartupPhasesFile, readDaemonStartupPhaseMarkers, @@ -14,179 +14,138 @@ import { import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { mkdtempForTest } from '@agent-device/test-kit'; -describe('per-launch startup marker files', () => { - it('scopes marker files to the writing pid so a fresh launch never reads an older launch phases', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const first = resolveDaemonStartupPhasesPath(stateDir, 1111); - const second = resolveDaemonStartupPhasesPath(stateDir, 2222); - - recordDaemonStartupMarker(first, 'lock-acquired'); - - expect(second).not.toBe(first); - expect(existsSync(second)).toBe(false); - - recordDaemonStartupMarker(second, 'modules-loaded'); - const firstPhases = readDaemonStartupPhaseMarkers(readFileSync(first, 'utf8')); - const secondPhases = readDaemonStartupPhaseMarkers(readFileSync(second, 'utf8')); - expect(Object.keys(firstPhases)).toEqual(['lock-acquired']); - expect(Object.keys(secondPhases)).toEqual(['modules-loaded']); - }); +test('per-launch marker files: a fresh launch never reads an older launch phases', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const first = resolveDaemonStartupPhasesPath(stateDir, 1111); + const second = resolveDaemonStartupPhasesPath(stateDir, 2222); + recordDaemonStartupMarker(first, 'lock-acquired'); + expect(second).not.toBe(first); + expect(existsSync(second)).toBe(false); + recordDaemonStartupMarker(second, 'modules-loaded'); + const firstPhases = readDaemonStartupPhaseMarkers(readFileSync(first, 'utf8')); + const secondPhases = readDaemonStartupPhaseMarkers(readFileSync(second, 'utf8')); + expect(Object.keys(firstPhases)).toEqual(['lock-acquired']); + expect(Object.keys(secondPhases)).toEqual(['modules-loaded']); +}); - it('keeps the earliest record when a replayed phase is appended after a later one', () => { - const text = [ - '{"data":{"startupPhase":"listening","timestampMs":500}}', - '{"data":{"startupPhase":"lock-acquired","timestampMs":20}}', - '{"data":{"startupPhase":"lock-acquired","timestampMs":35}}', - ].join('\n'); - const phases = readDaemonStartupPhaseMarkers(text); - expect(phases['lock-acquired']).toBe(20); - expect(phases.listening).toBe(500); - }); +test('marker reading keeps the earliest record when a phase replays after a later one', () => { + const text = [ + '{"data":{"startupPhase":"listening","timestampMs":500}}', + '{"data":{"startupPhase":"lock-acquired","timestampMs":20}}', + '{"data":{"startupPhase":"lock-acquired","timestampMs":35}}', + ].join('\n'); + const phases = readDaemonStartupPhaseMarkers(text); + expect(phases['lock-acquired']).toBe(20); + expect(phases.listening).toBe(500); +}); - it('ignores unknown phases and unparsable lines so the protocol stays additive', () => { - const text = [ - '{"data":{"startupPhase":"heartbeat","timestampMs":10}}', - '{"data":{"startupPhase":"registered","timestampMs":"7"}}', - 'not json', - '{"data":{"startupPhase":"listening","timestampMs":30}}', - ].join('\n'); - expect(readDaemonStartupPhaseMarkers(text)).toEqual({ listening: 30 }); - }); +test('marker reading ignores unknown phases and unparsable lines', () => { + const text = [ + '{"data":{"startupPhase":"heartbeat","timestampMs":10}}', + '{"data":{"startupPhase":"registered","timestampMs":"7"}}', + 'not json', + '{"data":{"startupPhase":"listening","timestampMs":30}}', + ].join('\n'); + expect(readDaemonStartupPhaseMarkers(text)).toEqual({ listening: 30 }); }); -describe('producer lifetime claim', () => { - it('empties a same-pid leftover so only the fresh lifetime markers survive', async () => { - // The scenario the per-pid name alone cannot cover: both the client and the daemon of an old - // launch died before either cleanup ran, the host reused the pid, and the new writer finds the - // old lifetime's record under its own name. The entry claims its record before its first - // marker; this child repeats that producer sequence against its own pid, which no test can - // arrange from the outside. - const stateDir = await mkdtempForTest('agent-device-markers-'); - writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}'); - const entry = path.join(stateDir, 'claim-producer.js'); - const registrationUrl = new URL('./daemon-registration-owner.ts', import.meta.url).href; - writeFileSync( - entry, - `import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; +test('the producer lifetime claim empties a same-pid leftover', async () => { + // Pid reuse after both the old client and daemon died before either cleanup: the entry claims + // its record before its first marker; this child repeats that producer sequence itself. + const stateDir = await mkdtempForTest('agent-device-markers-'); + writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}'); + const entry = path.join(stateDir, 'claim-producer.js'); + const registrationUrl = new URL('./daemon-registration-owner.ts', import.meta.url).href; + writeFileSync( + entry, + `import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; const file = resolveDaemonStartupPhasesPath(${JSON.stringify(stateDir)}, process.pid); recordDaemonStartupMarker(file, 'listening'); recordDaemonStartupMarker(file, 'registered'); claimDaemonStartupPhasesFile(file); recordDaemonStartupMarker(file, 'modules-loaded'); `, - ); - const child = spawnSync(process.execPath, ['--experimental-strip-types', entry], { - encoding: 'utf8', - }); - if (child.status !== 0) throw new Error(child.stderr); - const phases = readDaemonStartupPhaseMarkers( - readFileSync(resolveDaemonStartupPhasesPath(stateDir, child.pid!), 'utf8'), - ); - expect(phases).toEqual({ 'modules-loaded': expect.any(Number) }); - }); - - it('reports a refused reset through the callback instead of throwing', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - // A directory in place of the marker file makes the unlink fail. - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 778); - mkdirSync(markerPath, { recursive: true }); - - const reported: unknown[] = []; - expect(() => - claimDaemonStartupPhasesFile(markerPath, (error) => { - reported.push(error); - }), - ).not.toThrow(); - - expect(reported).toHaveLength(1); - expect(reported[0]).toHaveProperty('message'); + ); + const child = spawnSync(process.execPath, ['--experimental-strip-types', entry], { + encoding: 'utf8', }); + if (child.status !== 0) throw new Error(child.stderr); + const phases = readDaemonStartupPhaseMarkers( + readFileSync(resolveDaemonStartupPhasesPath(stateDir, child.pid!), 'utf8'), + ); + expect(phases).toEqual({ 'modules-loaded': expect.any(Number) }); }); -describe('registration-time sweep', () => { - it('retires marker files of provably-dead pids and keeps files of live pids', async () => { - const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); - const deadPid = spawnDeadPid(); - const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); - const dead = resolveDaemonStartupPhasesPath(paths.baseDir, deadPid); - const legacy = path.join(paths.baseDir, 'daemon-startup-phases.jsonl'); - await Promise.all([ - writeFile(own, 'x\n', 'utf8'), - writeFile(dead, 'x\n', 'utf8'), - writeFile(legacy, 'x\n', 'utf8'), - ]); - - const attempt = await tryAcquireDaemonRegistration(paths); - expect(attempt.status).toBe('acquired'); - if (attempt.status !== 'acquired') throw new Error('registration refused'); - - expect(existsSync(dead)).toBe(false); - expect(existsSync(own)).toBe(true); - // The sweep retires only its own kind: an unknown entry stays as evidence. - expect(existsSync(legacy)).toBe(true); - await attempt.owner.finish(); - }); - - it('retires files whose pid is unparseable', async () => { - const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); - const stale = path.join(paths.baseDir, 'daemon-startup-phases-.jsonl'); - await writeFile(stale, 'x\n', 'utf8'); - - const attempt = await tryAcquireDaemonRegistration(paths); - expect(attempt.status).toBe('acquired'); - if (attempt.status !== 'acquired') throw new Error('registration refused'); - - expect(existsSync(stale)).toBe(false); - await attempt.owner.finish(); - }); +test('a refused claim reset reports through the callback instead of throwing', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 778); + mkdirSync(markerPath, { recursive: true }); + const reported: unknown[] = []; + expect(() => + claimDaemonStartupPhasesFile(markerPath, (error) => { + reported.push(error); + }), + ).not.toThrow(); + expect(reported).toHaveLength(1); + expect(reported[0]).toHaveProperty('message'); }); -describe('removeDaemonStartupPhasesFile', () => { - it('removes a marker file and tolerates a file already gone', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const file = resolveDaemonStartupPhasesPath(stateDir, 4242); - await writeFile(file, 'x\n', 'utf8'); - - removeDaemonStartupPhasesFile(file); - expect(existsSync(file)).toBe(false); - - expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow(); - }); +test('acquiring the registration retires files of dead or unowned pids only', async () => { + const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); + const dead = resolveDaemonStartupPhasesPath(paths.baseDir, spawnDeadPid()); + const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); + const unparseable = path.join(paths.baseDir, 'daemon-startup-phases-.jsonl'); + const legacy = path.join(paths.baseDir, 'daemon-startup-phases.jsonl'); + await Promise.all([ + writeFile(own, 'x\n', 'utf8'), + writeFile(dead, 'x\n', 'utf8'), + writeFile(unparseable, 'x\n', 'utf8'), + writeFile(legacy, 'x\n', 'utf8'), + ]); + const attempt = await tryAcquireDaemonRegistration(paths); + expect(attempt.status).toBe('acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + expect(existsSync(dead)).toBe(false); + expect(existsSync(unparseable)).toBe(false); + expect(existsSync(own)).toBe(true); + // The sweep retires only its own kind: an unknown entry stays as evidence. + expect(existsSync(legacy)).toBe(true); + await attempt.owner.finish(); }); -describe('recordDaemonStartupMarker failure seam', () => { - it('reports a write failure through the callback instead of throwing, keeping startup running', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - // A directory in place of the marker file makes every append fail. - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); - mkdirSync(markerPath, { recursive: true }); - - const reported: unknown[] = []; - expect(() => - recordDaemonStartupMarker(markerPath, 'modules-loaded', (error) => { - reported.push(error); - }), - ).not.toThrow(); - - expect(reported).toHaveLength(1); - expect(reported[0]).toHaveProperty('message'); - }); - - it('keeps appending valid JSONL when an incomplete line precedes a crash-replay append', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999); - await appendFile(markerPath, '{"data":{"startupPhase":"modules-lo', 'utf8'); +test('removeDaemonStartupPhasesFile removes a file and tolerates one already gone', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const file = resolveDaemonStartupPhasesPath(stateDir, 4242); + await writeFile(file, 'x\n', 'utf8'); + removeDaemonStartupPhasesFile(file); + expect(existsSync(file)).toBe(false); + expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow(); +}); - recordDaemonStartupMarker(markerPath, 'composition-loaded'); +test('a marker write failure reports through the callback, keeping startup running', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); + mkdirSync(markerPath, { recursive: true }); + const reported: unknown[] = []; + expect(() => + recordDaemonStartupMarker(markerPath, 'modules-loaded', (error) => { + reported.push(error); + }), + ).not.toThrow(); + expect(reported).toHaveLength(1); + expect(reported[0]).toHaveProperty('message'); +}); - const tail = await readFile(markerPath, 'utf8'); - expect(Object.keys(readDaemonStartupPhaseMarkers(tail))).toEqual(['composition-loaded']); - }); +test('a crash-replay append keeps the file valid JSONL', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999); + await appendFile(markerPath, '{"data":{"startupPhase":"modules-lo', 'utf8'); + recordDaemonStartupMarker(markerPath, 'composition-loaded'); + const tail = await readFile(markerPath, 'utf8'); + expect(Object.keys(readDaemonStartupPhaseMarkers(tail))).toEqual(['composition-loaded']); }); function spawnDeadPid(): number { - // A process that has already exited proves the liveness check rejects it, - // the same way the daemon-process takeover guards prove it. const child = spawnSync(process.execPath, ['-e', ''], { encoding: 'utf8' }); if (child.error) throw child.error; const pid = child.pid; diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 9734bfde77..4b933cb054 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -49,18 +49,10 @@ import { export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); /** - * Startup phase markers (#3782): the record a daemon writes beside its registration of how far its - * startup got. The client waiting for registration reads this record to tell "slow but progressing" - * from "stuck". One launch owns exactly one file, named for the pid that wrote it: an earlier or a - * concurrent launch cannot add to, replace, or hide another launch's markers, and neither the file - * nor a waiting client ever needs a timestamp to decide which startup a marker belongs to. A phase - * is recorded when the work it names has finished; the record never carries a heartbeat, so silence - * stays what a stalled startup looks like. - * - * Ownership, so the record never becomes residue: the writer removes the file once its startup - * completes; the client that launched a process removes it once that process's exit is joined; and - * each new registration owner sweeps the files of provably dead pids when it takes the lock. Each - * marker is one JSONL line appended to the launch's own file. + * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup + * got, so a waiting client tells "slow but progressing" from "stuck" without any timestamp, and + * never a heartbeat. Ownership, so it never becomes residue: the writer removes it on completion, + * the launching client once the exit is joined, each new registration owner sweeps dead pids. */ const DAEMON_STARTUP_PHASES = Object.freeze([ 'modules-loaded', @@ -83,12 +75,8 @@ export function resolveDaemonStartupPhasesPath(stateDir: string, pid: number): s } /** - * Appends one phase marker for the launch owning `startupPhasesPath`. A marker is progress - * evidence, never a startup requirement: a host that refuses the append must not lose a daemon - * that can still register. Producers that own a diagnostics channel pass `reportFailure` so the - * refusal is observable where the waiting client already reads failure evidence (the daemon log - * tail in the startup failure details); a client never receives a false marker from a broken - * channel — it reads a missing record as no evidence. + * Appends one phase marker. A refused append must not lose a daemon that can still register, so + * `reportFailure` carries it and a client reads a missing record as no evidence. */ export function recordDaemonStartupMarker( startupPhasesPath: string, @@ -98,8 +86,7 @@ export function recordDaemonStartupMarker( try { const timestampMs = Date.now(); fs.mkdirSync(path.dirname(startupPhasesPath), { recursive: true }); - // A crash mid-append can leave the file without a trailing newline; without a separator the - // next append would glue into one unparsable line and lose both records. + // A crash mid-append can leave no trailing newline; glue two appends and both are lost. const separator = endsWithoutNewline(startupPhasesPath) ? '\n' : ''; fs.appendFileSync( startupPhasesPath, @@ -127,13 +114,9 @@ function endsWithoutNewline(file: string): boolean { } /** - * Establishes a fresh record for this launch at `startupPhasesPath`, before the producer's first - * marker. A per-pid file names its writer, not its lifetime: if both the client and the daemon of - * an old launch died before either cleanup ran and the host later reuses that pid, the old record - * is still on disk, and only this new writer can tell a same-name file apart from its own record. - * It therefore empties its own pid's record before appending; no caller claims or resets a file - * named for another pid. A refused reset follows the marker contract: reported, never fatal, and - * a later append cannot silently mix lifetimes because the reset ran first when it could run. + * Empties this launch's own record before its first marker: a per-pid name identifies a writer, + * not a lifetime, so after pid reuse only this new writer can tell a same-name file apart from + * its own. No caller resets a file named for another pid; a refused reset is reported, never fatal. */ export function claimDaemonStartupPhasesFile( startupPhasesPath: string, @@ -146,25 +129,14 @@ export function claimDaemonStartupPhasesFile( } } -/** - * Removes one startup's marker file. The writer calls it once its own startup completes; the client - * that launched a process calls it once that process's exit is joined. Both callers can prove no - * other writer can still append to this file, which is what the pid-specific name encodes. - */ +/** Removes a marker file whose writer is provably gone: a finished startup or a joined exit. */ export function removeDaemonStartupPhasesFile(startupPhasesPath: string): void { try { fs.rmSync(startupPhasesPath, { force: true }); - } catch { - // Evidence removal never fails a daemon that is already up or a client that already joined. - } + } catch {} } -/** - * Sweeps marker files left by provably dead launches, from the position of a process that holds the - * registration lock. A live pid's file is kept: its launch may still be between phases, and only - * that launch's own completion or a joined exit may remove it. A file whose name carries no usable - * pid cannot be owned by any launch, so this owner retires it too. - */ +/** Sweeps marker files of provably dead (or unnameable) pids; a live pid's file is kept. */ function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { let entries: string[]; try { @@ -184,9 +156,8 @@ function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { } /** - * The phases observed in one launch's marker tail, keyed by phase, holding each phase's earliest - * occurrence. Unparsable and unknown lines are ignored: the protocol is additive, and a client must - * never fail a startup over a line shape it does not know. + * The phases observed in a marker tail, keeping each phase's earliest occurrence. Unknown and + * unparsable lines are ignored: a client never fails a startup over a line shape it doesn't know. */ export function readDaemonStartupPhaseMarkers( markerTail: string, @@ -258,8 +229,6 @@ export async function tryAcquireDaemonRegistration( try { acquisition.assertHeld(); fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true }); - // A dead launch's markers are this owner's to sweep: only the launch itself or a client that - // joined its exit may remove a live launch's file, and neither can still reach this one. sweepDeadDaemonStartupPhasesFiles(boundPaths.baseDir); } catch (error) { await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 416a072994..eec544823a 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -270,10 +270,7 @@ async function startDaemonRuntimeOnHost( const { env } = host; const stdout = options.stdout ?? process.stdout; const stderr = options.stderr ?? process.stderr; - // The daemon-log diagnostics and registration-completion helpers enter behind a - // function-scoped `await import`: they are startup/lifecycle evidence work this server - // performs after composition, and the daemon entry's eager closure stays exactly the - // operational server's (ADR 0019). This is their first possible use point. + // Function-scoped import: these helpers stay out of the daemon entry's eager closure (ADR 0019). const { armDaemonMetadataLossWatch, emitDaemonDiagnostic, @@ -593,14 +590,9 @@ async function startDaemonRuntimeOnHost( rollback.onRollback(async () => { await finishDaemonRegistration({ registration, infoPath, logPath }); }); - // #3782: the client waiting on this state dir reads these markers to tell a slow daemon from a - // stuck one. Each marker names a phase whose work has finished; this launch's file is named for - // its own pid, so no other launch's evidence can add to or hide it. const startupPhasesPath = resolveDaemonStartupPhasesPath(baseDir, process.pid); const recordStartupPhase = (phase: Parameters[1]): void => recordDaemonStartupMarker(startupPhasesPath, phase, (error) => { - // Same contract as the entry's markers: the refusal goes where the waiting client already - // reads failure evidence (the daemon log), and startup keeps going. stderr.write(`Daemon startup marker write failed: ${error.message}\n`); }); recordStartupPhase('lock-acquired'); @@ -658,9 +650,6 @@ async function startDaemonRuntimeOnHost( data: { error: error instanceof Error ? error.message : String(error) }, }); }); - // #3782: the recovery chain above (owner-lifecycle configuration, legacy-marker and app-log - // recovery, owned-process reaping, web-orphan cleanup) is real completed work: its boundary - // extends the stall window exactly as much as it took, never more. recordStartupPhase('recovery-finished'); const { openDaemonServers, reconcileDeviceClaimsForDaemonStartup } = await import('./server-startup.ts'); @@ -701,13 +690,9 @@ async function startDaemonRuntimeOnHost( }, rollback, ); - // #3782: the last two progress markers. A client that polls the registration and the markers - // together can tell "listening but slow to answer" from "never progressed". recordStartupPhase('listening'); publishDaemonInfo(socketPort, httpPort); recordStartupPhase('registered'); - // This startup's evidence has done its job: no waiting client needs it anymore, and this - // process is the only writer of the file, so this is the removal its owner contract allows. removeDaemonStartupPhasesFile(startupPhasesPath); stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); diff --git a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts index 3d8c75a3b7..ba5ccbc2d9 100644 --- a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts +++ b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts @@ -21,11 +21,7 @@ export async function flushDaemonStartupDiagnostics( }); } -/** - * Records one daemon-level event. These run outside any request, so there is no request scope and no - * resolved debug level to inherit; debug is forced on for the same reason the #2681 handoff forces it — - * the event is the point of the record and must not be dropped by a level that was never set for it. - */ +/** Records one daemon-level event outside any request scope, with debug forced on. */ export async function emitDaemonDiagnostic( logPath: string, phase: string, @@ -59,11 +55,7 @@ export async function finishDaemonRegistration(params: { } } -/** - * Starts the watch that reports this daemon's registration being taken over, and returns the handle - * that stops it. It is armed only once this process has published its own record: before publication - * the file legitimately describes a predecessor, and losing that is not this daemon's event. - */ +/** Watches this registration being taken over; arm only after this process published. */ export function armDaemonMetadataLossWatch( stateDir: string, infoPath: string, diff --git a/src/__tests__/test-utils/registered-daemon-fixture.test.ts b/src/__tests__/test-utils/registered-daemon-fixture.test.ts index ced6b449d0..59592da4aa 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.test.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.test.ts @@ -18,11 +18,8 @@ import { } from './registered-daemon-fixture.ts'; test('args built before any launch keep their own entry script under a concurrent second build', async () => { - // #3791 review: a second registeredDaemonFixtureArgs call must not rewrite the script file an - // earlier call already generated. Both arg sets are built before either child starts (that build - // order is the overwrite window); child 1 then runs its own switches and child 2 exits busy. - // Under a shared entry path, child 1 would execute child 2's script and its own record would - // carry child 2's marker switch instead of its own. + // Both arg sets are built before either child starts — the overwrite window. Under a shared + // entry path child 1 would run child 2's switches and its record would carry the wrong marker. const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-fixture-entry-isolation-')); fs.mkdirSync(paths.baseDir, { recursive: true }); fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); @@ -42,8 +39,6 @@ test('args built before any launch keep their own entry script under a concurren assert.notEqual(firstArgs[1], secondArgs[1]); const first = launchRegisteredDaemonFixtureArgs(paths, firstArgs, { stdio: 'ignore' }); try { - // Child 1 holds the registration before child 2 starts, so child 2's busy exit is ordered, - // not scheduled. const held = path.join(paths.baseDir, `registration-held-${first.pid}`); for (let attempt = 0; !fs.existsSync(held); attempt += 1) { assert.ok(attempt < 400, 'child 1 never acquired the registration'); diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index c865280101..a29061913a 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -20,16 +20,9 @@ const children = new Map< /** Per-child behavior switches, bound into this child's entry script at write time. */ export type RegisteredDaemonFixtureOptions = Readonly<{ - /** - * Record `lock-acquired` through the real protocol after this child acquires the registration, - * as the daemon runtime does. Scoped to this child: no later fixture in the same state dir - * inherits it. - */ + /** Record `lock-acquired` after acquisition, as the daemon runtime does, for this child only. */ emitStartupMarker?: boolean; - /** - * Record `modules-loaded` through the real protocol before acquisition attempts, as the daemon - * entry does, so a child that exits busy or unproven leaves a real pre-lock marker behind. - */ + /** Record `modules-loaded` before acquisition, as the daemon entry does, for this child only. */ emitStartupMarkerBeforeAcquisition?: boolean; }>; @@ -40,10 +33,8 @@ export function registeredDaemonFixtureArgs( acquisitionBarrier?: string, fixtureOptions?: RegisteredDaemonFixtureOptions, ): string[] { - // One directory per construction: a later call must never overwrite a script an earlier - // launched child has not loaded yet. The per-launch directory still ends in the real daemon - // entry path so the daemon-command identity checks classify this child exactly as they - // classify a real daemon. + // One directory per launch so a later build cannot overwrite a script an earlier child has + // not loaded; the path still ends in the real daemon entry name for the identity checks. const launchDir = path.join(paths.baseDir, 'fixture-entries', crypto.randomUUID()); const entry = path.join(launchDir, 'dist', 'src', 'internal', 'daemon.js'); fs.mkdirSync(path.dirname(entry), { recursive: true }); diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index fc2e851830..4d8baca2cb 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -1,8 +1,3 @@ -// #3782: the local-daemon startup budget. A live daemon that starts slowly is waited for; a -// daemon that exits is failed fast; a live daemon that publishes no startup progress is retired at -// the stall window, not at the ceiling; and the ceiling itself is operator-settable. These cases -// pin each outcome at the `sendToDaemon` transport boundary, the route CLI, MCP, and Node share. - import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; @@ -79,12 +74,7 @@ function fields(httpPort: number) { }; } -/** - * A registered-fixture daemon with deferred publication: it acquires the registration, publishes - * no daemon.json, and never exits on its own — the startup the client must judge from process - * liveness and startup markers alone. Its command matches the daemon identity rules, so the - * client's retirement signals it and it exits, keeping each test's fixtures disposable. - */ +/** A deferred-publication fixture that never exits on its own; retirement can still stop it. */ function spawnUnregisteredDaemonChild( paths: DaemonPaths, options: Parameters[2], @@ -92,9 +82,6 @@ function spawnUnregisteredDaemonChild( ): ReturnType { fs.mkdirSync(paths.baseDir, { recursive: true }); fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); - // When asked, the child records its phases through the real protocol, into the file named for - // its own pid, exactly as the daemon entry does. The request is bound into this child's entry - // script, so no later fixture in this state dir inherits it. return spawnRegisteredDaemonFixture(paths, fields(1), options, undefined, { emitStartupMarker }); } @@ -124,7 +111,6 @@ function startupPhases(failure: AppError): Record { return failure.details.phases as Record; } -/** Waits (on the unmocked clock) for one line of the client's own sleep seam. */ async function realPause(ms = 5) { await actualRetry.sleep(ms); } @@ -140,11 +126,8 @@ test('a daemon slower than the retired 15 s budget starts successfully while it if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const http = await startHttpDaemonFixture({ devices: [] }); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-slow-success-')); - // Virtual time: the client waits ~20 s of startup — past the retired 15 s deadline — in real - // milliseconds. The clock stands still while the real fixture child is still booting (marker - // timestamps must come from this launch), and a progress marker every ~6 virtual seconds keeps - // the wait outside the 60 s stall window. Markers go to this launch's own pid-named file, which - // is where a real daemon writes them. + // Virtual time waits ~20 s — past the retired 15 s deadline — while markers arrive in real + // time from this launch's own pid-named file, keeping the wait outside the stall window. const startedAt = Date.now(); const now = { value: startedAt }; vi.spyOn(Date, 'now').mockImplementation(() => now.value); @@ -175,8 +158,6 @@ test('a daemon slower than the retired 15 s budget starts successfully while it fs.rmSync(path.join(paths.baseDir, 'defer-publication'), { force: true }); return await realPause(); } - // The client has waited past the retired 15 s budget; hold the virtual clock while the real - // child notices the lifted barrier and publishes, polling at full real speed. await realPause(20); }); try { @@ -199,12 +180,8 @@ test('a daemon that exits during startup fails with startup-exited within one se let resolveFirstExit: ((exit: ExecDetachedExit) => void) | undefined; let exitObservedAt = 0; spawn.mockImplementation(() => { - // Fabricated pid: the retirement path proves daemon identity through the process table, finds - // no such process, and signals nothing. const pid = 998_231 + spawn.mock.calls.length; if (spawn.mock.calls.length === 1) { - // The exit arrives ~400 ms of real time after the spawn: the client must observe the - // monitor and report within its one-second detection budget. const exited = new Promise((resolve) => { resolveFirstExit = resolve; }); @@ -242,8 +219,7 @@ test('a live daemon that publishes no startup marker is retired as stalled befor booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); if (!booted) return await realPause(); } - // Advance the virtual clock to the stall window; freeze it afterward so the real-time - // retirement below cannot stretch the measured detection past it. + // Freeze the clock past the window so real-time retirement cannot stretch the measurement. if (now.value - started < 70_000) now.value += 5_000; await realPause(); }); @@ -259,9 +235,7 @@ test('a live daemon that publishes no startup marker is retired as stalled befor assert.equal(startupPhases(failure).spawned, 0); assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); assert.equal(spawn.mock.calls.length, 1, 'a stalled daemon is not retried'); - // #3782 residue rule: the retired launch's exit was joined, so its marker file is removed — - // a startup the client owned must leave no evidence behind. The prefix check is name-agnostic - // so no marker-file shape can pass by accident. + // The joined exit means this owned startup must leave no evidence behind. await (child?.exited ?? Promise.resolve()); assert.deepEqual( fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), @@ -294,9 +268,6 @@ test('a live daemon that keeps progressing but never registers fails at the env booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); if (!booted) return await realPause(); } - // The real child has published its own `lock-acquired` marker: progress this wait keeps - // extending is the daemon's, not the test's. Virtual time advances 1 s per poll; the 30 s - // env ceiling stops the wait well before the 60 s stall window can. if (now.value - started < 30_000) now.value += 1_000; await realPause(); }); @@ -319,17 +290,11 @@ test('a live daemon that keeps progressing but never registers fails at the env }); test("a startup progressed by a fresh launch is not judged stalled by an earlier launch's leftover markers", async () => { - // F1 (#3782): a reused state dir still holds the phase record a dead earlier startup wrote. The - // fresh launch records its own progress through the real protocol, from its own process. The - // shared-record reader kept the earliest occurrence per phase, so the older record hid the fresh - // launch's `lock-acquired` from the failure details entirely — the exact loss of attribution that - // turns a progressing startup into an empty observation and a false stall. The state dir must - // also sweep what its own launches leave behind. + // A reused state dir holds a dead startup's record; the fresh launch must see its own markers. vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '30000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-stale-history-')); fs.mkdirSync(paths.baseDir, { recursive: true }); - // The leftover launch: a real daemon-shaped child that records its phase, is SIGTERMed before - // publishing, and leaves its record behind exactly as a crashed startup would. + // The leftover: a real daemon child SIGTERMed before publishing, record left behind. fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); const leftover = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }, undefined, { emitStartupMarker: true, @@ -343,7 +308,6 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier leftoverRecords().length > 0, 'the leftover launch must have left a real daemon-written marker record', ); - const started = Date.now(); const now = { value: started }; vi.spyOn(Date, 'now').mockImplementation(() => now.value); @@ -358,8 +322,6 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier booted = fs.existsSync(registrationHeldFor(paths, child.pid)); if (!booted) return await realPause(); } - // The 30 s env ceiling stops this wait before the 60 s stall window can; the fresh child's own - // `lock-acquired` marker is the only progress evidence and must reach the failure record. if (now.value - started < 30_000) now.value += 1_000; await realPause(); }); @@ -377,29 +339,75 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier !leftoverRecords().some((entry) => entry.includes(String(child?.pid))), "the joined exit must sweep this launch's own marker file", ); - // A dead launch's record that survived every sweep so far: written through the real protocol - // against a provably dead pid after the fresh launch's own acquisition sweep, so only the next - // registration owner can remove it. - recordDaemonStartupMarker(resolveDaemonStartupPhasesPath(paths.baseDir, 998_776), 'listening'); - const cleaner = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }); - try { - await awaitFile(registrationHeldFor(paths, cleaner.pid)); - assert.deepEqual( - leftoverRecords(), - [], - "acquiring the registration sweeps every dead launch's marker record", - ); - } finally { - process.kill(cleaner.pid, 'SIGTERM'); - await cleaner.exited; - } } finally { - // The client's own retirement (or the sweep test's SIGTERM) already stopped this child; the - // hermetic signal guard refuses any signal the worker did not need to send. await (child?.exited ?? Promise.resolve()); } }); +test('a busy exit while a live lock holder defers publication reaches the ceiling, not startup-exited', async () => { + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-holder-ceiling-')); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const winner = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }, undefined, { + emitStartupMarker: true, + }); + await awaitFile(registrationHeldFor(paths, winner.pid)); + let contender: ReturnType | undefined; + spawn.mockImplementation((_command, _args, options) => { + contender = spawnRegisteredDaemonFixture(paths, fields(1), options, undefined, { + emitStartupMarkerBeforeAcquisition: true, + }); + return contender; + }); + pause.mockImplementation(async () => { + await realPause(20); + }); + const signal = vi.spyOn(process, 'kill'); + try { + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-ceiling'); + assert.equal(failure.details?.startupTimeoutMs, 2_000); + assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); + assert.equal(typeof startupPhases(failure)['modules-loaded'], 'number'); + // The winner stays running and unsignaled with its record; the contender's is swept. + assert.equal(isProcessAlive(winner.pid), true); + assert.equal( + signal.mock.calls.some(([pid, kind]) => pid === winner.pid && kind !== 0), + false, + ); + assert.ok(contender); + assert.deepEqual( + fs + .readdirSync(paths.baseDir) + .filter((entry) => entry.startsWith('daemon-startup-phases')) + .sort(), + [path.basename(resolveDaemonStartupPhasesPath(paths.baseDir, winner.pid))], + 'the exited contender record must be swept; only the live winner record may remain', + ); + } finally { + signal.mockRestore(); + fs.rmSync(deferred, { force: true }); + await (contender?.exited ?? Promise.resolve()); + } +}); + +test('a child gone at the deadline without an observed exit and no live startup fails exited', async () => { + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-gone-at-deadline-')); + spawn.mockImplementation(() => ({ + pid: 997_842, + exited: new Promise(() => {}), + readIdentity: () => ({ pid: 997_842, startTime: null }), + })); + pause.mockImplementation(async () => { + await realPause(50); + }); + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-exited'); + assert.equal(failure.details?.startupTimeoutMs, 2_000); +}); + test('an unproven registration exit reports startup-exited even under a short env ceiling', async () => { vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-exit-under-ceiling-')); diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index b0f6727c70..aee9d149eb 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -386,11 +386,7 @@ for (const exit of [ } test('a busy contender that exits sweeps its own pre-lock marker before the retried attempt', async (t) => { - // #3782 residue rule on the retry path: the real daemon entry records `modules-loaded` before it - // acquires the registration, so a contender that exits busy has a record of its own. The client - // that launched it joins the exact exited child and removes only that child's file at the retry - // boundary; the second launch here is refused at the spawn seam, so nothing else can sweep the - // state dir and the assertion below can only pass through the client's own cleanup. + // The second launch is refused at the spawn seam, so only the client's own cleanup can pass. if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-busy-retry-marker-')); const http = await startHttpDaemonFixture({ devices: [] }); @@ -411,8 +407,6 @@ test('a busy contender that exits sweeps its own pre-lock marker before the retr void child.exited.then((exit) => { assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); if (!released) { - // The lock only frees once the real contender has exited busy, so this child always - // loses the registration and always leaves its pre-lock record behind. void claim.acquisition.release().then(() => { released = true; }); @@ -443,9 +437,6 @@ test('a busy contender that exits sweeps its own pre-lock marker before the retr }); test('attaching to a contending winner sweeps the busy contender it exited as', async (t) => { - // #3782 residue rule on the winner-attach path: `ready` with startedByClient false is reached - // only after this client's launch exited busy. That record belongs to the client to remove, and - // the live winner never sweeps it (sweeps run at acquisition, and the winner acquired first). if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-winner-attach-marker-')); const http = await startHttpDaemonFixture({ devices: [] }); @@ -464,8 +455,6 @@ test('attaching to a contending winner sweeps the busy contender it exited as', return child; }); pause.mockImplementation(async () => { - // The contender is only refused while the winner still publishes nothing; its busy exit then - // releases the winner to publish through the unmocked clock. if (contenderExit) fs.rmSync(deferred, { force: true }); await actualRetry.sleep(10); }); @@ -528,8 +517,6 @@ test('a joined busy contender waits for a published winner to become ready witho for (const held of [true, false]) { test(`startup uses one deadline when the claim is ${held ? 'held' : 'released for relaunch'}`, async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-budget-')); - // The one-deadline behavior is budget-independent; pin the ceiling so the test keeps its - // original 15 s shape while the production default is the #3782 ceiling. vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '15000'); const claim = tryAcquireProcessLock({ lockDirPath: paths.lockPath, diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index c4d4584919..7d45613d11 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -42,7 +42,6 @@ import { } from './daemon-lifecycle-errors.ts'; import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; -/** Whether the daemon a state dir registered is reusable by this client, and what to do if not. */ type LocalDaemonObservation = { info: DaemonInfo; decision: DaemonTakeoverDecision; @@ -52,38 +51,30 @@ type DaemonStartupWaitResult = | { kind: 'ready'; daemon: EnsuredDaemon } | { kind: 'early_exit'; exit: ExecDetachedExit } | { kind: 'unproven'; error: AppError } - | { kind: 'retry' | 'timeout' } + | { kind: 'retry' } + | { kind: 'exit_unobserved' } /** The process stayed alive but published no new startup phase within the stall window. */ | { kind: 'stalled' } /** The process stayed alive and kept progressing, but did not register before the ceiling. */ | { kind: 'ceiling' }; -/** Startup phase markers one launch published, as absolute host-clock timestamps. */ type StartupPhaseObservation = { spawnedAtMs: number; markers: Partial>; }; /** - * #3782: the ceiling on one local daemon startup, shared by the CLI, MCP, and Node client, because - * all three reach the daemon through `startLocalDaemon`. A slow host makes startup slower, not - * failed: while the spawned daemon stays alive and keeps publishing phase markers, the client keeps - * waiting. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides the ceiling for a caller that must - * fail sooner than the host allows. + * The ceiling on one local daemon startup: a live daemon that keeps publishing markers is waited + * for. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides it. */ const DAEMON_STARTUP_CEILING_MS = 120_000; -/** - * How long a live daemon may publish no new startup phase marker before the client stops waiting. - * Kept beside the ceiling as one pair: a stalled daemon is retired at the stall window, a - * progressing one is waited for up to the ceiling. An env ceiling below this window makes the - * ceiling the only stop; the stall never fires past the ceiling, only earlier than it. - */ +/** Silence this long ends the wait early; an env ceiling below it makes the ceiling the stop. */ const DAEMON_STARTUP_STALL_WINDOW_MS = 60_000; const MINIMUM_DAEMON_TAKEOVER_BUDGET_MS = 5_000; const DAEMON_STARTUP_TIMEOUT_ENV = 'AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS'; -/** Why a startup that never reached readiness failed, in terms a caller can branch on (#3782). */ +/** Why a startup that never reached readiness failed, in terms a caller can branch on. */ type DaemonStartupFailureReason = 'startup-exited' | 'startup-stalled' | 'startup-ceiling'; function resolveDaemonStartupTimeoutMs(env: Readonly): number { @@ -107,16 +98,14 @@ const DAEMON_STARTUP_LOG_TAIL_BYTES = 64_000; const DAEMON_STARTUP_MARKER_TAIL_BYTES = 16_000; /** - * Reads the registered daemon and judges it against this client, changing nothing. Whether the - * judgement is acted on belongs to the caller's lifecycle: a managed client replaces, an attach - * client reports, `ensureDaemon` replaces only through a quiesce. + * Judges the registered daemon against this client, changing nothing; acting on the judgement + * belongs to the caller's lifecycle. */ export async function observeLocalDaemon( settings: DaemonClientSettings, options: { deadline?: number; waitForLiveStartup?: boolean; - /** Tell a live daemon on another transport from one that answers nowhere. */ refineUnreachable?: boolean; } = {}, ): Promise { @@ -137,11 +126,7 @@ export async function observeLocalDaemon( return { info: existing, decision: await refineReplacement(existing, decision, options) }; } -/** - * Tells apart what the ladder lumps together, for callers that must not kill a daemon on a guess: a - * registration whose process is gone is unreachable whatever its version, and one that answers on - * another transport is alive. - */ +/** Separates a gone registration from a live one answering on another transport. */ async function refineReplacement( info: DaemonInfo, decision: DaemonTakeoverDecision, @@ -234,12 +219,8 @@ export async function retireDaemonForTakeover( } /** - * A daemon whose pid is still alive is probed again before it can be judged unreachable. A probe's - * budget is wall-clock time on this client's event loop, so a client that stalls past it (a large - * synchronous parse, a GC pause on a loaded host) reads a listening daemon as unreachable, and - * replacing it ends every session the daemon holds. Liveness is the signal-0 check, not the `ps` - * identity read: under the load that stalls the probe, `ps` misses its deadline too, and the - * takeover still proves identity before it signals anything. + * A live pid is probed again before being judged unreachable: this client's own event-loop stall + * must not read a listening daemon as gone. Liveness uses signal 0, which a busy `ps` cannot miss. */ async function canReachReusableDaemon( info: DaemonInfo, @@ -324,9 +305,7 @@ function emitDaemonTakeoverNotice(info: DaemonInfo, reason: string, stateDir: st try { const identity = info.version ? `pid ${info.pid}, v${info.version}` : `pid ${info.pid}`; process.stderr.write(`Replacing daemon (${identity}) in ${stateDir}: ${reason}\n`); - } catch { - // The takeover notice is best effort; never fail the command on stderr issues. - } + } catch {} } type FailedDaemonStartup = { @@ -335,7 +314,6 @@ type FailedDaemonStartup = { startupError?: NormalizedError; daemonProcess?: ExecDetachedExit | { pid: number }; retry: boolean; - /** Set when the wait ended on startup progress rather than on an exit or a race. */ reason?: DaemonStartupFailureReason; phases?: StartupPhaseObservation; }; @@ -397,8 +375,8 @@ async function attemptLocalDaemonStartup( const observation: StartupPhaseObservation = { spawnedAtMs, markers: {} }; const startup = await waitForDaemonStartup(deadline, settings, launch, observation); if (startup.kind === 'ready') { - // A contending winner's readiness (startedByClient false) is only reachable after this launch - // exited busy: its own record is this client's to sweep before the handoff. + // Winner readiness (startedByClient false) is only reachable after this launch exited busy: + // its own record is this client's to sweep before the handoff. if (!startup.daemon.startedByClient) await discardExitedLaunchRecord(settings, launch); return { daemon: startup.daemon }; } @@ -475,10 +453,7 @@ async function retireStartupAttempt( lockTimeoutMs: 0, }); const joined = await joinStartup(launch); - // The launch's exit is joined, so nothing can append to its marker file anymore, and the file - // has served its last purpose (the failure record carries the phases). Removing it here is what - // keeps a retired startup's evidence from becoming state-dir residue. Without a joined exit the - // file stays: its writer may still be alive, and only that writer or a later joined exit owns it. + // A joined exit is what licenses removing the record; without it the file stays. if (joined) removeDaemonStartupPhasesFile( resolveDaemonStartupPhasesPath(settings.paths.baseDir, launch.pid), @@ -500,13 +475,7 @@ async function joinStartup(launch: DaemonStartupLaunch): Promise { } } -/** - * #3782: sweeps the record of a launch whose exit this client just proved — a busy or unproven - * exit, or a launch that lost the registration to a contending winner. It joins this exact child - * first and removes only this child's own pid-named file, the same ownership rule a retired - * startup follows; an unjoined exit keeps its record for its writer or a later joined exit. The - * current winner's registration and record are untouched. - */ +/** Sweeps a proven-exited launch's own record, after joining that exact child. */ async function discardExitedLaunchRecord( settings: DaemonClientSettings, launch: DaemonStartupLaunch, @@ -547,16 +516,25 @@ async function waitForDaemonStartup( if (settled) return settled; } else { const settled = await observeLiveDaemonStartup(settings, launch, deadline, observation); - // An exit observed while this iteration was in flight outranks a `ready` read: the process - // that published is gone, and its classification owns the next step. + // An in-flight exit outranks a `ready` read: the publisher is gone. if (settled && !(settled.kind === 'ready' && earlyExit)) return settled; } await sleep(Math.min(100, remainingStartupBudget(deadline))); } - if (earlyExit || !isProcessAlive(launch.pid)) return { kind: 'timeout' }; - // A live daemon at the ceiling registered late, not never: only its retirement is this deadline's - // failure, and the reason says the host, not the daemon, ran out of room. - return { kind: 'ceiling' }; + // A deadline passing while any startup is live is the host running out of room, never an exit. + if (isProcessAlive(launch.pid) || isLiveLockOwner(inspectProcessLock(settings.paths.lockPath))) + return { kind: 'ceiling' }; + if (earlyExit) return { kind: 'early_exit', exit: earlyExit }; + return { kind: 'exit_unobserved' }; +} + +/** A lock owner is a pending startup unless provably dead or reused; never signals anything. */ +function isLiveLockOwner(inspection: ProcessLockInspection): boolean { + return ( + inspection.state === 'held' && + inspection.liveness !== 'owner-process-dead' && + inspection.liveness !== 'owner-process-reused' + ); } async function judgeExitedDaemonStartup( @@ -579,8 +557,7 @@ async function observeLiveDaemonStartup( ): Promise { const info = await readReadyLaunchedDaemon(settings, launch, deadline); if (info) return { kind: 'ready', daemon: { info, startedByClient: true } }; - // #3782: a live daemon that keeps publishing startup markers is slow, not stuck, and is - // waited for up to the ceiling. Only silence for a full stall window retires it early. + // A daemon still publishing markers is slow, not stuck; only silence retires it early. refreshStartupPhaseMarkers(settings, launch.pid, observation); if ( isProcessAlive(launch.pid) && @@ -591,10 +568,7 @@ async function observeLiveDaemonStartup( return null; } -/** - * #3782: folds this launch's startup phase markers into `observation`. The record is one launch's - * own file, named for the pid that wrote it, so no other launch's evidence can add to or hide it. - */ +/** Folds this launch's markers into `observation` from its own pid-named record. */ function refreshStartupPhaseMarkers( settings: DaemonClientSettings, launchPid: number, @@ -707,7 +681,6 @@ async function readReadyLaunchedDaemon( } } -/** Whether `info` names the daemon process this client launched: same pid and start time. */ function isLaunchedDaemon(info: DaemonInfo, launch: DaemonStartupLaunch): boolean { const identity = launch.readIdentity(); return ( diff --git a/src/daemon.ts b/src/daemon.ts index c3773a6922..b522dcf9d2 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -12,34 +12,20 @@ import { } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -// Marker failures go to stderr, which the launcher redirects into the daemon log — the same tail a -// startup failure already ships as evidence. A marker is evidence, not a requirement, so a refused -// append never stops a startup that can still register; it must only stay observable. +// Marker failures go to stderr, which the launcher redirects into the daemon log; a refused +// append is evidence, never a startup requirement. function reportMarkerFailure(error: unknown): void { process.stderr.write(`Daemon startup marker write failed: ${normalizeError(error).message}\n`); } -// #3782: the first marker the waiting client can read. It says this entry's module graph evaluated, -// which on an overloaded host is the slowest thing a daemon start does before it can prove itself. -// The file is named for this process's pid, so the client attributes every line to this launch. -// The claim first: a host that reused this pid after an old launch died before either cleanup ran -// would otherwise leave that lifetime's phases in this same file, and this launch's timings would -// read as older and farther-progressed than they are. -claimDaemonStartupPhasesFile( - resolveDaemonStartupPhasesPath( - resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, - process.pid, - ), - reportMarkerFailure, -); -recordDaemonStartupMarker( - resolveDaemonStartupPhasesPath( - resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, - process.pid, - ), - 'modules-loaded', - reportMarkerFailure, +const ownPhasesPath = resolveDaemonStartupPhasesPath( + resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, + process.pid, ); +// The claim first: after pid reuse it is the only defense against reading a dead lifetime's +// phases as this launch's own. +claimDaemonStartupPhasesFile(ownPhasesPath, reportMarkerFailure); +recordDaemonStartupMarker(ownPhasesPath, 'modules-loaded', reportMarkerFailure); // The executable owns the process: signals, fatal-error handlers and the exit code. The runtime // only reports when it has closed, whoever closed it. From 23601ca6df500ccc3a84154eac20c08df064d597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 03:32:33 +0200 Subject: [PATCH 15/22] fix(daemon): classify the expired startup wait by its observed exit and pid identity At an expired deadline, a delivered child exit is authoritative for the lifetime it watched, so it outranks an unobserved liveness probe of a pid that may have been reused. Unobserved, the launch's recorded birth time must prove the pid changed hands before it counts as a live startup; an unavailable identity stays fail-closed. Extracted classifyExpiredDaemonStartup keeps the wait loop within its complexity thresholds. The budget test pins liveness of its fabricated pids at the process seam instead of trusting the host, and the winner-sweep race coverage is deduplicated into the joined contender test. --- .github/workflows/ios.yml | 3 +- ...registration-owner-startup-markers.test.ts | 1 - .../src/daemon-registration-owner.ts | 16 ++---- .../daemon-client-startup-budget.test.ts | 50 +++++++++++++++-- .../daemon-client-startup-race.test.ts | 56 ++++--------------- src/daemon-client/daemon-client-startup.ts | 51 ++++++++--------- src/daemon.ts | 15 ++--- 7 files changed, 94 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 36323451fd..873399c608 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -169,8 +169,7 @@ jobs: - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - # The public-SDK timing record is the cold-startup evidence; the CLI `daemon_startup` - # timer only wraps attach, after the ensured transport already started the daemon. + # The public-SDK record is the cold-startup evidence; the CLI timer only wraps attach. node --input-type=module -e ' import { performance } from "node:perf_hooks"; const { ensureDaemon } = await import("./dist/src/index.js"); diff --git a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts index 783e927e36..aa4f99fb49 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts @@ -108,7 +108,6 @@ test('acquiring the registration retires files of dead or unowned pids only', as expect(existsSync(dead)).toBe(false); expect(existsSync(unparseable)).toBe(false); expect(existsSync(own)).toBe(true); - // The sweep retires only its own kind: an unknown entry stays as evidence. expect(existsSync(legacy)).toBe(true); await attempt.owner.finish(); }); diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 4b933cb054..7e3310b2d9 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -50,9 +50,8 @@ export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 /** * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup - * got, so a waiting client tells "slow but progressing" from "stuck" without any timestamp, and - * never a heartbeat. Ownership, so it never becomes residue: the writer removes it on completion, - * the launching client once the exit is joined, each new registration owner sweeps dead pids. + * got, so a waiting client tells "slow but progressing" from "stuck" without a timestamp and with + * no heartbeat. The writer removes it on completion, the launching client after a joined exit. */ const DAEMON_STARTUP_PHASES = Object.freeze([ 'modules-loaded', @@ -74,10 +73,7 @@ export function resolveDaemonStartupPhasesPath(stateDir: string, pid: number): s return path.join(stateDir, `daemon-startup-phases-${pid}.jsonl`); } -/** - * Appends one phase marker. A refused append must not lose a daemon that can still register, so - * `reportFailure` carries it and a client reads a missing record as no evidence. - */ +/** Appends one phase marker; a refused append goes to `reportFailure`, never fatal to a startup. */ export function recordDaemonStartupMarker( startupPhasesPath: string, phase: DaemonStartupPhase, @@ -115,8 +111,8 @@ function endsWithoutNewline(file: string): boolean { /** * Empties this launch's own record before its first marker: a per-pid name identifies a writer, - * not a lifetime, so after pid reuse only this new writer can tell a same-name file apart from - * its own. No caller resets a file named for another pid; a refused reset is reported, never fatal. + * not a lifetime. No caller resets a file named for another pid; a refused reset never stops the + * startup because the reset ran first when it could run. */ export function claimDaemonStartupPhasesFile( startupPhasesPath: string, @@ -157,7 +153,7 @@ function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { /** * The phases observed in a marker tail, keeping each phase's earliest occurrence. Unknown and - * unparsable lines are ignored: a client never fails a startup over a line shape it doesn't know. + * unparsable lines are ignored: a client never fails a startup over a line it does not know. */ export function readDaemonStartupPhaseMarkers( markerTail: string, diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 4d8baca2cb..2d0bf4983f 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -37,6 +37,15 @@ vi.mock('@agent-device/host-kit/retry', async (importOriginal) => ({ ...(await importOriginal()), sleep: vi.fn(), })); +// A pinned fabricated pid answers the signal-0 probe from this map; afterEach drops the map. +let livenessOverrides: Map | undefined; +vi.mock('@agent-device/host-kit/process', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + isProcessAlive: (pid: number) => livenessOverrides?.get(pid) ?? actual.isProcessAlive(pid), + }; +}); import { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; import { sleep } from '@agent-device/host-kit/retry'; import { sendToDaemon } from '../daemon-client.ts'; @@ -48,6 +57,7 @@ const spawn = vi.mocked(runCmdDetachedMonitored); const pause = vi.mocked(sleep); afterEach(async () => { + livenessOverrides = undefined; vi.restoreAllMocks(); vi.unstubAllEnvs(); await finishRegisteredDaemonFixtures(); @@ -126,8 +136,7 @@ test('a daemon slower than the retired 15 s budget starts successfully while it if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const http = await startHttpDaemonFixture({ devices: [] }); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-slow-success-')); - // Virtual time waits ~20 s — past the retired 15 s deadline — while markers arrive in real - // time from this launch's own pid-named file, keeping the wait outside the stall window. + // Virtual time waits ~20 s past the retired 15 s deadline; real markers hold off the stall. const startedAt = Date.now(); const now = { value: startedAt }; vi.spyOn(Date, 'now').mockImplementation(() => now.value); @@ -219,7 +228,6 @@ test('a live daemon that publishes no startup marker is retired as stalled befor booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); if (!booted) return await realPause(); } - // Freeze the clock past the window so real-time retirement cannot stretch the measurement. if (now.value - started < 70_000) now.value += 5_000; await realPause(); }); @@ -235,7 +243,6 @@ test('a live daemon that publishes no startup marker is retired as stalled befor assert.equal(startupPhases(failure).spawned, 0); assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); assert.equal(spawn.mock.calls.length, 1, 'a stalled daemon is not retried'); - // The joined exit means this owned startup must leave no evidence behind. await (child?.exited ?? Promise.resolve()); assert.deepEqual( fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), @@ -294,7 +301,6 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '30000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-stale-history-')); fs.mkdirSync(paths.baseDir, { recursive: true }); - // The leftover: a real daemon child SIGTERMed before publishing, record left behind. fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); const leftover = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }, undefined, { emitStartupMarker: true, @@ -395,6 +401,7 @@ test('a busy exit while a live lock holder defers publication reaches the ceilin test('a child gone at the deadline without an observed exit and no live startup fails exited', async () => { vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-gone-at-deadline-')); + livenessOverrides = new Map([[997_842, false]]); spawn.mockImplementation(() => ({ pid: 997_842, exited: new Promise(() => {}), @@ -408,6 +415,39 @@ test('a child gone at the deadline without an observed exit and no live startup assert.equal(failure.details?.startupTimeoutMs, 2_000); }); +test('an observed exit on a reused launch pid is not a live startup at the deadline', async () => { + // A delivered exit is authoritative: the reused pid (pinned alive) is not a startup to wait + // for. The busy exit resolves in the poll that carries the clock past the ceiling. + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-reused-pid-exit-')); + const reusedPid = 997_603; + livenessOverrides = new Map([[reusedPid, true]]); + let resolveExit: ((exit: ExecDetachedExit) => void) | undefined; + spawn.mockImplementation(() => ({ + pid: reusedPid, + exited: new Promise((resolve) => { + resolveExit = resolve; + }), + readIdentity: () => ({ pid: reusedPid, startTime: null }), + })); + const started = Date.now(); + const now = { value: started }; + vi.spyOn(Date, 'now').mockImplementation(() => now.value); + let advanced = false; + pause.mockImplementation(async () => { + if (!advanced) { + advanced = true; + now.value += 2_000; + resolveExit?.({ pid: reusedPid, exitCode: DAEMON_STARTUP_EXIT_CODES.busy }); + } + await realPause(20); + }); + const failure = await expectStartupFailure(sendToDaemon(request(paths))); + assert.equal(failure.details?.reason, 'startup-exited'); + assert.equal(failure.details?.startupTimeoutMs, 2_000); + assert.equal(spawn.mock.calls.length, 1, 'the deadline had passed, so no second attempt started'); +}); + test('an unproven registration exit reports startup-exited even under a short env ceiling', async () => { vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-exit-under-ceiling-')); diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index aee9d149eb..11078b40ed 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -386,7 +386,6 @@ for (const exit of [ } test('a busy contender that exits sweeps its own pre-lock marker before the retried attempt', async (t) => { - // The second launch is refused at the spawn seam, so only the client's own cleanup can pass. if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-busy-retry-marker-')); const http = await startHttpDaemonFixture({ devices: [] }); @@ -397,13 +396,11 @@ test('a busy contender that exits sweeps its own pre-lock marker before the retr assert.equal(claim.status, 'acquired'); if (claim.status !== 'acquired') throw new Error('fixture claim refused'); let released = false; - let contenderPid = 0; spawn.mockImplementation((_command, _args, options) => { const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options, undefined, { emitStartupMarkerBeforeAcquisition: true, }); if (spawn.mock.calls.length === 1) { - contenderPid = child.pid; void child.exited.then((exit) => { assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); if (!released) { @@ -423,7 +420,6 @@ test('a busy contender that exits sweeps its own pre-lock marker before the retr (error: unknown) => error instanceof AppError && error.details?.kind === 'daemon_startup_failed', ); - assert.ok(contenderPid > 0); assert.deepEqual( fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), [], @@ -436,46 +432,8 @@ test('a busy contender that exits sweeps its own pre-lock marker before the retr } }); -test('attaching to a contending winner sweeps the busy contender it exited as', async (t) => { - if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); - const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-winner-attach-marker-')); - const http = await startHttpDaemonFixture({ devices: [] }); - const deferred = path.join(paths.baseDir, 'defer-publication'); - fs.writeFileSync(deferred, 'wait'); - const winner = spawnRegisteredDaemonFixture(paths, fields(http.port), { stdio: 'ignore' }); - await awaitFile(path.join(paths.baseDir, 'registration-held')); - let contenderExit: ExecDetachedExit | undefined; - spawn.mockImplementation((_command, _args, options) => { - const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options, undefined, { - emitStartupMarkerBeforeAcquisition: true, - }); - void child.exited.then((exit) => { - contenderExit = exit; - }); - return child; - }); - pause.mockImplementation(async () => { - if (contenderExit) fs.rmSync(deferred, { force: true }); - await actualRetry.sleep(10); - }); - try { - const response = await sendToDaemon(request(paths)); - assert.equal(response.ok, true); - assert.equal(contenderExit?.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); - assert.equal(spawn.mock.calls.length, 1); - assert.equal(http.rpcRequests.length, 1); - assert.equal(isProcessAlive(winner.pid), true); - assert.deepEqual( - fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), - [], - "the winner handoff must not leave the exited contender's record behind", - ); - } finally { - await closeLoopbackServer(http.server); - } -}); - test('a joined busy contender waits for a published winner to become ready without signaling it', async (t) => { + // The contender records its pre-lock phase through the real protocol; the handoff must sweep it. if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-delayed-ready-')); let probes = 0; @@ -486,7 +444,9 @@ test('a joined busy contender waits for a published winner to become ready witho await awaitFile(path.join(paths.baseDir, 'registration-held')); let contenderExit: ExecDetachedExit | undefined; spawn.mockImplementation((_command, _args, options) => { - const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options, undefined, { + emitStartupMarkerBeforeAcquisition: true, + }); void child.exited.then((exit) => { contenderExit = exit; }); @@ -508,6 +468,11 @@ test('a joined busy contender waits for a published winner to become ready witho signal.mock.calls.some(([pid, kind]) => pid === winner.pid && kind !== 0), false, ); + assert.deepEqual( + fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + [], + "the winner handoff must not leave the exited contender's record behind", + ); } finally { signal.mockRestore(); await closeLoopbackServer(http.server); @@ -576,8 +541,7 @@ test.for([ ])('an older winner is replaced only with enough startup time ($budget)', async (expected, t) => { if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-older-winner-')); - // The replacement-window behavior is budget-independent; pin the ceiling so this test keeps its - // original 15 s shape while the production default is the #3782 ceiling. + // The replacement-window behavior is budget-independent; this pins the original 15 s shape. vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '15000'); const http = await startHttpDaemonFixture({ devices: [] }); const deferred = path.join(paths.baseDir, 'defer-publication'); diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index 7d45613d11..52da7deb62 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -3,7 +3,7 @@ import { AppError, normalizeError, type NormalizedError } from '@agent-device/ke import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import { processLockHoldsDaemonIdentity } from '@agent-device/daemon-contracts/daemon-registration'; import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/host-kit/file'; -import { isProcessAlive } from '@agent-device/host-kit/process'; +import { classifyOwnerLiveness, isProcessAlive } from '@agent-device/host-kit/process'; import { sleep } from '@agent-device/host-kit/retry'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { type ExecDetachedExit } from '@agent-device/host-kit/command'; @@ -53,9 +53,7 @@ type DaemonStartupWaitResult = | { kind: 'unproven'; error: AppError } | { kind: 'retry' } | { kind: 'exit_unobserved' } - /** The process stayed alive but published no new startup phase within the stall window. */ | { kind: 'stalled' } - /** The process stayed alive and kept progressing, but did not register before the ceiling. */ | { kind: 'ceiling' }; type StartupPhaseObservation = { @@ -63,10 +61,7 @@ type StartupPhaseObservation = { markers: Partial>; }; -/** - * The ceiling on one local daemon startup: a live daemon that keeps publishing markers is waited - * for. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides it. - */ +/** The ceiling on one local daemon startup; `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` overrides it. */ const DAEMON_STARTUP_CEILING_MS = 120_000; /** Silence this long ends the wait early; an env ceiling below it makes the ceiling the stop. */ const DAEMON_STARTUP_STALL_WINDOW_MS = 60_000; @@ -74,7 +69,6 @@ const MINIMUM_DAEMON_TAKEOVER_BUDGET_MS = 5_000; const DAEMON_STARTUP_TIMEOUT_ENV = 'AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS'; -/** Why a startup that never reached readiness failed, in terms a caller can branch on. */ type DaemonStartupFailureReason = 'startup-exited' | 'startup-stalled' | 'startup-ceiling'; function resolveDaemonStartupTimeoutMs(env: Readonly): number { @@ -97,10 +91,7 @@ const DAEMON_STARTUP_ATTEMPTS = 2; const DAEMON_STARTUP_LOG_TAIL_BYTES = 64_000; const DAEMON_STARTUP_MARKER_TAIL_BYTES = 16_000; -/** - * Judges the registered daemon against this client, changing nothing; acting on the judgement - * belongs to the caller's lifecycle. - */ +/** Judges the registered daemon against this client, changing nothing. */ export async function observeLocalDaemon( settings: DaemonClientSettings, options: { @@ -126,7 +117,6 @@ export async function observeLocalDaemon( return { info: existing, decision: await refineReplacement(existing, decision, options) }; } -/** Separates a gone registration from a live one answering on another transport. */ async function refineReplacement( info: DaemonInfo, decision: DaemonTakeoverDecision, @@ -218,10 +208,7 @@ export async function retireDaemonForTakeover( if (retirement.status === 'retained') throw daemonRetirementUnconfirmedError(retirement, paths); } -/** - * A live pid is probed again before being judged unreachable: this client's own event-loop stall - * must not read a listening daemon as gone. Liveness uses signal 0, which a busy `ps` cannot miss. - */ +/** A live pid is probed again before being judged unreachable; liveness is signal 0, not `ps`. */ async function canReachReusableDaemon( info: DaemonInfo, preference: DaemonTransportPreference, @@ -475,7 +462,7 @@ async function joinStartup(launch: DaemonStartupLaunch): Promise { } } -/** Sweeps a proven-exited launch's own record, after joining that exact child. */ +/** Sweeps a proven-exited launch's own record after joining that exact child. */ async function discardExitedLaunchRecord( settings: DaemonClientSettings, launch: DaemonStartupLaunch, @@ -521,14 +508,30 @@ async function waitForDaemonStartup( } await sleep(Math.min(100, remainingStartupBudget(deadline))); } - // A deadline passing while any startup is live is the host running out of room, never an exit. - if (isProcessAlive(launch.pid) || isLiveLockOwner(inspectProcessLock(settings.paths.lockPath))) - return { kind: 'ceiling' }; + return classifyExpiredDaemonStartup(settings, launch, earlyExit); +} + +/** Judges the wait at an expired deadline; never signals anything. */ +function classifyExpiredDaemonStartup( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + earlyExit: ExecDetachedExit | undefined, +): DaemonStartupWaitResult { + // A live lock owner outranks an observed exit; a delivered exit is authoritative for its lifetime. + const liveLockOwner = isLiveLockOwner(inspectProcessLock(settings.paths.lockPath)); + if (liveLockOwner) return { kind: 'ceiling' }; if (earlyExit) return { kind: 'early_exit', exit: earlyExit }; - return { kind: 'exit_unobserved' }; + return isLiveLaunchedStartup(launch) ? { kind: 'ceiling' } : { kind: 'exit_unobserved' }; +} + +function isLiveLaunchedStartup(launch: DaemonStartupLaunch): boolean { + if (!isProcessAlive(launch.pid)) return false; + const identity = launch.readIdentity(); + if (!identity.startTime) return true; + return classifyOwnerLiveness({ owner: identity }) !== 'owner-process-reused'; } -/** A lock owner is a pending startup unless provably dead or reused; never signals anything. */ +/** A lock owner is a pending startup unless provably dead or reused. */ function isLiveLockOwner(inspection: ProcessLockInspection): boolean { return ( inspection.state === 'held' && @@ -557,7 +560,6 @@ async function observeLiveDaemonStartup( ): Promise { const info = await readReadyLaunchedDaemon(settings, launch, deadline); if (info) return { kind: 'ready', daemon: { info, startedByClient: true } }; - // A daemon still publishing markers is slow, not stuck; only silence retires it early. refreshStartupPhaseMarkers(settings, launch.pid, observation); if ( isProcessAlive(launch.pid) && @@ -568,7 +570,6 @@ async function observeLiveDaemonStartup( return null; } -/** Folds this launch's markers into `observation` from its own pid-named record. */ function refreshStartupPhaseMarkers( settings: DaemonClientSettings, launchPid: number, diff --git a/src/daemon.ts b/src/daemon.ts index b522dcf9d2..ec10db0a1b 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -12,8 +12,8 @@ import { } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -// Marker failures go to stderr, which the launcher redirects into the daemon log; a refused -// append is evidence, never a startup requirement. +// Marker failures go to stderr, redirected into the daemon log; a refused append never stops a +// startup that can still register. function reportMarkerFailure(error: unknown): void { process.stderr.write(`Daemon startup marker write failed: ${normalizeError(error).message}\n`); } @@ -22,18 +22,15 @@ const ownPhasesPath = resolveDaemonStartupPhasesPath( resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, process.pid, ); -// The claim first: after pid reuse it is the only defense against reading a dead lifetime's -// phases as this launch's own. +// The claim first: after pid reuse it is the only defense against a dead lifetime's phases. claimDaemonStartupPhasesFile(ownPhasesPath, reportMarkerFailure); recordDaemonStartupMarker(ownPhasesPath, 'modules-loaded', reportMarkerFailure); // The executable owns the process: signals, fatal-error handlers and the exit code. The runtime // only reports when it has closed, whoever closed it. -// The root's composition enters as a factory behind a function-scoped `await import`: the server -// calls it before it acquires the registration, so startup order is unchanged, and the eager -// closure of this entry stays exactly the operational server's (ADR 0019). The factory records -// `composition-loaded` when that import and dep build finish: the second module-graph edge this -// startup crosses, and a real completed boundary the waiting client can attribute progress to. +// Composition enters as a factory behind `await import`: startup order is unchanged and this +// entry's eager closure stays the operational server's (ADR 0019). The factory records +// `composition-loaded` when that import and dep build finish. async function runDaemon(): Promise { const runtime = await startDaemonRuntime({ host: captureHostEnvironment(process), From 5172f7205d8d0eac39fd4c6f113d8d6a0d2f7d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 10:55:07 +0200 Subject: [PATCH 16/22] docs(daemon): scope the startup ceiling env to daemon starts and restore the marker sweep owner Both facts were stated incompletely against the code: - resolveDaemonStartupTimeoutMs runs only inside startLocalDaemon, so AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS is validated only when a client must start a daemon. A client that attaches to or reuses a running daemon never reads the variable. CHANGELOG, cli-help, and the configuration guide now say so; the guide already named INVALID_ARGS. - The startup-marker ownership comment dropped the registration owner's sweep of provably dead pids, which sweepDeadDaemonStartupPhasesFiles still performs at each new registration lock acquisition. Addresses review threads r4305110030 and r4305299946 on #3791. Co-authored-by: Claude Opus 4.8 --- CHANGELOG.md | 2 +- .../src/daemon-registration-owner.ts | 3 ++- .../test-utils/registered-daemon-fixture.test.ts | 3 +-- src/commands/schema/cli-help.ts | 2 +- .../daemon-client-startup-budget.test.ts | 3 +-- src/daemon.ts | 15 +++++---------- website/docs/docs/configuration.md | 2 +- 7 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d30b483e9f..a71c517c39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - The PNG worker no longer calls `process.emitWarning` when it is unavailable, and it starts with an empty environment (`env: {}`). The daemon registers a sink when it pre-warms the worker, so every transition to unavailable (spawn failure, missing entry, load error, exit) leaves exactly one `png_worker_unavailable` event, with a typed `reason` (`entry-not-found`, `spawn-failed`, `worker-error`, `worker-exited`) and its `detail`, in `daemon.log`, with or without `--debug`. Requests then continue on the in-process fallback without repeating the event. - Behavior change: `resolveHubAppReference`, shipped through `agent-device/plugins/webdriver`, now needs `cwd` to resolve a relative app path. Without it the call fails with `INVALID_ARGS` and `details.reason: provider_app_cwd_missing`; it used to resolve against the process's working directory. Absolute paths, URLs and provider references are unaffected. The daemon fills it from the client's working directory (or its host's) for every provider lease request, so `connect` and `lease_allocate` are unchanged. -- A client starting a local daemon now waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower instead of failing it. The wait stops at a ceiling of 120 s by default; set `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` to a positive integer number of milliseconds to override it for the CLI, MCP, and Node clients alike. A failed startup names the outcome in `error.details.reason`, and you should branch on it, never on the message: `startup-exited` (the process ended before registering), `startup-stalled` (alive but no startup progress for a fixed 60 s window, which the ceiling variable does not change; the daemon is stopped), or `startup-ceiling` (a startup was still live — this client's daemon or another client's — when the ceiling passed; the client stops its own launch and never signals another client's live daemon). `details.phases` carries the startup phase timings the client observed and `details.startupTimeoutMs` the effective ceiling. +- A client starting a local daemon now waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower instead of failing it. The wait stops at a ceiling of 120 s by default; set `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` to a positive integer number of milliseconds to override it for the CLI, MCP, and Node clients alike. The variable is read only when the client must start a daemon; a value that is not a positive integer fails with `INVALID_ARGS` before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A failed startup names the outcome in `error.details.reason`, and you should branch on it, never on the message: `startup-exited` (the process ended before registering), `startup-stalled` (alive but no startup progress for a fixed 60 s window, which the ceiling variable does not change; the daemon is stopped), or `startup-ceiling` (a startup was still live — this client's daemon or another client's — when the ceiling passed; the client stops its own launch and never signals another client's live daemon). `details.phases` carries the startup phase timings the client observed and `details.startupTimeoutMs` the effective ceiling. - The `daemon-not-running` hint now always names `--state-dir `, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`. - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir `. diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 7e3310b2d9..206c79ac0c 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -51,7 +51,8 @@ export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 /** * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup * got, so a waiting client tells "slow but progressing" from "stuck" without a timestamp and with - * no heartbeat. The writer removes it on completion, the launching client after a joined exit. + * no heartbeat. The writer removes it on completion, the launching client after a joined exit, and + * each new registration owner sweeps the files of provably dead pids. */ const DAEMON_STARTUP_PHASES = Object.freeze([ 'modules-loaded', diff --git a/src/__tests__/test-utils/registered-daemon-fixture.test.ts b/src/__tests__/test-utils/registered-daemon-fixture.test.ts index 59592da4aa..92a026eeac 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.test.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.test.ts @@ -18,8 +18,7 @@ import { } from './registered-daemon-fixture.ts'; test('args built before any launch keep their own entry script under a concurrent second build', async () => { - // Both arg sets are built before either child starts — the overwrite window. Under a shared - // entry path child 1 would run child 2's switches and its record would carry the wrong marker. + // Under a shared entry path child 1 would run child 2's switches and carry the wrong marker. const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-fixture-entry-isolation-')); fs.mkdirSync(paths.baseDir, { recursive: true }); fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index e1a307286a..012e339873 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -512,7 +512,7 @@ Runner and daemon lifecycle (applies to simulators too): No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. - A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window, so a ceiling set below 60 s also ends silent startups this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. + A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window, so a ceiling set below 60 s also ends silent startups this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 2d0bf4983f..dfadd14af4 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -416,8 +416,7 @@ test('a child gone at the deadline without an observed exit and no live startup }); test('an observed exit on a reused launch pid is not a live startup at the deadline', async () => { - // A delivered exit is authoritative: the reused pid (pinned alive) is not a startup to wait - // for. The busy exit resolves in the poll that carries the clock past the ceiling. + // A delivered exit is authoritative: the reused pid (pinned alive) is not a startup to wait for. vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-reused-pid-exit-')); const reusedPid = 997_603; diff --git a/src/daemon.ts b/src/daemon.ts index ec10db0a1b..8ad7bc3f96 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,9 +1,5 @@ import { asAppError, normalizeError } from '@agent-device/kernel/errors'; -import { - daemonStartupExitCode, - reportDaemonClose, - startDaemonRuntime, -} from '@agent-device/daemon'; +import { daemonStartupExitCode, reportDaemonClose, startDaemonRuntime } from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import { claimDaemonStartupPhasesFile, @@ -12,8 +8,7 @@ import { } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -// Marker failures go to stderr, redirected into the daemon log; a refused append never stops a -// startup that can still register. +// A refused marker append goes to stderr, which the launcher redirects into the daemon log. function reportMarkerFailure(error: unknown): void { process.stderr.write(`Daemon startup marker write failed: ${normalizeError(error).message}\n`); } @@ -28,9 +23,9 @@ recordDaemonStartupMarker(ownPhasesPath, 'modules-loaded', reportMarkerFailure); // The executable owns the process: signals, fatal-error handlers and the exit code. The runtime // only reports when it has closed, whoever closed it. -// Composition enters as a factory behind `await import`: startup order is unchanged and this -// entry's eager closure stays the operational server's (ADR 0019). The factory records -// `composition-loaded` when that import and dep build finish. +// The root's composition enters as a factory behind a function-scoped `await import`: the server +// calls it before it acquires the registration, so startup order is unchanged, and the eager +// closure of this entry stays exactly the operational server's (ADR 0019). async function runDaemon(): Promise { const runtime = await startDaemonRuntime({ host: captureHostEnvironment(process), diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index 15a887bd64..2a79cfff9b 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -122,7 +122,7 @@ Only the variables below, and those documented on command pages, are supported. | Category | Env vars | Decision | | --- | --- | --- | -| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (a positive integer number of milliseconds, default `120000`; any other value fails with `INVALID_ARGS`) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for a fixed 60 s window this variable does not change), or `startup-ceiling` (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window). Branch on `details.reason`, never on the message. | +| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (a positive integer number of milliseconds, default `120000`; any other value fails with `INVALID_ARGS` before a daemon spawns, and only when the client must start a daemon — a client that attaches to or reuses a running daemon never reads it) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for a fixed 60 s window this variable does not change), or `startup-ceiling` (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window). Branch on `details.reason`, never on the message. | | Device scoping | `AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST`, `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` | Public. `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` is read by the daemon: every iOS simulator request that names no `--ios-simulator-device-set` uses that set, so iOS physical devices are not listed, and a request naming a different path string fails with `INVALID_ARGS` (`details.reason: "ios_simulator_device_set_outside_daemon_set"`). macOS and non-iOS requests ignore it. A daemon started by the CLI inherits the CLI's value; restart the daemon after changing it. | | Android test IME | `AGENT_DEVICE_TEST_IME` | Public. Same setting as `--test-ime` / `--no-test-ime` on `open`, `test`, and `replay`; see known limitations. | | Local daemon storage | `AGENT_DEVICE_STATE_DIR` | Public | From 489b0d5e668af0eb9fb9b3c18a0d5e711f5f7bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 11:08:12 +0200 Subject: [PATCH 17/22] test(daemon): point the host test's observe mock at the startup module The startup split moved observeLocalDaemon out of daemon-client-lifecycle.ts, so the host test's lifecycle-module mock no longer stopped the ensure path and the CLI-run case fell through to the real observation. Mock the seam the subject imports it from now. Drop two narration comments the assertions already state. Co-authored-by: Claude Opus 4.8 --- .../__tests__/daemon-client-startup-budget.test.ts | 3 --- src/daemon-client/daemon-client-host.test.ts | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index dfadd14af4..0d03971373 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -297,7 +297,6 @@ test('a live daemon that keeps progressing but never registers fails at the env }); test("a startup progressed by a fresh launch is not judged stalled by an earlier launch's leftover markers", async () => { - // A reused state dir holds a dead startup's record; the fresh launch must see its own markers. vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '30000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-stale-history-')); fs.mkdirSync(paths.baseDir, { recursive: true }); @@ -376,7 +375,6 @@ test('a busy exit while a live lock holder defers publication reaches the ceilin assert.equal(failure.details?.startupTimeoutMs, 2_000); assert.match(String(failure.details?.hint), /AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS/); assert.equal(typeof startupPhases(failure)['modules-loaded'], 'number'); - // The winner stays running and unsignaled with its record; the contender's is swept. assert.equal(isProcessAlive(winner.pid), true); assert.equal( signal.mock.calls.some(([pid, kind]) => pid === winner.pid && kind !== 0), @@ -416,7 +414,6 @@ test('a child gone at the deadline without an observed exit and no live startup }); test('an observed exit on a reused launch pid is not a live startup at the deadline', async () => { - // A delivered exit is authoritative: the reused pid (pinned alive) is not a startup to wait for. vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '2000'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-reused-pid-exit-')); const reusedPid = 997_603; diff --git a/src/daemon-client/daemon-client-host.test.ts b/src/daemon-client/daemon-client-host.test.ts index 3e3f7578c4..0902267605 100644 --- a/src/daemon-client/daemon-client-host.test.ts +++ b/src/daemon-client/daemon-client-host.test.ts @@ -9,6 +9,9 @@ vi.mock('./daemon-client-lifecycle.ts', async (importOriginal) => ({ ensured.settings = settings; throw new Error('stop before talking to a daemon'); }, +})); +vi.mock('./daemon-client-startup.ts', async (importOriginal) => ({ + ...(await importOriginal()), observeLocalDaemon: async (settings: unknown) => { ensured.settings = settings; throw new Error('stop before talking to a daemon'); From 054ed6251a1389124091dcea13c3690040d5d2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 14:41:47 +0200 Subject: [PATCH 18/22] refactor(daemon): give startup phase records their own contracts module The per-pid startup phase file has its own record format, writer, reader and dead-pid sweep, so it no longer belongs inside the registration owner the review pointed at (#3791). The declarations move verbatim into daemon-contracts/daemon-startup-phases.ts beside daemon-shutdown-report and the suffix-named markers test follows them as daemon-startup-phases.test.ts. Registration acquisition imports only sweepDeadDaemonStartupPhasesFiles; the runtime, the daemon entry, the client wait and the fixture import the record owner directly, with no re-export shim (unreleased surface). The daemon entry must write its earliest modules-loaded marker at module load, so the eager-closure gate gets one APPROVED_CLOSURE_ADDITIONS row naming the new module; the entry pays for the record it writes first. Addresses review 6248815887 item 1 on #3791. Refs #3782, #3791 --- packages/daemon-contracts/package.json | 4 + .../src/daemon-startup-phases.test.ts | 153 ++++++++++++++++++ .../src/daemon-startup-phases.ts | 144 +++++++++++++++++ scripts/__tests__/eager-closure-budgets.ts | 7 + 4 files changed, 308 insertions(+) create mode 100644 packages/daemon-contracts/src/daemon-startup-phases.test.ts create mode 100644 packages/daemon-contracts/src/daemon-startup-phases.ts diff --git a/packages/daemon-contracts/package.json b/packages/daemon-contracts/package.json index bc4d0cf0af..56faa64f4f 100644 --- a/packages/daemon-contracts/package.json +++ b/packages/daemon-contracts/package.json @@ -44,6 +44,10 @@ "types": "./src/daemon-shutdown-report.ts", "default": "./src/daemon-shutdown-report.ts" }, + "./daemon-startup-phases": { + "types": "./src/daemon-startup-phases.ts", + "default": "./src/daemon-startup-phases.ts" + }, "./daemon-stop": { "types": "./src/daemon-stop.ts", "default": "./src/daemon-stop.ts" diff --git a/packages/daemon-contracts/src/daemon-startup-phases.test.ts b/packages/daemon-contracts/src/daemon-startup-phases.test.ts new file mode 100644 index 0000000000..1182de4348 --- /dev/null +++ b/packages/daemon-contracts/src/daemon-startup-phases.test.ts @@ -0,0 +1,153 @@ +import { appendFile, readFile, writeFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { tryAcquireDaemonRegistration } from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { + claimDaemonStartupPhasesFile, + readDaemonStartupPhaseMarkers, + recordDaemonStartupMarker, + removeDaemonStartupPhasesFile, + resolveDaemonStartupPhasesPath, +} from '@agent-device/daemon-contracts/daemon-startup-phases'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { mkdtempForTest } from '@agent-device/test-kit'; + +test('per-launch marker files: a fresh launch never reads an older launch phases', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const first = resolveDaemonStartupPhasesPath(stateDir, 1111); + const second = resolveDaemonStartupPhasesPath(stateDir, 2222); + recordDaemonStartupMarker(first, 'lock-acquired'); + expect(second).not.toBe(first); + expect(existsSync(second)).toBe(false); + recordDaemonStartupMarker(second, 'modules-loaded'); + const firstPhases = readDaemonStartupPhaseMarkers(readFileSync(first, 'utf8')); + const secondPhases = readDaemonStartupPhaseMarkers(readFileSync(second, 'utf8')); + expect(Object.keys(firstPhases)).toEqual(['lock-acquired']); + expect(Object.keys(secondPhases)).toEqual(['modules-loaded']); +}); + +test('marker reading keeps the earliest record when a phase replays after a later one', () => { + const text = [ + '{"data":{"startupPhase":"listening","timestampMs":500}}', + '{"data":{"startupPhase":"lock-acquired","timestampMs":20}}', + '{"data":{"startupPhase":"lock-acquired","timestampMs":35}}', + ].join('\n'); + const phases = readDaemonStartupPhaseMarkers(text); + expect(phases['lock-acquired']).toBe(20); + expect(phases.listening).toBe(500); +}); + +test('marker reading ignores unknown phases and unparsable lines', () => { + const text = [ + '{"data":{"startupPhase":"heartbeat","timestampMs":10}}', + '{"data":{"startupPhase":"registered","timestampMs":"7"}}', + 'not json', + '{"data":{"startupPhase":"listening","timestampMs":30}}', + ].join('\n'); + expect(readDaemonStartupPhaseMarkers(text)).toEqual({ listening: 30 }); +}); + +test('the producer lifetime claim empties a same-pid leftover', async () => { + // Pid reuse after both the old client and daemon died before either cleanup: the entry claims + // its record before its first marker; this child repeats that producer sequence itself. + const stateDir = await mkdtempForTest('agent-device-markers-'); + writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}'); + const entry = path.join(stateDir, 'claim-producer.js'); + const registrationUrl = new URL('./daemon-startup-phases.ts', import.meta.url).href; + writeFileSync( + entry, + `import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; +const file = resolveDaemonStartupPhasesPath(${JSON.stringify(stateDir)}, process.pid); +recordDaemonStartupMarker(file, 'listening'); +recordDaemonStartupMarker(file, 'registered'); +claimDaemonStartupPhasesFile(file); +recordDaemonStartupMarker(file, 'modules-loaded'); +`, + ); + const child = spawnSync(process.execPath, ['--experimental-strip-types', entry], { + encoding: 'utf8', + }); + if (child.status !== 0) throw new Error(child.stderr); + const phases = readDaemonStartupPhaseMarkers( + readFileSync(resolveDaemonStartupPhasesPath(stateDir, child.pid!), 'utf8'), + ); + expect(phases).toEqual({ 'modules-loaded': expect.any(Number) }); +}); + +test('a refused claim reset reports through the callback instead of throwing', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 778); + mkdirSync(markerPath, { recursive: true }); + const reported: unknown[] = []; + expect(() => + claimDaemonStartupPhasesFile(markerPath, (error) => { + reported.push(error); + }), + ).not.toThrow(); + expect(reported).toHaveLength(1); + expect(reported[0]).toHaveProperty('message'); +}); + +test('acquiring the registration retires files of dead or unowned pids only', async () => { + const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); + const dead = resolveDaemonStartupPhasesPath(paths.baseDir, spawnDeadPid()); + const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); + const unparseable = path.join(paths.baseDir, 'daemon-startup-phases-.jsonl'); + const legacy = path.join(paths.baseDir, 'daemon-startup-phases.jsonl'); + await Promise.all([ + writeFile(own, 'x\n', 'utf8'), + writeFile(dead, 'x\n', 'utf8'), + writeFile(unparseable, 'x\n', 'utf8'), + writeFile(legacy, 'x\n', 'utf8'), + ]); + const attempt = await tryAcquireDaemonRegistration(paths); + expect(attempt.status).toBe('acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + expect(existsSync(dead)).toBe(false); + expect(existsSync(unparseable)).toBe(false); + expect(existsSync(own)).toBe(true); + expect(existsSync(legacy)).toBe(true); + await attempt.owner.finish(); +}); + +test('removeDaemonStartupPhasesFile removes a file and tolerates one already gone', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const file = resolveDaemonStartupPhasesPath(stateDir, 4242); + await writeFile(file, 'x\n', 'utf8'); + removeDaemonStartupPhasesFile(file); + expect(existsSync(file)).toBe(false); + expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow(); +}); + +test('a marker write failure reports through the callback, keeping startup running', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); + mkdirSync(markerPath, { recursive: true }); + const reported: unknown[] = []; + expect(() => + recordDaemonStartupMarker(markerPath, 'modules-loaded', (error) => { + reported.push(error); + }), + ).not.toThrow(); + expect(reported).toHaveLength(1); + expect(reported[0]).toHaveProperty('message'); +}); + +test('a crash-replay append keeps the file valid JSONL', async () => { + const stateDir = await mkdtempForTest('agent-device-markers-'); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999); + await appendFile(markerPath, '{"data":{"startupPhase":"modules-lo', 'utf8'); + recordDaemonStartupMarker(markerPath, 'composition-loaded'); + const tail = await readFile(markerPath, 'utf8'); + expect(Object.keys(readDaemonStartupPhaseMarkers(tail))).toEqual(['composition-loaded']); +}); + +function spawnDeadPid(): number { + const child = spawnSync(process.execPath, ['-e', ''], { encoding: 'utf8' }); + if (child.error) throw child.error; + const pid = child.pid; + if (pid === undefined) throw new Error('spawnSync returned no pid'); + return pid; +} diff --git a/packages/daemon-contracts/src/daemon-startup-phases.ts b/packages/daemon-contracts/src/daemon-startup-phases.ts new file mode 100644 index 0000000000..1a5f746da4 --- /dev/null +++ b/packages/daemon-contracts/src/daemon-startup-phases.ts @@ -0,0 +1,144 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; +import { isProcessAlive } from '@agent-device/host-kit/process'; + +/** + * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup + * got, so a waiting client tells "slow but progressing" from "stuck" without a timestamp and with + * no heartbeat. The writer removes it on completion, the launching client after a joined exit, and + * each new registration owner sweeps the files of provably dead pids. + */ +const DAEMON_STARTUP_PHASES = Object.freeze([ + 'modules-loaded', + 'composition-loaded', + 'lock-acquired', + 'recovery-finished', + 'listening', + 'registered', +] as const); + +export type DaemonStartupPhase = (typeof DAEMON_STARTUP_PHASES)[number]; + +const DAEMON_STARTUP_PHASE_SET: ReadonlySet = new Set(DAEMON_STARTUP_PHASES); + +const DAEMON_STARTUP_PHASES_ENTRY = /^daemon-startup-phases-(\d*)\.jsonl$/; + +/** The marker file the daemon process `pid` writes in `stateDir`; an absent file is no evidence. */ +export function resolveDaemonStartupPhasesPath(stateDir: string, pid: number): string { + return path.join(stateDir, `daemon-startup-phases-${pid}.jsonl`); +} + +/** Appends one phase marker; a refused append goes to `reportFailure`, never fatal to a startup. */ +export function recordDaemonStartupMarker( + startupPhasesPath: string, + phase: DaemonStartupPhase, + reportFailure?: (error: NormalizedError) => void, +): void { + try { + const timestampMs = Date.now(); + fs.mkdirSync(path.dirname(startupPhasesPath), { recursive: true }); + // A crash mid-append can leave no trailing newline; glue two appends and both are lost. + const separator = endsWithoutNewline(startupPhasesPath) ? '\n' : ''; + fs.appendFileSync( + startupPhasesPath, + `${separator}${JSON.stringify({ data: { startupPhase: phase, timestampMs } })}\n`, + ); + } catch (error) { + reportFailure?.(normalizeError(error)); + } +} + +function endsWithoutNewline(file: string): boolean { + let fd: number | undefined; + try { + const stats = fs.statSync(file); + if (stats.size === 0) return false; + const buffer = Buffer.alloc(1); + fd = fs.openSync(file, 'r'); + if (fs.readSync(fd, buffer, 0, 1, stats.size - 1) !== 1) return false; + return buffer[0] !== 0x0a; + } catch { + return false; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +/** + * Empties this launch's own record before its first marker: a per-pid name identifies a writer, + * not a lifetime. No caller resets a file named for another pid; a refused reset never stops the + * startup because the reset ran first when it could run. + */ +export function claimDaemonStartupPhasesFile( + startupPhasesPath: string, + reportFailure?: (error: NormalizedError) => void, +): void { + try { + fs.rmSync(startupPhasesPath, { force: true }); + } catch (error) { + reportFailure?.(normalizeError(error)); + } +} + +/** Removes a marker file whose writer is provably gone: a finished startup or a joined exit. */ +export function removeDaemonStartupPhasesFile(startupPhasesPath: string): void { + try { + fs.rmSync(startupPhasesPath, { force: true }); + } catch {} +} + +/** Sweeps marker files of provably dead (or unnameable) pids; a live pid's file is kept. */ +export function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { + let entries: string[]; + try { + entries = fs.readdirSync(stateDir); + } catch { + return; + } + for (const entry of entries) { + const match = DAEMON_STARTUP_PHASES_ENTRY.exec(entry); + if (match === null) continue; + const deadPid = Number(match[1]); + if (deadPid === process.pid) continue; + if (!Number.isSafeInteger(deadPid) || deadPid <= 0 || !isProcessAlive(deadPid)) { + removeDaemonStartupPhasesFile(path.join(stateDir, entry)); + } + } +} + +/** + * The phases observed in a marker tail, keeping each phase's earliest occurrence. Unknown and + * unparsable lines are ignored: a client never fails a startup over a line it does not know. + */ +export function readDaemonStartupPhaseMarkers( + markerTail: string, +): Partial> { + const phases: Partial> = {}; + for (const line of markerTail.split('\n')) { + if (!line.includes('"startupPhase"')) continue; + let record: DaemonStartupMarkerRecord; + try { + record = JSON.parse(line) as DaemonStartupMarkerRecord; + } catch { + continue; + } + const phase = record.data?.startupPhase; + const timestampMs = record.data?.timestampMs; + if ( + typeof phase !== 'string' || + !DAEMON_STARTUP_PHASE_SET.has(phase) || + typeof timestampMs !== 'number' || + !Number.isFinite(timestampMs) + ) { + continue; + } + const seen = phases[phase as DaemonStartupPhase]; + if (seen === undefined || timestampMs < seen) phases[phase as DaemonStartupPhase] = timestampMs; + } + return phases; +} + +type DaemonStartupMarkerRecord = { + data?: { startupPhase?: unknown; timestampMs?: unknown }; +}; diff --git a/scripts/__tests__/eager-closure-budgets.ts b/scripts/__tests__/eager-closure-budgets.ts index 73851ada96..012ed1c46b 100644 --- a/scripts/__tests__/eager-closure-budgets.ts +++ b/scripts/__tests__/eager-closure-budgets.ts @@ -152,6 +152,13 @@ export const APPROVED_CLOSURE_ADDITIONS: readonly Readonly<{ 'The one owner of per-runtime async-local state: the store, its standalone fallback and its disposal, so Apple, request-cancel and the root declare only slot keys instead of each keeping a copy.', owner: 'thymikee', }, + { + module: 'packages/daemon-contracts/src/daemon-startup-phases.ts', + issue: '#3782', + reason: + 'The startup phase record owner the #3791 review asked for (its own file/format/writer/reader/sweep beside daemon-shutdown-report). The daemon entry records its earliest `modules-loaded` marker at module load, so the owner must be eager there; the entry pays for the record it writes first.', + owner: 'thymikee', + }, ]); /** The category is a function of the path, never a hand-written column. */ From f21179d2d01396efc21c7723867a425162a3b67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 14:41:55 +0200 Subject: [PATCH 19/22] refactor(daemon): point every startup-marker consumer at the new owner Importers attach to daemon-contracts/daemon-startup-phases directly; daemon-registration-owner keeps only the sweep call at acquisition. src/daemon.ts captures its host environment once and resolves the phases path from the same env and cwd startDaemonRuntime resolves its state dir from, so a relative AGENT_DEVICE_STATE_DIR cannot land the earliest markers beside a different directory than the registration (the module top-level second resolveDaemonPaths(process.env) is gone), and the modules-loaded marker stays the first thing recorded. Addresses review 6248815887 items 1 and 2 on #3791. Refs #3782, #3791 --- ...registration-owner-startup-markers.test.ts | 153 ------------------ .../src/daemon-registration-owner.ts | 141 +--------------- .../daemon/src/server/http/daemon-runtime.ts | 4 +- .../registered-daemon-fixture.test.ts | 6 +- .../test-utils/registered-daemon-fixture.ts | 7 +- src/daemon-client/daemon-client-startup.ts | 10 +- src/daemon.ts | 9 +- 7 files changed, 24 insertions(+), 306 deletions(-) delete mode 100644 packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts diff --git a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts b/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts deleted file mode 100644 index aa4f99fb49..0000000000 --- a/packages/daemon-contracts/src/daemon-registration-owner-startup-markers.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { appendFile, readFile, writeFile } from 'node:fs/promises'; -import { spawnSync } from 'node:child_process'; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import path from 'node:path'; -import { expect, test } from 'vitest'; -import { - claimDaemonStartupPhasesFile, - readDaemonStartupPhaseMarkers, - recordDaemonStartupMarker, - removeDaemonStartupPhasesFile, - resolveDaemonStartupPhasesPath, - tryAcquireDaemonRegistration, -} from '@agent-device/daemon-contracts/daemon-registration-owner'; -import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import { mkdtempForTest } from '@agent-device/test-kit'; - -test('per-launch marker files: a fresh launch never reads an older launch phases', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const first = resolveDaemonStartupPhasesPath(stateDir, 1111); - const second = resolveDaemonStartupPhasesPath(stateDir, 2222); - recordDaemonStartupMarker(first, 'lock-acquired'); - expect(second).not.toBe(first); - expect(existsSync(second)).toBe(false); - recordDaemonStartupMarker(second, 'modules-loaded'); - const firstPhases = readDaemonStartupPhaseMarkers(readFileSync(first, 'utf8')); - const secondPhases = readDaemonStartupPhaseMarkers(readFileSync(second, 'utf8')); - expect(Object.keys(firstPhases)).toEqual(['lock-acquired']); - expect(Object.keys(secondPhases)).toEqual(['modules-loaded']); -}); - -test('marker reading keeps the earliest record when a phase replays after a later one', () => { - const text = [ - '{"data":{"startupPhase":"listening","timestampMs":500}}', - '{"data":{"startupPhase":"lock-acquired","timestampMs":20}}', - '{"data":{"startupPhase":"lock-acquired","timestampMs":35}}', - ].join('\n'); - const phases = readDaemonStartupPhaseMarkers(text); - expect(phases['lock-acquired']).toBe(20); - expect(phases.listening).toBe(500); -}); - -test('marker reading ignores unknown phases and unparsable lines', () => { - const text = [ - '{"data":{"startupPhase":"heartbeat","timestampMs":10}}', - '{"data":{"startupPhase":"registered","timestampMs":"7"}}', - 'not json', - '{"data":{"startupPhase":"listening","timestampMs":30}}', - ].join('\n'); - expect(readDaemonStartupPhaseMarkers(text)).toEqual({ listening: 30 }); -}); - -test('the producer lifetime claim empties a same-pid leftover', async () => { - // Pid reuse after both the old client and daemon died before either cleanup: the entry claims - // its record before its first marker; this child repeats that producer sequence itself. - const stateDir = await mkdtempForTest('agent-device-markers-'); - writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}'); - const entry = path.join(stateDir, 'claim-producer.js'); - const registrationUrl = new URL('./daemon-registration-owner.ts', import.meta.url).href; - writeFileSync( - entry, - `import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; -const file = resolveDaemonStartupPhasesPath(${JSON.stringify(stateDir)}, process.pid); -recordDaemonStartupMarker(file, 'listening'); -recordDaemonStartupMarker(file, 'registered'); -claimDaemonStartupPhasesFile(file); -recordDaemonStartupMarker(file, 'modules-loaded'); -`, - ); - const child = spawnSync(process.execPath, ['--experimental-strip-types', entry], { - encoding: 'utf8', - }); - if (child.status !== 0) throw new Error(child.stderr); - const phases = readDaemonStartupPhaseMarkers( - readFileSync(resolveDaemonStartupPhasesPath(stateDir, child.pid!), 'utf8'), - ); - expect(phases).toEqual({ 'modules-loaded': expect.any(Number) }); -}); - -test('a refused claim reset reports through the callback instead of throwing', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 778); - mkdirSync(markerPath, { recursive: true }); - const reported: unknown[] = []; - expect(() => - claimDaemonStartupPhasesFile(markerPath, (error) => { - reported.push(error); - }), - ).not.toThrow(); - expect(reported).toHaveLength(1); - expect(reported[0]).toHaveProperty('message'); -}); - -test('acquiring the registration retires files of dead or unowned pids only', async () => { - const paths = resolveDaemonPaths(await mkdtempForTest('agent-device-markers-')); - const dead = resolveDaemonStartupPhasesPath(paths.baseDir, spawnDeadPid()); - const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); - const unparseable = path.join(paths.baseDir, 'daemon-startup-phases-.jsonl'); - const legacy = path.join(paths.baseDir, 'daemon-startup-phases.jsonl'); - await Promise.all([ - writeFile(own, 'x\n', 'utf8'), - writeFile(dead, 'x\n', 'utf8'), - writeFile(unparseable, 'x\n', 'utf8'), - writeFile(legacy, 'x\n', 'utf8'), - ]); - const attempt = await tryAcquireDaemonRegistration(paths); - expect(attempt.status).toBe('acquired'); - if (attempt.status !== 'acquired') throw new Error('registration refused'); - expect(existsSync(dead)).toBe(false); - expect(existsSync(unparseable)).toBe(false); - expect(existsSync(own)).toBe(true); - expect(existsSync(legacy)).toBe(true); - await attempt.owner.finish(); -}); - -test('removeDaemonStartupPhasesFile removes a file and tolerates one already gone', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const file = resolveDaemonStartupPhasesPath(stateDir, 4242); - await writeFile(file, 'x\n', 'utf8'); - removeDaemonStartupPhasesFile(file); - expect(existsSync(file)).toBe(false); - expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow(); -}); - -test('a marker write failure reports through the callback, keeping startup running', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); - mkdirSync(markerPath, { recursive: true }); - const reported: unknown[] = []; - expect(() => - recordDaemonStartupMarker(markerPath, 'modules-loaded', (error) => { - reported.push(error); - }), - ).not.toThrow(); - expect(reported).toHaveLength(1); - expect(reported[0]).toHaveProperty('message'); -}); - -test('a crash-replay append keeps the file valid JSONL', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999); - await appendFile(markerPath, '{"data":{"startupPhase":"modules-lo', 'utf8'); - recordDaemonStartupMarker(markerPath, 'composition-loaded'); - const tail = await readFile(markerPath, 'utf8'); - expect(Object.keys(readDaemonStartupPhaseMarkers(tail))).toEqual(['composition-loaded']); -}); - -function spawnDeadPid(): number { - const child = spawnSync(process.execPath, ['-e', ''], { encoding: 'utf8' }); - if (child.error) throw child.error; - const pid = child.pid; - if (pid === undefined) throw new Error('spawnSync returned no pid'); - return pid; -} diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 206c79ac0c..bf280ca4e1 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -10,7 +10,6 @@ import { type DaemonTerminationResult, } from './daemon-process.ts'; import { - isProcessAlive, readCurrentOwnerIdentity, readProcessStartTime, type OwnerIdentity, @@ -45,148 +44,10 @@ import { resolveDaemonShutdownReportPath, type DaemonShutdownOutcome, } from './daemon-shutdown-report.ts'; +import { sweepDeadDaemonStartupPhasesFiles } from './daemon-startup-phases.ts'; export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); -/** - * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup - * got, so a waiting client tells "slow but progressing" from "stuck" without a timestamp and with - * no heartbeat. The writer removes it on completion, the launching client after a joined exit, and - * each new registration owner sweeps the files of provably dead pids. - */ -const DAEMON_STARTUP_PHASES = Object.freeze([ - 'modules-loaded', - 'composition-loaded', - 'lock-acquired', - 'recovery-finished', - 'listening', - 'registered', -] as const); - -export type DaemonStartupPhase = (typeof DAEMON_STARTUP_PHASES)[number]; - -const DAEMON_STARTUP_PHASE_SET: ReadonlySet = new Set(DAEMON_STARTUP_PHASES); - -const DAEMON_STARTUP_PHASES_ENTRY = /^daemon-startup-phases-(\d*)\.jsonl$/; - -/** The marker file the daemon process `pid` writes in `stateDir`; an absent file is no evidence. */ -export function resolveDaemonStartupPhasesPath(stateDir: string, pid: number): string { - return path.join(stateDir, `daemon-startup-phases-${pid}.jsonl`); -} - -/** Appends one phase marker; a refused append goes to `reportFailure`, never fatal to a startup. */ -export function recordDaemonStartupMarker( - startupPhasesPath: string, - phase: DaemonStartupPhase, - reportFailure?: (error: NormalizedError) => void, -): void { - try { - const timestampMs = Date.now(); - fs.mkdirSync(path.dirname(startupPhasesPath), { recursive: true }); - // A crash mid-append can leave no trailing newline; glue two appends and both are lost. - const separator = endsWithoutNewline(startupPhasesPath) ? '\n' : ''; - fs.appendFileSync( - startupPhasesPath, - `${separator}${JSON.stringify({ data: { startupPhase: phase, timestampMs } })}\n`, - ); - } catch (error) { - reportFailure?.(normalizeError(error)); - } -} - -function endsWithoutNewline(file: string): boolean { - let fd: number | undefined; - try { - const stats = fs.statSync(file); - if (stats.size === 0) return false; - const buffer = Buffer.alloc(1); - fd = fs.openSync(file, 'r'); - if (fs.readSync(fd, buffer, 0, 1, stats.size - 1) !== 1) return false; - return buffer[0] !== 0x0a; - } catch { - return false; - } finally { - if (fd !== undefined) fs.closeSync(fd); - } -} - -/** - * Empties this launch's own record before its first marker: a per-pid name identifies a writer, - * not a lifetime. No caller resets a file named for another pid; a refused reset never stops the - * startup because the reset ran first when it could run. - */ -export function claimDaemonStartupPhasesFile( - startupPhasesPath: string, - reportFailure?: (error: NormalizedError) => void, -): void { - try { - fs.rmSync(startupPhasesPath, { force: true }); - } catch (error) { - reportFailure?.(normalizeError(error)); - } -} - -/** Removes a marker file whose writer is provably gone: a finished startup or a joined exit. */ -export function removeDaemonStartupPhasesFile(startupPhasesPath: string): void { - try { - fs.rmSync(startupPhasesPath, { force: true }); - } catch {} -} - -/** Sweeps marker files of provably dead (or unnameable) pids; a live pid's file is kept. */ -function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { - let entries: string[]; - try { - entries = fs.readdirSync(stateDir); - } catch { - return; - } - for (const entry of entries) { - const match = DAEMON_STARTUP_PHASES_ENTRY.exec(entry); - if (match === null) continue; - const deadPid = Number(match[1]); - if (deadPid === process.pid) continue; - if (!Number.isSafeInteger(deadPid) || deadPid <= 0 || !isProcessAlive(deadPid)) { - removeDaemonStartupPhasesFile(path.join(stateDir, entry)); - } - } -} - -/** - * The phases observed in a marker tail, keeping each phase's earliest occurrence. Unknown and - * unparsable lines are ignored: a client never fails a startup over a line it does not know. - */ -export function readDaemonStartupPhaseMarkers( - markerTail: string, -): Partial> { - const phases: Partial> = {}; - for (const line of markerTail.split('\n')) { - if (!line.includes('"startupPhase"')) continue; - let record: DaemonStartupMarkerRecord; - try { - record = JSON.parse(line) as DaemonStartupMarkerRecord; - } catch { - continue; - } - const phase = record.data?.startupPhase; - const timestampMs = record.data?.timestampMs; - if ( - typeof phase !== 'string' || - !DAEMON_STARTUP_PHASE_SET.has(phase) || - typeof timestampMs !== 'number' || - !Number.isFinite(timestampMs) - ) { - continue; - } - const seen = phases[phase as DaemonStartupPhase]; - if (seen === undefined || timestampMs < seen) phases[phase as DaemonStartupPhase] = timestampMs; - } - return phases; -} - -type DaemonStartupMarkerRecord = { - data?: { startupPhase?: unknown; timestampMs?: unknown }; -}; export type DaemonRegistrationFields = Readonly<{ socketPort?: number; httpPort?: number; diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index eec544823a..aa869d6721 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -87,10 +87,12 @@ import { import { tryAcquireDaemonRegistration, DAEMON_STARTUP_EXIT_CODES, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { recordDaemonStartupMarker, removeDaemonStartupPhasesFile, resolveDaemonStartupPhasesPath, -} from '@agent-device/daemon-contracts/daemon-registration-owner'; +} from '@agent-device/daemon-contracts/daemon-startup-phases'; import { retainPngWorker } from '@agent-device/capture-kit/png-worker-client'; import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; diff --git a/src/__tests__/test-utils/registered-daemon-fixture.test.ts b/src/__tests__/test-utils/registered-daemon-fixture.test.ts index 92a026eeac..37b382ec65 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.test.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.test.ts @@ -4,10 +4,8 @@ import path from 'node:path'; import { test } from 'vitest'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { stopDaemonProcess } from '@agent-device/daemon-contracts/daemon-process'; -import { - DAEMON_STARTUP_EXIT_CODES, - resolveDaemonStartupPhasesPath, -} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { DAEMON_STARTUP_EXIT_CODES } from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { resolveDaemonStartupPhasesPath } from '@agent-device/daemon-contracts/daemon-startup-phases'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { launchRegisteredDaemonFixtureArgs, diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index a29061913a..bbc9807336 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -43,11 +43,16 @@ export function registeredDaemonFixtureArgs( '../../../packages/daemon-contracts/src/daemon-registration-owner.ts', import.meta.url, ).href; + const phasesUrl = new URL( + '../../../packages/daemon-contracts/src/daemon-startup-phases.ts', + import.meta.url, + ).href; fs.writeFileSync( entry, `import fs from 'node:fs'; import path from 'node:path'; -import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration, claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; +import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; +import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(phasesUrl)}; const paths = ${JSON.stringify(paths)}; const barrier = ${JSON.stringify(acquisitionBarrier)}; const emitBefore = ${fixtureOptions?.emitStartupMarkerBeforeAcquisition ? 'true' : 'false'}; diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index 52da7deb62..b02220d85c 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -9,16 +9,18 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { type ExecDetachedExit } from '@agent-device/host-kit/command'; import { DAEMON_STARTUP_EXIT_CODES, - readDaemonStartupPhaseMarkers, recoverAbandonedDaemonRegistration, - removeDaemonStartupPhasesFile, - resolveDaemonStartupPhasesPath, type DaemonRetirementResult, - type DaemonStartupPhase, launchDaemonProcess, stopAndRetireDaemon, type DaemonStartupLaunch, } from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { + readDaemonStartupPhaseMarkers, + removeDaemonStartupPhasesFile, + resolveDaemonStartupPhasesPath, + type DaemonStartupPhase, +} from '@agent-device/daemon-contracts/daemon-startup-phases'; import type { DaemonPaths, DaemonTransportPreference, diff --git a/src/daemon.ts b/src/daemon.ts index 8ad7bc3f96..32635c1cfe 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -5,7 +5,7 @@ import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath, -} from '@agent-device/daemon-contracts/daemon-registration-owner'; +} from '@agent-device/daemon-contracts/daemon-startup-phases'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; // A refused marker append goes to stderr, which the launcher redirects into the daemon log. @@ -13,8 +13,11 @@ function reportMarkerFailure(error: unknown): void { process.stderr.write(`Daemon startup marker write failed: ${normalizeError(error).message}\n`); } +// Captured once: the earliest marker and the runtime resolve the state dir from this one host, +// so a relative AGENT_DEVICE_STATE_DIR cannot land the markers beside another directory's lock. +const host = captureHostEnvironment(process); const ownPhasesPath = resolveDaemonStartupPhasesPath( - resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR).baseDir, + resolveDaemonPaths(host.env.AGENT_DEVICE_STATE_DIR, { env: host.env, cwd: host.cwd }).baseDir, process.pid, ); // The claim first: after pid reuse it is the only defense against a dead lifetime's phases. @@ -28,7 +31,7 @@ recordDaemonStartupMarker(ownPhasesPath, 'modules-loaded', reportMarkerFailure); // closure of this entry stays exactly the operational server's (ADR 0019). async function runDaemon(): Promise { const runtime = await startDaemonRuntime({ - host: captureHostEnvironment(process), + host, composeDeps: async (input) => { const deps = await (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input); recordDaemonStartupMarker( From d4b27ba493897a00d5885155c3b7c848dc45bdd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 14:42:12 +0200 Subject: [PATCH 20/22] test(daemon): prove the progress-aware budget on successive phases The headline slow-start test rewrote lock-acquired repeatedly; the reader keeps each phase's earliest timestamp, so those writes never advanced progress and the run spanned only 20 s against the 60 s stall window. It now schedules four distinct phases (modules-loaded, composition-loaded, lock-acquired, recovery-finished) across 70 s of virtual time under a 150 s ceiling and asserts the wait survived on observed successive progress. With the marker read removed, the same scenario fails as startup-stalled at the 60 s window (verified with a throwaway stub, not committed), so the test is credibly red without the progress rule. Addresses review 6248815887 item 3 on #3791. Refs #3782, #3791 --- .../daemon-client-startup-budget.test.ts | 53 +++++++++++++++---- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 0d03971373..26342b3d71 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -4,11 +4,13 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import type { ExecDetachedExit } from '@agent-device/host-kit/command'; +import { DAEMON_STARTUP_EXIT_CODES } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { - DAEMON_STARTUP_EXIT_CODES, + readDaemonStartupPhaseMarkers, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath, -} from '@agent-device/daemon-contracts/daemon-registration-owner'; + type DaemonStartupPhase, +} from '@agent-device/daemon-contracts/daemon-startup-phases'; import { resolveDaemonPaths, type DaemonPaths, @@ -134,9 +136,20 @@ async function awaitFile(file: string) { test('a daemon slower than the retired 15 s budget starts successfully while it keeps progressing', async (t) => { if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '150000'); const http = await startHttpDaemonFixture({ devices: [] }); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-slow-success-')); - // Virtual time waits ~20 s past the retired 15 s deadline; real markers hold off the stall. + // Virtual time runs past the 60 s stall window with the ceiling above the span, so only the + // successive phase markers keep this wait alive: each phase is new, so the last observed + // progress never goes 60 s stale, while the same span with no readable markers fails as + // startup-stalled. A repeated rewrite of one phase would not advance progress, because the + // reader keeps each phase's earliest timestamp. + const phaseSchedule: Array = [ + ['modules-loaded', 10_000], + ['composition-loaded', 25_000], + ['lock-acquired', 40_000], + ['recovery-finished', 70_000], + ]; const startedAt = Date.now(); const now = { value: startedAt }; vi.spyOn(Date, 'now').mockImplementation(() => now.value); @@ -149,21 +162,25 @@ test('a daemon slower than the retired 15 s budget starts successfully while it return child; }); fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); - let polls = 0; + let nextPhase = 0; pause.mockImplementation(async () => { - polls += 1; if (!booted) { booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); if (!booted) return await realPause(); } - if (now.value - spawned.at < 20_000) { + if (spawned.child && now.value - spawned.at < 70_000) { now.value += 2_000; - if (polls % 3 === 0 && spawned.child) + while ( + nextPhase < phaseSchedule.length && + now.value - spawned.at >= phaseSchedule[nextPhase]![1] + ) { recordDaemonStartupMarker( resolveDaemonStartupPhasesPath(paths.baseDir, spawned.child.pid), - 'lock-acquired', + phaseSchedule[nextPhase]![0], ); - if (now.value - spawned.at >= 20_000) + nextPhase += 1; + } + if (nextPhase === phaseSchedule.length) fs.rmSync(path.join(paths.baseDir, 'defer-publication'), { force: true }); return await realPause(); } @@ -172,13 +189,27 @@ test('a daemon slower than the retired 15 s budget starts successfully while it try { const response = await sendToDaemon(request(paths)); assert.equal(response.ok, true); + const elapsed = now.value - spawned.at; assert.ok( - now.value - spawned.at >= 20_000, - 'the startup this test proves must exceed the retired 15 s budget', + elapsed >= 60_000, + `the startup this test proves must outlast the stall window (${elapsed}ms)`, ); assert.equal(spawn.mock.calls.length, 1); assert.equal(http.rpcRequests.length, 1); assert.ok(spawned.child && isProcessAlive(spawned.child.pid), 'the client must not retire it'); + assert.ok(spawned.child); + // The wait survived the stall window on distinct successive phases, not on one repeated + // marker: every scheduled phase is in this launch's own record. + const markers = readDaemonStartupPhaseMarkers( + fs.readFileSync(resolveDaemonStartupPhasesPath(paths.baseDir, spawned.child.pid), 'utf8'), + ); + for (const [phase] of phaseSchedule) { + assert.equal( + typeof markers[phase], + 'number', + `the observed progress must include the ${phase} phase`, + ); + } } finally { await closeLoopbackServer(http.server); } From d22edfef3ce705cbf044a288e59b2368e4674366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 15:11:48 +0200 Subject: [PATCH 21/22] refactor(daemon): converge the marker helpers and shared fixture waits --- .github/workflows/ios.yml | 14 +- .../src/daemon-registration-owner.ts | 1 - .../src/daemon-startup-phases.test.ts | 33 ++--- .../src/daemon-startup-phases.ts | 21 +-- scripts/__tests__/eager-closure-budgets.ts | 2 +- .../registered-daemon-fixture.test.ts | 16 +- .../test-utils/registered-daemon-fixture.ts | 16 +- .../daemon-client-startup-budget.test.ts | 137 +++++++----------- .../daemon-client-startup-race.test.ts | 4 +- src/daemon.ts | 8 +- 10 files changed, 98 insertions(+), 154 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 873399c608..c138a600d9 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -171,17 +171,11 @@ jobs: pnpm clean:daemon # The public-SDK record is the cold-startup evidence; the CLI timer only wraps attach. node --input-type=module -e ' - import { performance } from "node:perf_hooks"; + const { performance } = await import("node:perf_hooks"); const { ensureDaemon } = await import("./dist/src/index.js"); - const stateDir = process.env.AGENT_DEVICE_STATE_DIR; - const startedAt = performance.now(); - const connection = await ensureDaemon({ stateDir }); - console.log(JSON.stringify({ - phase: "daemon_cold_startup", - durationMs: Math.round(performance.now() - startedAt), - started: connection.started, - pid: connection.daemon.pid, - })); + const t0 = performance.now(); + const c = await ensureDaemon({ stateDir: process.env.AGENT_DEVICE_STATE_DIR }); + console.log(JSON.stringify({ phase: "daemon_cold_startup", started: c.started, pid: c.daemon.pid, durationMs: Math.round(performance.now() - t0) })); ' node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index bf280ca4e1..2e452699ad 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -47,7 +47,6 @@ import { import { sweepDeadDaemonStartupPhasesFiles } from './daemon-startup-phases.ts'; export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); - export type DaemonRegistrationFields = Readonly<{ socketPort?: number; httpPort?: number; diff --git a/packages/daemon-contracts/src/daemon-startup-phases.test.ts b/packages/daemon-contracts/src/daemon-startup-phases.test.ts index 1182de4348..513567f921 100644 --- a/packages/daemon-contracts/src/daemon-startup-phases.test.ts +++ b/packages/daemon-contracts/src/daemon-startup-phases.test.ts @@ -5,7 +5,6 @@ import path from 'node:path'; import { expect, test } from 'vitest'; import { tryAcquireDaemonRegistration } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { - claimDaemonStartupPhasesFile, readDaemonStartupPhaseMarkers, recordDaemonStartupMarker, removeDaemonStartupPhasesFile, @@ -50,7 +49,7 @@ test('marker reading ignores unknown phases and unparsable lines', () => { }); test('the producer lifetime claim empties a same-pid leftover', async () => { - // Pid reuse after both the old client and daemon died before either cleanup: the entry claims + // Pid reuse after both the old client and daemon died before either cleanup: the entry removes // its record before its first marker; this child repeats that producer sequence itself. const stateDir = await mkdtempForTest('agent-device-markers-'); writeFileSync(path.join(stateDir, 'package.json'), '{"type":"module"}'); @@ -58,11 +57,11 @@ test('the producer lifetime claim empties a same-pid leftover', async () => { const registrationUrl = new URL('./daemon-startup-phases.ts', import.meta.url).href; writeFileSync( entry, - `import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; + `import { removeDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(registrationUrl)}; const file = resolveDaemonStartupPhasesPath(${JSON.stringify(stateDir)}, process.pid); recordDaemonStartupMarker(file, 'listening'); recordDaemonStartupMarker(file, 'registered'); -claimDaemonStartupPhasesFile(file); +removeDaemonStartupPhasesFile(file); recordDaemonStartupMarker(file, 'modules-loaded'); `, ); @@ -76,13 +75,19 @@ recordDaemonStartupMarker(file, 'modules-loaded'); expect(phases).toEqual({ 'modules-loaded': expect.any(Number) }); }); -test('a refused claim reset reports through the callback instead of throwing', async () => { +test.each([ + ['remove', (p: string, cb: (e: unknown) => void) => removeDaemonStartupPhasesFile(p, cb)], + [ + 'record', + (p: string, cb: (e: unknown) => void) => recordDaemonStartupMarker(p, 'modules-loaded', cb), + ], +] as const)('a refused %s reports through the callback instead of throwing', async (_name, run) => { const stateDir = await mkdtempForTest('agent-device-markers-'); - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 778); + const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); mkdirSync(markerPath, { recursive: true }); const reported: unknown[] = []; expect(() => - claimDaemonStartupPhasesFile(markerPath, (error) => { + run(markerPath, (error) => { reported.push(error); }), ).not.toThrow(); @@ -121,20 +126,6 @@ test('removeDaemonStartupPhasesFile removes a file and tolerates one already gon expect(() => removeDaemonStartupPhasesFile(file)).not.toThrow(); }); -test('a marker write failure reports through the callback, keeping startup running', async () => { - const stateDir = await mkdtempForTest('agent-device-markers-'); - const markerPath = resolveDaemonStartupPhasesPath(stateDir, 777); - mkdirSync(markerPath, { recursive: true }); - const reported: unknown[] = []; - expect(() => - recordDaemonStartupMarker(markerPath, 'modules-loaded', (error) => { - reported.push(error); - }), - ).not.toThrow(); - expect(reported).toHaveLength(1); - expect(reported[0]).toHaveProperty('message'); -}); - test('a crash-replay append keeps the file valid JSONL', async () => { const stateDir = await mkdtempForTest('agent-device-markers-'); const markerPath = resolveDaemonStartupPhasesPath(stateDir, 999); diff --git a/packages/daemon-contracts/src/daemon-startup-phases.ts b/packages/daemon-contracts/src/daemon-startup-phases.ts index 1a5f746da4..129fb4b963 100644 --- a/packages/daemon-contracts/src/daemon-startup-phases.ts +++ b/packages/daemon-contracts/src/daemon-startup-phases.ts @@ -5,9 +5,9 @@ import { isProcessAlive } from '@agent-device/host-kit/process'; /** * Startup phase markers: a per-launch record, named for the writing pid, of how far that startup - * got, so a waiting client tells "slow but progressing" from "stuck" without a timestamp and with - * no heartbeat. The writer removes it on completion, the launching client after a joined exit, and - * each new registration owner sweeps the files of provably dead pids. + * got, so a waiting client tells "slow but progressing" from "stuck" without a heartbeat. The + * writer removes it on completion, the client after a joined exit, and each new registration + * owner sweeps the files of provably dead pids. */ const DAEMON_STARTUP_PHASES = Object.freeze([ 'modules-loaded', @@ -66,11 +66,11 @@ function endsWithoutNewline(file: string): boolean { } /** - * Empties this launch's own record before its first marker: a per-pid name identifies a writer, - * not a lifetime. No caller resets a file named for another pid; a refused reset never stops the - * startup because the reset ran first when it could run. + * Removes this launch's record: before its first marker (a per-pid name identifies a writer, not + * a lifetime, so the entry claims its file first), and after a finished startup or joined exit. + * A refused removal only reports through `reportFailure`; it never stops a startup. */ -export function claimDaemonStartupPhasesFile( +export function removeDaemonStartupPhasesFile( startupPhasesPath: string, reportFailure?: (error: NormalizedError) => void, ): void { @@ -81,13 +81,6 @@ export function claimDaemonStartupPhasesFile( } } -/** Removes a marker file whose writer is provably gone: a finished startup or a joined exit. */ -export function removeDaemonStartupPhasesFile(startupPhasesPath: string): void { - try { - fs.rmSync(startupPhasesPath, { force: true }); - } catch {} -} - /** Sweeps marker files of provably dead (or unnameable) pids; a live pid's file is kept. */ export function sweepDeadDaemonStartupPhasesFiles(stateDir: string): void { let entries: string[]; diff --git a/scripts/__tests__/eager-closure-budgets.ts b/scripts/__tests__/eager-closure-budgets.ts index 012ed1c46b..403ec79032 100644 --- a/scripts/__tests__/eager-closure-budgets.ts +++ b/scripts/__tests__/eager-closure-budgets.ts @@ -156,7 +156,7 @@ export const APPROVED_CLOSURE_ADDITIONS: readonly Readonly<{ module: 'packages/daemon-contracts/src/daemon-startup-phases.ts', issue: '#3782', reason: - 'The startup phase record owner the #3791 review asked for (its own file/format/writer/reader/sweep beside daemon-shutdown-report). The daemon entry records its earliest `modules-loaded` marker at module load, so the owner must be eager there; the entry pays for the record it writes first.', + 'The startup phase record owner the #3791 review asked for. The daemon entry records its earliest `modules-loaded` marker at module load, so the owner must be eager there; the entry pays for the record it writes first.', owner: 'thymikee', }, ]); diff --git a/src/__tests__/test-utils/registered-daemon-fixture.test.ts b/src/__tests__/test-utils/registered-daemon-fixture.test.ts index 37b382ec65..d179429605 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.test.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.test.ts @@ -11,6 +11,7 @@ import { launchRegisteredDaemonFixtureArgs, registeredDaemonFixtureArgs, spawnRegisteredDaemonFixture, + waitForFixtureFile, waitForRegisteredDaemonFixture, finishRegisteredDaemonFixture, } from './registered-daemon-fixture.ts'; @@ -36,21 +37,12 @@ test('args built before any launch keep their own entry script under a concurren assert.notEqual(firstArgs[1], secondArgs[1]); const first = launchRegisteredDaemonFixtureArgs(paths, firstArgs, { stdio: 'ignore' }); try { - const held = path.join(paths.baseDir, `registration-held-${first.pid}`); - for (let attempt = 0; !fs.existsSync(held); attempt += 1) { - assert.ok(attempt < 400, 'child 1 never acquired the registration'); - await new Promise((resolve) => setTimeout(resolve, 10)); - } + await waitForFixtureFile(path.join(paths.baseDir, `registration-held-${first.pid}`)); const second = launchRegisteredDaemonFixtureArgs(paths, secondArgs, { stdio: 'ignore' }); assert.equal((await second.exited).exitCode, DAEMON_STARTUP_EXIT_CODES.busy); - const ownRecord = resolveDaemonStartupPhasesPath(paths.baseDir, first.pid); - const text = fs.readFileSync(ownRecord, 'utf8'); + const text = fs.readFileSync(resolveDaemonStartupPhasesPath(paths.baseDir, first.pid), 'utf8'); assert.match(text, /modules-loaded/); - assert.doesNotMatch( - text, - /lock-acquired/, - "child 1 must run the script built for it, not the second build's switches", - ); + assert.doesNotMatch(text, /lock-acquired/, 'child 1 must run the script built for it'); } finally { await finishRegisteredDaemonFixture(paths.baseDir); } diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index bbc9807336..295a78ae51 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -52,7 +52,7 @@ export function registeredDaemonFixtureArgs( `import fs from 'node:fs'; import path from 'node:path'; import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; -import { claimDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(phasesUrl)}; +import { removeDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath } from ${JSON.stringify(phasesUrl)}; const paths = ${JSON.stringify(paths)}; const barrier = ${JSON.stringify(acquisitionBarrier)}; const emitBefore = ${fixtureOptions?.emitStartupMarkerBeforeAcquisition ? 'true' : 'false'}; @@ -61,11 +61,7 @@ if (barrier) { fs.writeFileSync(barrier + '.ready-' + process.pid, 'ready'); while (!fs.existsSync(barrier)) await new Promise(resolve => setTimeout(resolve, 10)); } -if (emitBefore) { - const ownRecord = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); - claimDaemonStartupPhasesFile(ownRecord); - recordDaemonStartupMarker(ownRecord, 'modules-loaded'); -} +if (emitBefore) { const own = resolveDaemonStartupPhasesPath(paths.baseDir, process.pid); removeDaemonStartupPhasesFile(own); recordDaemonStartupMarker(own, 'modules-loaded'); } const acquired = await tryAcquireDaemonRegistration(paths); if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); if (emitAfter) @@ -117,6 +113,14 @@ export function spawnRegisteredDaemonFixture( ); } +/** Waits up to 4 s for a fixture child to publish `file`; one shared wait for every probe. */ +export async function waitForFixtureFile(file: string): Promise { + for (let attempt = 0; !fs.existsSync(file); attempt += 1) { + assert.ok(attempt < 400, `fixture did not publish ${file}`); + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + export async function waitForRegisteredDaemonFixture( paths: DaemonPaths, child: ReturnType, diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 26342b3d71..634e277c78 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -25,6 +25,7 @@ import { readVersion } from '@agent-device/host-kit/version'; import { spawnRegisteredDaemonFixture, finishRegisteredDaemonFixtures, + waitForFixtureFile, } from '../../__tests__/test-utils/registered-daemon-fixture.ts'; import { startHttpDaemonFixture, @@ -127,11 +128,21 @@ async function realPause(ms = 5) { await actualRetry.sleep(ms); } -async function awaitFile(file: string) { - for (let attempt = 0; !fs.existsSync(file); attempt += 1) { - assert.ok(attempt < 400, `fixture did not publish ${file}`); - await actualRetry.sleep(10); - } +/** Pins `Date.now` to a mutable value: advancing the wait costs no real time. */ +function virtualClock(): { value: number } { + const now = { value: Date.now() }; + vi.spyOn(Date, 'now').mockImplementation(() => now.value); + return now; +} + +/** The startup phase records left in this state dir, by file name. */ +function phaseRecordEntries(paths: DaemonPaths): string[] { + return fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')); +} + +function registrationBooted(paths: DaemonPaths): () => boolean { + let booted = false; + return () => (booted ||= fs.existsSync(path.join(paths.baseDir, 'registration-held'))); } test('a daemon slower than the retired 15 s budget starts successfully while it keeps progressing', async (t) => { @@ -139,22 +150,18 @@ test('a daemon slower than the retired 15 s budget starts successfully while it vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '150000'); const http = await startHttpDaemonFixture({ devices: [] }); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-slow-success-')); - // Virtual time runs past the 60 s stall window with the ceiling above the span, so only the - // successive phase markers keep this wait alive: each phase is new, so the last observed - // progress never goes 60 s stale, while the same span with no readable markers fails as - // startup-stalled. A repeated rewrite of one phase would not advance progress, because the - // reader keeps each phase's earliest timestamp. - const phaseSchedule: Array = [ - ['modules-loaded', 10_000], - ['composition-loaded', 25_000], - ['lock-acquired', 40_000], - ['recovery-finished', 70_000], + // The success must rest on the markers: four successive phases 20 s apart across 80 s of virtual + // time under a 150 s ceiling keep progress inside the 60 s stall window; the same span with no + // readable markers fails as startup-stalled, one rewritten phase never advances progress. + const phases: readonly DaemonStartupPhase[] = [ + 'modules-loaded', + 'composition-loaded', + 'lock-acquired', + 'recovery-finished', ]; - const startedAt = Date.now(); - const now = { value: startedAt }; - vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const now = virtualClock(); const spawned = { at: 0, child: undefined as { pid: number } | undefined }; - let booted = false; + const booted = registrationBooted(paths); spawn.mockImplementation((_command, _args, options) => { const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); spawned.at = now.value; @@ -164,23 +171,17 @@ test('a daemon slower than the retired 15 s budget starts successfully while it fs.writeFileSync(path.join(paths.baseDir, 'defer-publication'), 'wait'); let nextPhase = 0; pause.mockImplementation(async () => { - if (!booted) { - booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); - if (!booted) return await realPause(); - } - if (spawned.child && now.value - spawned.at < 70_000) { - now.value += 2_000; - while ( - nextPhase < phaseSchedule.length && - now.value - spawned.at >= phaseSchedule[nextPhase]![1] - ) { + if (!booted()) return await realPause(); + if (spawned.child && now.value - spawned.at < 80_000) { + now.value += 20_000; + if (nextPhase < phases.length) { recordDaemonStartupMarker( resolveDaemonStartupPhasesPath(paths.baseDir, spawned.child.pid), - phaseSchedule[nextPhase]![0], + phases[nextPhase]!, ); nextPhase += 1; } - if (nextPhase === phaseSchedule.length) + if (nextPhase === phases.length) fs.rmSync(path.join(paths.baseDir, 'defer-publication'), { force: true }); return await realPause(); } @@ -189,26 +190,16 @@ test('a daemon slower than the retired 15 s budget starts successfully while it try { const response = await sendToDaemon(request(paths)); assert.equal(response.ok, true); - const elapsed = now.value - spawned.at; - assert.ok( - elapsed >= 60_000, - `the startup this test proves must outlast the stall window (${elapsed}ms)`, - ); + assert.ok(now.value - spawned.at >= 60_000, 'this startup must outlast the stall window'); assert.equal(spawn.mock.calls.length, 1); assert.equal(http.rpcRequests.length, 1); assert.ok(spawned.child && isProcessAlive(spawned.child.pid), 'the client must not retire it'); - assert.ok(spawned.child); - // The wait survived the stall window on distinct successive phases, not on one repeated - // marker: every scheduled phase is in this launch's own record. + // The wait survived on four distinct phases, not one repeated marker. const markers = readDaemonStartupPhaseMarkers( - fs.readFileSync(resolveDaemonStartupPhasesPath(paths.baseDir, spawned.child.pid), 'utf8'), + fs.readFileSync(resolveDaemonStartupPhasesPath(paths.baseDir, spawned.child!.pid), 'utf8'), ); - for (const [phase] of phaseSchedule) { - assert.equal( - typeof markers[phase], - 'number', - `the observed progress must include the ${phase} phase`, - ); + for (const phase of phases) { + assert.equal(typeof markers[phase], 'number', `the ${phase} phase was never observed`); } } finally { await closeLoopbackServer(http.server); @@ -244,21 +235,17 @@ test('a daemon that exits during startup fails with startup-exited within one se test('a live daemon that publishes no startup marker is retired as stalled before the ceiling', async () => { vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '120000'); - const started = Date.now(); - const now = { value: started }; - vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const now = virtualClock(); + const started = now.value; const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-no-marker-')); let child: ReturnType | undefined; spawn.mockImplementation((_command, _args, options) => { child = spawnUnregisteredDaemonChild(paths, options); return child; }); - let booted = false; + const booted = registrationBooted(paths); pause.mockImplementation(async () => { - if (!booted) { - booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); - if (!booted) return await realPause(); - } + if (!booted()) return await realPause(); if (now.value - started < 70_000) now.value += 5_000; await realPause(); }); @@ -276,7 +263,7 @@ test('a live daemon that publishes no startup marker is retired as stalled befor assert.equal(spawn.mock.calls.length, 1, 'a stalled daemon is not retried'); await (child?.exited ?? Promise.resolve()); assert.deepEqual( - fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + phaseRecordEntries(paths), [], 'a joined startup must sweep its own phase-marker file', ); @@ -287,25 +274,17 @@ test('a live daemon that publishes no startup marker is retired as stalled befor test('a live daemon that keeps progressing but never registers fails at the env ceiling with phases', async () => { vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '30000'); - const started = Date.now(); - const now = { value: started }; - vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const now = virtualClock(); + const started = now.value; const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-ceiling-')); let child: ReturnType | undefined; - let booted = false; - const spawned = { at: 0 }; + const booted = registrationBooted(paths); spawn.mockImplementation((_command, _args, options) => { child = spawnUnregisteredDaemonChild(paths, options, true); - spawned.at = now.value; return child; }); - let polls = 0; pause.mockImplementation(async () => { - polls += 1; - if (!booted) { - booted = fs.existsSync(path.join(paths.baseDir, 'registration-held')); - if (!booted) return await realPause(); - } + if (!booted()) return await realPause(); if (now.value - started < 30_000) now.value += 1_000; await realPause(); }); @@ -335,18 +314,15 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier const leftover = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }, undefined, { emitStartupMarker: true, }); - await awaitFile(registrationHeldFor(paths, leftover.pid)); + await waitForFixtureFile(registrationHeldFor(paths, leftover.pid)); process.kill(leftover.pid, 'SIGTERM'); await leftover.exited; - const leftoverRecords = () => - fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')); assert.ok( - leftoverRecords().length > 0, + phaseRecordEntries(paths).length > 0, 'the leftover launch must have left a real daemon-written marker record', ); - const started = Date.now(); - const now = { value: started }; - vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const now = virtualClock(); + const started = now.value; let child: ReturnType | undefined; let booted = false; spawn.mockImplementation((_command, _args, options) => { @@ -372,7 +348,7 @@ test("a startup progressed by a fresh launch is not judged stalled by an earlier ); await (child?.exited ?? Promise.resolve()); assert.ok( - !leftoverRecords().some((entry) => entry.includes(String(child?.pid))), + !phaseRecordEntries(paths).some((entry) => entry.includes(String(child?.pid))), "the joined exit must sweep this launch's own marker file", ); } finally { @@ -388,7 +364,7 @@ test('a busy exit while a live lock holder defers publication reaches the ceilin const winner = spawnRegisteredDaemonFixture(paths, fields(1), { stdio: 'ignore' }, undefined, { emitStartupMarker: true, }); - await awaitFile(registrationHeldFor(paths, winner.pid)); + await waitForFixtureFile(registrationHeldFor(paths, winner.pid)); let contender: ReturnType | undefined; spawn.mockImplementation((_command, _args, options) => { contender = spawnRegisteredDaemonFixture(paths, fields(1), options, undefined, { @@ -413,10 +389,7 @@ test('a busy exit while a live lock holder defers publication reaches the ceilin ); assert.ok(contender); assert.deepEqual( - fs - .readdirSync(paths.baseDir) - .filter((entry) => entry.startsWith('daemon-startup-phases')) - .sort(), + phaseRecordEntries(paths).sort(), [path.basename(resolveDaemonStartupPhasesPath(paths.baseDir, winner.pid))], 'the exited contender record must be swept; only the live winner record may remain', ); @@ -457,9 +430,7 @@ test('an observed exit on a reused launch pid is not a live startup at the deadl }), readIdentity: () => ({ pid: reusedPid, startTime: null }), })); - const started = Date.now(); - const now = { value: started }; - vi.spyOn(Date, 'now').mockImplementation(() => now.value); + const now = virtualClock(); let advanced = false; pause.mockImplementation(async () => { if (!advanced) { diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index 11078b40ed..d1c1a26430 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -421,7 +421,7 @@ test('a busy contender that exits sweeps its own pre-lock marker before the retr error instanceof AppError && error.details?.kind === 'daemon_startup_failed', ); assert.deepEqual( - fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + fs.readdirSync(paths.baseDir).filter((e) => e.startsWith('daemon-startup-phases')), [], "the retry result must not leave the joined busy contender's record behind", ); @@ -469,7 +469,7 @@ test('a joined busy contender waits for a published winner to become ready witho false, ); assert.deepEqual( - fs.readdirSync(paths.baseDir).filter((entry) => entry.startsWith('daemon-startup-phases')), + fs.readdirSync(paths.baseDir).filter((e) => e.startsWith('daemon-startup-phases')), [], "the winner handoff must not leave the exited contender's record behind", ); diff --git a/src/daemon.ts b/src/daemon.ts index 32635c1cfe..471280fd5e 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -2,16 +2,16 @@ import { asAppError, normalizeError } from '@agent-device/kernel/errors'; import { daemonStartupExitCode, reportDaemonClose, startDaemonRuntime } from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import { - claimDaemonStartupPhasesFile, + removeDaemonStartupPhasesFile, recordDaemonStartupMarker, resolveDaemonStartupPhasesPath, } from '@agent-device/daemon-contracts/daemon-startup-phases'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; // A refused marker append goes to stderr, which the launcher redirects into the daemon log. -function reportMarkerFailure(error: unknown): void { +const reportMarkerFailure = (error: unknown): void => { process.stderr.write(`Daemon startup marker write failed: ${normalizeError(error).message}\n`); -} +}; // Captured once: the earliest marker and the runtime resolve the state dir from this one host, // so a relative AGENT_DEVICE_STATE_DIR cannot land the markers beside another directory's lock. @@ -21,7 +21,7 @@ const ownPhasesPath = resolveDaemonStartupPhasesPath( process.pid, ); // The claim first: after pid reuse it is the only defense against a dead lifetime's phases. -claimDaemonStartupPhasesFile(ownPhasesPath, reportMarkerFailure); +removeDaemonStartupPhasesFile(ownPhasesPath, reportMarkerFailure); recordDaemonStartupMarker(ownPhasesPath, 'modules-loaded', reportMarkerFailure); // The executable owns the process: signals, fatal-error handlers and the exit code. The runtime From 0295ee5f23dab5290c4e0ccc0be535383eb3805f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 16:13:35 +0200 Subject: [PATCH 22/22] fix(daemon): scope the startup-ceiling claim to the owned launch and prove the default ceiling The help and configuration docs claimed only a progressing daemon reaches a ceiling above the stall window. On the contended path this client's launch exits busy and the wait runs on the exit branch, where the stall check never runs, so a live startup owned by another client reaches the ceiling with no observed progress. Qualify the clause to this client's own launch, matching the CHANGELOG definition. The no-marker budget test stubbed 120000 instead of unsetting the variable, so it never proved the default. Unset it (stubEnv with undefined) so the resolved ceiling is the unoverridden default even on a host that exports the variable. Also drop a stray blank line in the Unreleased list. --- CHANGELOG.md | 1 - src/commands/schema/cli-help.ts | 2 +- .../__tests__/daemon-client-startup-budget.test.ts | 3 ++- website/docs/docs/configuration.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71c517c39..adb4e8a75f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ - The PNG worker no longer calls `process.emitWarning` when it is unavailable, and it starts with an empty environment (`env: {}`). The daemon registers a sink when it pre-warms the worker, so every transition to unavailable (spawn failure, missing entry, load error, exit) leaves exactly one `png_worker_unavailable` event, with a typed `reason` (`entry-not-found`, `spawn-failed`, `worker-error`, `worker-exited`) and its `detail`, in `daemon.log`, with or without `--debug`. Requests then continue on the in-process fallback without repeating the event. - Behavior change: `resolveHubAppReference`, shipped through `agent-device/plugins/webdriver`, now needs `cwd` to resolve a relative app path. Without it the call fails with `INVALID_ARGS` and `details.reason: provider_app_cwd_missing`; it used to resolve against the process's working directory. Absolute paths, URLs and provider references are unaffected. The daemon fills it from the client's working directory (or its host's) for every provider lease request, so `connect` and `lease_allocate` are unchanged. - A client starting a local daemon now waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower instead of failing it. The wait stops at a ceiling of 120 s by default; set `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` to a positive integer number of milliseconds to override it for the CLI, MCP, and Node clients alike. The variable is read only when the client must start a daemon; a value that is not a positive integer fails with `INVALID_ARGS` before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A failed startup names the outcome in `error.details.reason`, and you should branch on it, never on the message: `startup-exited` (the process ended before registering), `startup-stalled` (alive but no startup progress for a fixed 60 s window, which the ceiling variable does not change; the daemon is stopped), or `startup-ceiling` (a startup was still live — this client's daemon or another client's — when the ceiling passed; the client stops its own launch and never signals another client's live daemon). `details.phases` carries the startup phase timings the client observed and `details.startupTimeoutMs` the effective ceiling. - - The `daemon-not-running` hint now always names `--state-dir `, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`. - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir `. - `@agent-device/testmu` is now a version 2 provider plugin. `import testmu from '@agent-device/testmu'` gives a factory, `testmu({ credentials: { username, accessKey } })`, that reads those credentials instead of `LT_USERNAME` and `LT_ACCESS_KEY`; without options it reads the variables as before. The package still declares the version 1 plugin entry, so `agent-device plugins add @agent-device/testmu` keeps working on hosts that predate the version 2 contract. diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 012e339873..12fbdb0894 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -512,7 +512,7 @@ Runner and daemon lifecycle (applies to simulators too): No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up. close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision. Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. - A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window, so a ceiling set below 60 s also ends silent startups this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. + A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts index 634e277c78..577aedb236 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-budget.test.ts @@ -234,7 +234,8 @@ test('a daemon that exits during startup fails with startup-exited within one se }); test('a live daemon that publishes no startup marker is retired as stalled before the ceiling', async () => { - vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', '120000'); + // Unsets the variable, even when the host exports it: the resolved ceiling must be the default. + vi.stubEnv('AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS', undefined); const now = virtualClock(); const started = now.value; const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-no-marker-')); diff --git a/website/docs/docs/configuration.md b/website/docs/docs/configuration.md index 2a79cfff9b..b326fa917d 100644 --- a/website/docs/docs/configuration.md +++ b/website/docs/docs/configuration.md @@ -122,7 +122,7 @@ Only the variables below, and those documented on command pages, are supported. | Category | Env vars | Decision | | --- | --- | --- | -| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (a positive integer number of milliseconds, default `120000`; any other value fails with `INVALID_ARGS` before a daemon spawns, and only when the client must start a daemon — a client that attaches to or reuses a running daemon never reads it) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for a fixed 60 s window this variable does not change), or `startup-ceiling` (alive, not registered by the effective ceiling; only a progressing daemon reaches a ceiling above the stall window). Branch on `details.reason`, never on the message. | +| CLI defaults and config | `AGENT_DEVICE_HOME`, `AGENT_DEVICE_CONFIG`, `AGENT_DEVICE_SESSION`, `AGENT_DEVICE_PLATFORM`, `AGENT_DEVICE_SCREENSHOT_SCALE`, `AGENT_DEVICE_SESSION_LOCK`, `AGENT_DEVICE_DAEMON_BASE_URL`, `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS`, `AGENT_DEVICE_CLOUD_BASE_URL` | Public. `AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS` is the ceiling (a positive integer number of milliseconds, default `120000`; any other value fails with `INVALID_ARGS` before a daemon spawns, and only when the client must start a daemon — a client that attaches to or reuses a running daemon never reads it) a client waits for a local daemon it starts to register; CLI, MCP, and Node clients share it. A live daemon is waited for while it records startup progress, so an overloaded host slows startup instead of failing it. A failed startup names the outcome in `details.reason`: `startup-exited` (the process ended before registering), `startup-stalled` (alive, no startup progress for a fixed 60 s window this variable does not change), or `startup-ceiling` (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the fixed stall window). Branch on `details.reason`, never on the message. | | Device scoping | `AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST`, `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` | Public. `AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET` is read by the daemon: every iOS simulator request that names no `--ios-simulator-device-set` uses that set, so iOS physical devices are not listed, and a request naming a different path string fails with `INVALID_ARGS` (`details.reason: "ios_simulator_device_set_outside_daemon_set"`). macOS and non-iOS requests ignore it. A daemon started by the CLI inherits the CLI's value; restart the daemon after changing it. | | Android test IME | `AGENT_DEVICE_TEST_IME` | Public. Same setting as `--test-ime` / `--no-test-ime` on `open`, `test`, and `replay`; see known limitations. | | Local daemon storage | `AGENT_DEVICE_STATE_DIR` | Public |