fix(daemon-client): document the drain timeout and tighten ensureDaemon follow-ups

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- The `daemon-not-running` hint now always names `--state-dir <dir>`, and a daemon force-retired for outlasting its drain is reported as `Replaced unreachable daemon` instead of `Replaced idle daemon`. `daemon-drain-timeout` and the post-drain force retire are documented in the client API and `help`.
 - Fix `ensureDaemon()`, the CLI and MCP killing a daemon they had just asked to leave. After the daemon answers the quiesce request with `draining`, the client now waits up to 60 s for it to exit by itself and clean up its registration, so plugin disposal and lease finalization finish (on Windows a signal ends the process outright, and on POSIX a drain longer than 3 s was killed). A daemon still running at the deadline that fails its liveness probes is force-retired; one that still answers is left running and the call fails with `details.reason: 'daemon-drain-timeout'`, with a hint naming `agent-device daemon stop --state-dir <path>`.
 - `@agent-device/testmu` is now a version 2 provider plugin. `import testmu from '@agent-device/testmu'` gives a factory, `testmu({ credentials: { username, accessKey } })`, that reads those credentials instead of `LT_USERNAME` and `LT_ACCESS_KEY`; without options it reads the variables as before. The package still declares the version 1 plugin entry, so `agent-device plugins add @agent-device/testmu` keeps working on hosts that predate the version 2 contract.
 - Provider plugins have a new contract. A package exports a factory at `./agent-device` that returns `{ apiVersion: 2, provider, credentials, connection?, setup(host) }`, typed by `AgentDevicePluginFactory` from `agent-device/plugins`. The store calls the factory with the package's `config.json` options, and a JavaScript host can call it directly. Version 1 plugins (`agentDevicePlugin.apiVersion: 1`) keep loading.

packages/contracts/src/client-connection.ts

@@ -52,7 +52,8 @@ export type EnsureDaemonOptions = {
    * usable daemon was registered; `incompatible` means a daemon this client cannot use was
    * replaced after it agreed to leave. `previous` is the daemon that was replaced, when there was
    * one: a `missing` start with a `previous` replaced a registered daemon that answered nothing,
-   * either unreachable or hung and force-retired.
+   * either unreachable or hung and force-retired, or one that agreed to leave but outlasted its drain
+   * and was force-retired.
    */
   onStart?: (reason: 'missing' | 'incompatible', previous?: DaemonIdentity) => void;
 };

packages/daemon/src/core/session-store.ts

@@ -3,6 +3,7 @@ import {
   resolveSessionAppLogPath,
   resolveSessionAppLogPidPath,
 } from '@agent-device/daemon-contracts/session-artifact-paths';
+import { DAEMON_SHUTTING_DOWN_REASON } from '@agent-device/daemon-contracts/daemon-resolution';
 import path from 'node:path';
 import fs from 'node:fs';
 import { AppError } from '@agent-device/kernel/errors';
@@ -74,7 +75,7 @@ export class SessionStore {
   assertAdmissionOpen(address: string): void {
     if (!this.acceptingSessions) {
       throw new AppError('COMMAND_FAILED', 'Daemon is shutting down', {
-        reason: 'daemon_shutting_down',
+        reason: DAEMON_SHUTTING_DOWN_REASON,
         session: address,
       });
     }

src/commands/schema/cli-help.ts

@@ -508,7 +508,7 @@ Runner and daemon lifecycle (applies to simulators too):
   No runner read launches a session app that is not running: snapshot, wait, is, get, a reading find, and an interaction's leading reads (a gesture's viewport read, the capture that resolves a selector click/fill) answer the retriable APP_NOT_RUNNING instead of bare-launching over a launch SpringBoard still holds behind its deep-link confirmation. Only open, activate, and a command that mutates without a leading read bring a stopped app up.
   close keeps a healthy iOS simulator XCTest runner warm by default so the next open on that simulator (same udid in the same simulator set) skips the runner build, unless --shutdown was requested, the session was recording, or the session held a device lease. A retained runner auto-stops after an idle window (default 5 minutes); set AGENT_DEVICE_IOS_RUNNER_IDLE_STOP_MS to override, or 0 to disable idle stop and retain until daemon exit. Only a ready runner with an observed listener is retained; close retires unfinished prewarm recovery. Losing the retained listener stops that generation, and daemon shutdown stops idle retained runners rather than handing them off without supervision.
   Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap.
-  A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir <path>. Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly.
+  A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir <path>. Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly.
   On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way.
   A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores).
 

src/daemon-client/__tests__/daemon-client-ensure-drain.test.ts

@@ -179,9 +179,16 @@ test('a daemon still running past the deadline that answers no probe is force-re
   launchesFreshDaemonAt(spawn, paths, fresh.port);
 
   try {
-    const connection = await ensureDaemon({ stateDir, daemonTransport: 'http' });
+    const started: Array<[string, number | undefined]> = [];
+    const connection = await ensureDaemon({
+      stateDir,
+      daemonTransport: 'http',
+      onStart: (reason, previous) => started.push([reason, previous?.pid]),
+    });
 
     assert.equal(connection.started, true);
+    // Force-retired for outlasting its drain, it was never shown idle: reported as a missing daemon.
+    assert.deepEqual(started, [['missing', childPid]]);
     assert.equal(drain.signals.length > 0, true, 'the stuck daemon was signaled');
     assert.ok(
       drain.signals.every(({ atMs }) => atMs >= DRAIN_DEADLINE_MS),
@@ -194,6 +201,7 @@ test('a daemon still running past the deadline that answers no probe is force-re
   }
 });
 
+// A drain that never finishes while the daemon keeps serving: it accepted the request but is stuck.
 test('a daemon still running past the deadline that answers a probe is left alone', async (t) => {
   if (!(await withLoopback(t))) return;
   const { stateDir, paths } = stateDirAndPaths();

src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts

@@ -107,6 +107,7 @@ test('attach reports a missing daemon without starting one', async (t) => {
   const error = await failureOf(sendToDaemon(request(stateDir), { daemonLifecycle: 'attach' }));
 
   assert.equal(error.details?.reason, 'daemon-not-running');
+  assert.match(String(error.details?.hint), /agent-device devices --state-dir /);
   assert.equal(spawn.mock.calls.length, 0);
   assert.equal(fs.existsSync(paths.infoPath), false);
 });
@@ -316,6 +317,32 @@ test.for([
   }
 });
 
+test('ensureDaemon tells a daemon that is still starting up apart from a busy one', async (t) => {
+  if (!(await withLoopback(t))) return;
+  const { stateDir, paths } = stateDirAndPaths();
+  const stale = await startScriptedDaemon(
+    busy({ startingUp: true, inFlightRequests: 0, sessions: 0, repairSessions: 0, leases: 0 }),
+  );
+  const { child } = await registerDaemon(paths, {
+    httpPort: stale.port,
+    version: '0.0.0-stale',
+    capabilities: ['quiesce'],
+  });
+
+  try {
+    const error = await failureOf(ensureDaemon({ stateDir, daemonTransport: 'http' }));
+
+    assert.equal(error.details?.reason, 'daemon-busy');
+    assert.equal(error.details?.startingUp, true);
+    assert.match(String(error.details?.hint), /still starting up\. Retry in a moment/);
+    assert.match(String(error.details?.hint), /agent-device daemon stop --state-dir /);
+    assert.equal(isProcessAlive(child.pid), true);
+    assert.equal(spawn.mock.calls.length, 0);
+  } finally {
+    await closeLoopbackServer(stale.server);
+  }
+});
+
 test('ensureDaemon takes a refusal from its typed reason, not from its wording', async (t) => {
   if (!(await withLoopback(t))) return;
   const { stateDir, paths } = stateDirAndPaths();

src/daemon-client/__tests__/explicit-lifecycle-fixture.ts

@@ -106,7 +106,7 @@ export async function startScriptedSocketDaemon(
 
 export const okResult = (data: Record<string, unknown>) => ({ ok: true, data });
 export const draining = okResult({ outcome: 'draining' });
-export const busy = (census: Record<string, number>) => ({
+export const busy = (census: Record<string, number | boolean>) => ({
   ok: false,
   error: {
     code: 'COMMAND_FAILED',

src/daemon-client/daemon-client-ensure.ts

@@ -4,17 +4,15 @@ import type {
   EnsureDaemonOptions,
 } from '@agent-device/contracts/client';
 import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process';
-import {
-  recoverAbandonedDaemonRegistration,
-  type DaemonRetirementResult,
-} from '@agent-device/daemon-contracts/daemon-registration-owner';
+import { recoverAbandonedDaemonRegistration } from '@agent-device/daemon-contracts/daemon-registration-owner';
 import { sleep } from '@agent-device/host-kit/retry';
 import { AppError } from '@agent-device/kernel/errors';
 import {
   assertDaemonPolicyMatches,
   observeLocalDaemon,
   resolveLocalDaemonSettings,
   retireDaemonForTakeover,
+  stopDaemonForTakeover,
   startLocalDaemon,
   type DaemonClientSettings,
   type EnsuredDaemon,
@@ -27,6 +25,7 @@ import {
   daemonIdentityOf,
   daemonDrainTimeoutError,
   daemonIncompatibleError,
+  daemonRetirementUnconfirmedError,
 } from './daemon-lifecycle-errors.ts';
 
 // A racing `ensureDaemon` in another process can retire or replace the daemon this call judged, or
@@ -128,10 +127,11 @@ async function replaceIncompatible(
   }
   const outcome = await quiesceOrRetireUnresponsive(info, settings);
   if (outcome === 'gone') return { kind: 'judge-again' };
-  if (outcome === 'draining') await awaitDrainedDaemon(info, startTime, settings);
+  const drained =
+    outcome === 'draining' ? await awaitDrainedDaemon(info, startTime, settings) : outcome;
   // A daemon force-retired for answering nothing is reported like an unreachable one: it was never
   // shown to be idle, so "incompatible" would overstate what happened to it.
-  const reason = outcome === 'retired' ? 'missing' : 'incompatible';
+  const reason = drained === 'retired' ? 'missing' : 'incompatible';
   return await start(settings, onStart, reason, daemonIdentityOf(info));
 }
 
@@ -150,7 +150,7 @@ async function awaitDrainedDaemon(
   info: DaemonInfo,
   startTime: string,
   settings: DaemonClientSettings,
-): Promise<void> {
+): Promise<'left' | 'retired'> {
   const { exited } = await waitForDaemonExit(
     { pid: info.pid, startTime },
     { timeoutMs: DRAIN_DEADLINE_MS, pollMs: DRAIN_POLL_MS },
@@ -161,30 +161,21 @@ async function awaitDrainedDaemon(
       paths: settings.paths,
       observed: { pid: info.pid, startTime },
     });
-    return;
+    return 'left';
   }
   if (await answersLivenessProbe(info)) throw await daemonDrainTimeoutError(info, settings);
-  try {
-    await retireDaemonForTakeover(info, settings.paths, 'force');
-  } catch (error) {
-    if (isRefusalToSignalNonDaemon(error)) throw await daemonDrainTimeoutError(info, settings);
-    throw error;
-  }
-}
-
-/**
- * The identity check could not prove the live process is the daemon it observed (an embedded host
- * runs no daemon entry; a start time or command line may also be unreadable), so nothing was signaled.
- */
-function isRefusalToSignalNonDaemon(error: unknown): boolean {
-  if (!(error instanceof AppError) || error.details?.reason !== 'daemon_retirement_unconfirmed') {
-    return false;
+  const retirement = await stopDaemonForTakeover(info, settings.paths, 'force');
+  if (retirement.status !== 'retained') return 'retired';
+  // The identity check could not prove the live process is the daemon it observed (an embedded host
+  // runs no daemon entry; a start time or command line may also be unreadable), so nothing was
+  // signaled.
+  if (
+    retirement.termination?.status === 'retained' &&
+    retirement.termination.reason === 'identity-unverified'
+  ) {
+    throw await daemonDrainTimeoutError(info, settings);
   }
-  const { termination } = error.details.retirement as Extract<
-    DaemonRetirementResult,
-    { status: 'retained' }
-  >;
-  return termination?.status === 'retained' && termination.reason === 'identity-unverified';
+  throw daemonRetirementUnconfirmedError(retirement, settings.paths);
 }
 
 /**

src/daemon-client/daemon-client-ensuring.ts

@@ -1,7 +1,7 @@
-import { AppError } from '@agent-device/kernel/errors';
 import type { DaemonIdentity, EnsureDaemonOptions } from '@agent-device/contracts/client';
 import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution';
 import type { sendToDaemon } from './daemon-client.ts';
+import { isDaemonNotRunningError } from './daemon-lifecycle-errors.ts';
 
 type DaemonTransport = typeof sendToDaemon;
 
@@ -66,8 +66,7 @@ export function withEnsuredDaemon(
     } catch (error) {
       // The daemon this invocation ensured is gone (idle reap, a crash, another client's stop).
       // An attach that finds none sent nothing, so ensuring again and sending once is safe.
-      if (!(error instanceof AppError) || error.details?.reason !== 'daemon-not-running')
-        throw error;
+      if (!isDaemonNotRunningError(error)) throw error;
       // Only the promise this attach relied on is dropped: a concurrent request may already have
       // replaced it with a fresh one.
       if (ensured.get(stateDir) === relied) ensured.delete(stateDir);

src/daemon-client/daemon-client-lifecycle.ts

@@ -1,7 +1,11 @@
 import type { DaemonWireRequest } from '@agent-device/contracts/command';
 import type { DaemonLifecycle, EnsureDaemonOptions } from '@agent-device/contracts/client';
 import type { DaemonResponse } from '@agent-device/kernel/contracts';
-import { daemonContendedError, daemonStopCommand } from './daemon-lifecycle-errors.ts';
+import {
+  daemonContendedError,
+  daemonRetirementUnconfirmedError,
+  daemonStopCommand,
+} from './daemon-lifecycle-errors.ts';
 import { attachActiveSessionAddressHint } from './daemon-client-address-hints.ts';
 import fs from 'node:fs';
 import { processLockHoldsDaemonIdentity } from '@agent-device/daemon-contracts/daemon-registration';
@@ -354,24 +358,25 @@ function daemonRegistrationUnprovenError(
   });
 }
 
-export async function retireDaemonForTakeover(
+export async function stopDaemonForTakeover(
   existing: DaemonInfo,
   paths: DaemonPaths,
   mode: 'graceful' | 'force' = 'graceful',
-): Promise<void> {
-  const retirement = await stopAndRetireDaemon({
+): Promise<DaemonRetirementResult> {
+  return await stopAndRetireDaemon({
     paths: paths,
     observed: { pid: existing.pid, startTime: existing.processStartTime ?? null },
     mode,
   });
-  if (retirement.status === 'retained') {
-    throw new AppError('COMMAND_FAILED', 'Daemon replacement could not be confirmed.', {
-      reason: 'daemon_retirement_unconfirmed',
-      retirement,
-      hint:
-        retirement.error?.hint ?? resolveDaemonStartupHint(getDaemonMetadataState(paths), paths),
-    });
-  }
+}
+
+export async function retireDaemonForTakeover(
+  existing: DaemonInfo,
+  paths: DaemonPaths,
+  mode: 'graceful' | 'force' = 'graceful',
+): Promise<void> {
+  const retirement = await stopDaemonForTakeover(existing, paths, mode);
+  if (retirement.status === 'retained') throw daemonRetirementUnconfirmedError(retirement, paths);
 }
 
 /**