diff --git a/packages/capture-kit/src/png-worker-client-unavailable.test.ts b/packages/capture-kit/src/png-worker-client-unavailable.test.ts index 0ba3a38d08..b2049b8093 100644 --- a/packages/capture-kit/src/png-worker-client-unavailable.test.ts +++ b/packages/capture-kit/src/png-worker-client-unavailable.test.ts @@ -5,18 +5,20 @@ import { PNG } from './png.ts'; test('a worker whose entry is missing is reported once and later registrations hear it', async () => { vi.resetModules(); vi.doMock('./internal-entry.ts', () => ({ resolveInternalEntryModulePath: () => null })); - const { decodePngAsync, prewarmPngWorker } = await import('./png-worker-client.ts'); + const { decodePngAsync, retainPngWorker } = await import('./png-worker-client.ts'); const png = PNG.sync.write(new PNG({ width: 1, height: 1 })); const first = vi.fn(); const second = vi.fn(); - prewarmPngWorker(first); + const releaseFirst = retainPngWorker(first); const decoded = await decodePngAsync(png, 'fallback'); - prewarmPngWorker(second); + const releaseSecond = retainPngWorker(second); expect(decoded.width).toBe(1); expect(first.mock.calls).toEqual([[{ kind: 'entry-not-found' }]]); expect(second.mock.calls).toEqual([[{ kind: 'entry-not-found' }]]); + await releaseFirst(); + await releaseSecond(); vi.doUnmock('./internal-entry.ts'); }); @@ -39,10 +41,10 @@ test('a worker that dies after it started is reported once, and its pending job ...(await importOriginal()), Worker: FakeWorker, })); - const { decodePngAsync, prewarmPngWorker } = await import('./png-worker-client.ts'); + const { decodePngAsync, retainPngWorker } = await import('./png-worker-client.ts'); const png = PNG.sync.write(new PNG({ width: 1, height: 1 })); const unavailable = vi.fn(); - prewarmPngWorker(unavailable); + const release = retainPngWorker(unavailable); expect(unavailable).not.toHaveBeenCalled(); const decoded = decodePngAsync(png, 'under-request'); @@ -53,5 +55,6 @@ test('a worker that dies after it started is reported once, and its pending job expect((await decoded).width).toBe(1); expect(unavailable.mock.calls).toEqual([[{ kind: 'worker-error', detail: 'worker crashed' }]]); + await release(); vi.doUnmock('node:worker_threads'); }); diff --git a/packages/capture-kit/src/png-worker-client.test.ts b/packages/capture-kit/src/png-worker-client.test.ts index 8c9835b713..99c8cac7ff 100644 --- a/packages/capture-kit/src/png-worker-client.test.ts +++ b/packages/capture-kit/src/png-worker-client.test.ts @@ -9,6 +9,7 @@ import { decodePngAsync, decodeScreenshotImageAsync, encodePngAsync, + retainPngWorker, terminatePngWorker, transcodeScreenshotToPngAsync, } from './png-worker-client.ts'; @@ -225,3 +226,19 @@ test('decodeScreenshotImageAsync rejects a container it cannot read with the can }, ); }); + +test('the worker serves every holder and stops only when the last one lets go', async () => { + const first = retainPngWorker(() => {}); + const second = retainPngWorker(() => {}); + const encoded = PNG.sync.write(buildPatternPng(5, 4, 2)); + + await first(); + await first(); + const stillServing = await runOnWorkerThread('decode after one release', () => + decodePngAsync(encoded, 'fixture'), + ); + assert.equal(stillServing.width, 5); + + await second(); + assert.equal(process.getActiveResourcesInfo().includes('MessagePort'), false); +}); diff --git a/packages/capture-kit/src/png-worker-client.ts b/packages/capture-kit/src/png-worker-client.ts index bffd2e7d48..85ccb187c5 100644 --- a/packages/capture-kit/src/png-worker-client.ts +++ b/packages/capture-kit/src/png-worker-client.ts @@ -44,7 +44,7 @@ type PendingJob = { let worker: Worker | null = null; let unavailableReason: PngWorkerUnavailableReason | undefined; -let reportUnavailable: ((reason: PngWorkerUnavailableReason) => void) | undefined; +const unavailableSinks = new Set<(reason: PngWorkerUnavailableReason) => void>(); let nextJobId = 0; const pendingJobs = new Map(); @@ -52,7 +52,7 @@ const pendingJobs = new Map(); function markWorkerUnavailable(reason: PngWorkerUnavailableReason): void { if (unavailableReason) return; unavailableReason = reason; - reportUnavailable?.(reason); + for (const sink of unavailableSinks) sink(reason); } function handleWorkerMessage(message: PngWorkerResponse): void { @@ -185,16 +185,29 @@ async function runPngJob( } } +let pngWorkerHolds = 0; + /** - * Daemon startup hook: spawns the worker before the first screenshot job and registers the daemon's - * sink, which hears every later transition to unavailable once, inside or outside a request. + * Daemon startup hook: spawns the worker before the first screenshot job and holds it for the + * caller. The sink hears every transition to unavailable once, inside or outside a request, and + * hears an earlier one at once. The worker is shared by every runtime in the process, so it stops + * when the last holder lets go; the returned release is idempotent. */ -export function prewarmPngWorker( +export function retainPngWorker( onUnavailable: (reason: PngWorkerUnavailableReason) => void, -): void { - reportUnavailable = onUnavailable; +): () => Promise { + unavailableSinks.add(onUnavailable); if (unavailableReason) onUnavailable(unavailableReason); else obtainWorker(); + pngWorkerHolds += 1; + let released = false; + return async () => { + if (released) return; + released = true; + unavailableSinks.delete(onUnavailable); + pngWorkerHolds -= 1; + if (pngWorkerHolds === 0) await terminatePngWorker(); + }; } /** Stops the worker thread (used by tests and shutdown); later calls respawn it. */ diff --git a/packages/contracts/src/application-lifecycle-runtime.ts b/packages/contracts/src/application-lifecycle-runtime.ts index 8db2fb2807..cf62df9d4d 100644 --- a/packages/contracts/src/application-lifecycle-runtime.ts +++ b/packages/contracts/src/application-lifecycle-runtime.ts @@ -382,8 +382,10 @@ export type AppleApplicationTools = Readonly<{ applyRuntimeHints(device: DeviceInfo, input: RuntimeHintsApplicationInput): Promise; clearRuntimeHints(device: DeviceInfo, input: RuntimeHintsApplicationInput): Promise; dismissCloseAlerts(device: DeviceInfo, input: CloseApplicationFinalizationInput): Promise; - detachRunnerSessionsForShutdown(): Promise; - finalizeRunnerSessionsForShutdown(): Promise; + /** The devices whose runner was handed to the next daemon instead of stopped. */ + detachRunnerSessionsForShutdown(): Promise; + /** The devices whose runner this daemon stopped. */ + finalizeRunnerSessionsForShutdown(): Promise; }>; /** Lazy Android mechanics used by the Android package's own lifecycle binding. */ @@ -413,6 +415,6 @@ export type AndroidApplicationTools = Readonly<{ /** Gateway-coordinated durable phases whose concrete mechanics stay package-owned. */ export type ApplicationLifecycleResourceLifecycle = Readonly<{ recoverStartupResources(input: Readonly<{ stateDir: string }>): Promise; - detachForDaemonShutdown(): Promise; - finalizeDaemonShutdown(): Promise; + detachForDaemonShutdown(): Promise; + finalizeDaemonShutdown(): Promise; }>; diff --git a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts index 85d3005d17..b404c18561 100644 --- a/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts +++ b/packages/daemon/src/__tests__/support/daemon-runtime-deps-fixture.ts @@ -1,6 +1,7 @@ import type { ApplicationLifecycleResourceLifecycle } from '@agent-device/contracts/application-lifecycle-runtime'; import { createTestDeviceInventoryGateways } from '@agent-device/device-selection/inventory-gateways-fixtures'; import { createOwnedProcessRecordStore } from '@agent-device/host-kit/process'; +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; import { vi } from 'vitest'; import type { ComposeDaemonRuntimeDeps, @@ -25,8 +26,8 @@ export function applicationLifecycleResourceFixture( ): ApplicationLifecycleResourceLifecycle { return Object.freeze({ recoverStartupResources: vi.fn(async () => {}), - detachForDaemonShutdown: vi.fn(async () => {}), - finalizeDaemonShutdown: vi.fn(async () => {}), + detachForDaemonShutdown: vi.fn(async (): Promise => []), + finalizeDaemonShutdown: vi.fn(async (): Promise => []), ...overrides, }); } @@ -59,6 +60,7 @@ export function daemonRuntimeDepsFixture( return async (input) => { const resolved = typeof overrides === 'function' ? overrides(input) : overrides; return Object.freeze({ + runtimeStates: [], ownedProcessRecords: createOwnedProcessRecordStore({ stateDir: input.baseDir, sessionsDir: input.sessionsDir, @@ -94,3 +96,48 @@ export function daemonRuntimeDepsFixture( }); }; } + +/** + * A plugin registry over provider runtimes, as the composition hands it to the daemon: the + * inventory of active plugins and the one `close` that shuts each runtime down and answers with a + * per-plugin outcome. + */ +export function pluginRegistryFixture( + runtimes: readonly Pick[], +): Pick { + return { + pluginInventory: runtimes.map((runtime) => ({ + provider: runtime.provider, + source: 'builtin' as const, + state: 'active' as const, + credentials: { variables: [], byBackend: {} }, + })), + shutdownProviderRuntimes: async () => + await Promise.all( + runtimes.map(async (runtime) => { + try { + await runtime.shutdown(); + return { + provider: runtime.provider, + source: 'builtin' as const, + shutdown: { status: 'ok' as const }, + dispose: { status: 'ok' as const }, + }; + } catch (error) { + return { + provider: runtime.provider, + source: 'builtin' as const, + shutdown: { + status: 'failed' as const, + error: { + code: 'COMMAND_FAILED', + message: error instanceof Error ? error.message : String(error), + }, + }, + dispose: { status: 'ok' as const }, + }; + } + }), + ), + }; +} diff --git a/packages/daemon/src/core/interactor-resolution.ts b/packages/daemon/src/core/interactor-resolution.ts index 2584685635..217bc6e49f 100644 --- a/packages/daemon/src/core/interactor-resolution.ts +++ b/packages/daemon/src/core/interactor-resolution.ts @@ -1,3 +1,4 @@ +import { defineRuntimeSlots, type RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import type { Interactor, RunnerContext } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -27,7 +28,18 @@ const NO_INTERACTOR_RESOLUTION: InteractorResolution = { }, }; -let installedInteractorResolution: InteractorResolution = NO_INTERACTOR_RESOLUTION; +type InteractorResolutionState = { resolution: InteractorResolution }; + +const interactorResolutionRuntime = defineRuntimeSlots(); +const INTERACTOR_RESOLUTION_SLOT = Symbol('interactorResolution'); +const interactorResolutionState = (): InteractorResolutionState => + interactorResolutionRuntime.slot(INTERACTOR_RESOLUTION_SLOT, (): InteractorResolutionState => ({ + resolution: NO_INTERACTOR_RESOLUTION, + })); + +/** The resolution one daemon runtime owns; the runtime installs its composition's into it. */ +export const createInteractorResolutionRuntimeState = (): RuntimeStateHandle => + interactorResolutionRuntime.createHandle(); /** * Root composition's installation point, beside the provider-device admission that shares the @@ -35,9 +47,9 @@ let installedInteractorResolution: InteractorResolution = NO_INTERACTOR_RESOLUTI * the module root composition imports, and the daemon never loads it itself. */ export function installInteractorResolution(resolution: InteractorResolution): void { - installedInteractorResolution = resolution; + interactorResolutionState().resolution = resolution; } export function interactorResolution(): InteractorResolution { - return installedInteractorResolution; + return interactorResolutionState().resolution; } diff --git a/packages/daemon/src/core/provider-device-admission.ts b/packages/daemon/src/core/provider-device-admission.ts index bae7032244..2a4bebef72 100644 --- a/packages/daemon/src/core/provider-device-admission.ts +++ b/packages/daemon/src/core/provider-device-admission.ts @@ -1,3 +1,4 @@ +import { defineRuntimeSlots, type RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import type { DeviceInfo } from '@agent-device/kernel/device'; /** @@ -18,7 +19,18 @@ const NO_PROVIDER_DEVICE_ADMISSION: ProviderDeviceAdmission = { isActive: () => false, }; -let installedProviderDeviceAdmission: ProviderDeviceAdmission = NO_PROVIDER_DEVICE_ADMISSION; +type ProviderAdmissionState = { admission: ProviderDeviceAdmission }; + +const providerAdmissionRuntime = defineRuntimeSlots(); +const PROVIDER_ADMISSION_SLOT = Symbol('providerDeviceAdmission'); +const providerAdmissionState = (): ProviderAdmissionState => + providerAdmissionRuntime.slot(PROVIDER_ADMISSION_SLOT, (): ProviderAdmissionState => ({ + admission: NO_PROVIDER_DEVICE_ADMISSION, + })); + +/** The admission one daemon runtime owns; the runtime installs its composition's into it. */ +export const createProviderDeviceAdmissionRuntimeState = (): RuntimeStateHandle => + providerAdmissionRuntime.createHandle(); /** * Root composition's installation point. The provider runtime scope is ambient per request, so @@ -26,9 +38,9 @@ let installedProviderDeviceAdmission: ProviderDeviceAdmission = NO_PROVIDER_DEVI * provider runtime internals itself. */ export function installProviderDeviceAdmission(admission: ProviderDeviceAdmission): void { - installedProviderDeviceAdmission = admission; + providerAdmissionState().admission = admission; } export function isActiveProviderDevice(device: DeviceInfo): boolean { - return installedProviderDeviceAdmission.isActive(device); + return providerAdmissionState().admission.isActive(device); } diff --git a/packages/daemon/src/index.ts b/packages/daemon/src/index.ts index b7ef46715a..a31a68d697 100644 --- a/packages/daemon/src/index.ts +++ b/packages/daemon/src/index.ts @@ -1,9 +1,9 @@ -export { startDaemonRuntime } from './server/http/daemon-runtime.ts'; +export { daemonStartupExitCode, startDaemonRuntime } from './server/http/daemon-runtime.ts'; +export { reportDaemonClose } from './server/http/server-shutdown.ts'; export type { ComposeDaemonRuntimeDeps, DaemonRuntimeController, DaemonRuntimeDeps, - DaemonRuntimeDepsInput, } from './server/http/daemon-runtime.ts'; export type { DaemonInvokeFn, DaemonRequest } from './core/daemon-request.ts'; export type { DaemonPlatformServices } from './core/platform-services.ts'; diff --git a/packages/daemon/src/resources/artifact-tracking.test.ts b/packages/daemon/src/resources/artifact-tracking.test.ts new file mode 100644 index 0000000000..c2e4538847 --- /dev/null +++ b/packages/daemon/src/resources/artifact-tracking.test.ts @@ -0,0 +1,32 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { + createArtifactTrackingRuntimeState, + trackDownloadableArtifact, + trackUploadedArtifact, +} from './artifact-tracking.ts'; + +test('disposing a runtime removes the temporary files it still tracks', () => { + const root = mkdtempForTestSync('agent-device-artifact-dispose-'); + const artifactPath = path.join(root, 'artifact.txt'); + const uploadDir = path.join(root, 'upload'); + fs.writeFileSync(artifactPath, 'artifact'); + fs.mkdirSync(uploadDir); + fs.writeFileSync(path.join(uploadDir, 'payload'), 'payload'); + const runtime = createArtifactTrackingRuntimeState(); + + runtime.run(() => { + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'artifact.txt', + }); + trackUploadedArtifact({ artifactPath: path.join(uploadDir, 'payload'), tempDir: uploadDir }); + }); + runtime.dispose(); + + expect(fs.existsSync(uploadDir)).toBe(false); + expect(fs.existsSync(artifactPath)).toBe(false); +}); diff --git a/packages/daemon/src/resources/artifact-tracking.ts b/packages/daemon/src/resources/artifact-tracking.ts index efccad7590..f3fa14bb69 100644 --- a/packages/daemon/src/resources/artifact-tracking.ts +++ b/packages/daemon/src/resources/artifact-tracking.ts @@ -1,3 +1,4 @@ +import { defineRuntimeSlots, type RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; @@ -44,7 +45,35 @@ type ArtifactEntry = { timer: ReturnType; }; -const pendingArtifacts = new Map(); +type ArtifactTrackingState = { + artifacts: Map; + uploads: Map; +}; + +const artifactTrackingRuntime = defineRuntimeSlots(); +const ARTIFACT_TRACKING_SLOT = Symbol('artifactTracking'); +const artifactTracking = (): ArtifactTrackingState => + artifactTrackingRuntime.slot( + ARTIFACT_TRACKING_SLOT, + (): ArtifactTrackingState => ({ artifacts: new Map(), uploads: new Map() }), + (state) => { + for (const entry of state.artifacts.values()) { + clearTimeout(entry.timer); + cleanupDirectoryArchive(entry); + if (entry.deleteAfterDownload) removeBestEffort(entry.artifactPath); + } + for (const entry of state.uploads.values()) { + clearTimeout(entry.timer); + removeBestEffort(entry.tempDir); + } + }, + ); +const pendingArtifacts = () => artifactTracking().artifacts; +const pendingUploads = () => artifactTracking().uploads; + +/** The artifacts and uploads one daemon runtime tracks; disposing it stops their expiry timers. */ +export const createArtifactTrackingRuntimeState = (): RuntimeStateHandle => + artifactTrackingRuntime.createHandle(); /** What a finished request registers for each artifact it hands back to the client. */ export type DownloadableArtifactRegistration = { @@ -73,7 +102,7 @@ export function trackDownloadableArtifact( cleanupDownloadableArtifact(artifactId); }, ARTIFACT_CLEANUP_TIMEOUT_MS); timer.unref(); - pendingArtifacts.set(artifactId, { + pendingArtifacts().set(artifactId, { artifactPath: params.artifactPath, tenantId: params.tenantId, artifactType: params.artifactType, @@ -105,7 +134,7 @@ export async function listDownloadableArtifacts( tenantId?: string, ): Promise { const artifacts: DaemonArtifactInventoryEntry[] = []; - for (const [id, entry] of pendingArtifacts) { + for (const [id, entry] of pendingArtifacts()) { if (!canReadArtifact(entry, tenantId)) continue; let stat: fs.Stats; try { @@ -148,7 +177,7 @@ function readDownloadableArtifactEntry( tenantId: string | undefined, ): { entry: ArtifactEntry; stat: fs.Stats } { const entry = requireTenantOwnedEntry( - pendingArtifacts, + pendingArtifacts(), artifactId, tenantId, DOWNLOADABLE_ARTIFACT_RESOURCE, @@ -249,7 +278,7 @@ async function createDirectoryArchive( sizeBytes: fs.statSync(archivePath).size, }; // The artifact may be consumed or expire while tar is still archiving. - if (pendingArtifacts.get(artifactId) !== entry) { + if (pendingArtifacts().get(artifactId) !== entry) { fs.rmSync(tempDir, { recursive: true, force: true }); throw expiredTenantOwnedEntryError(DOWNLOADABLE_ARTIFACT_RESOURCE, artifactId); } @@ -294,14 +323,17 @@ function hasUnsafeFilenameCharacter(fileName: string): boolean { } export function cleanupDownloadableArtifact(artifactId: string): void { - const entry = pendingArtifacts.get(artifactId); + const entry = pendingArtifacts().get(artifactId); if (!entry) return; clearTimeout(entry.timer); - pendingArtifacts.delete(artifactId); + pendingArtifacts().delete(artifactId); cleanupDirectoryArchive(entry); - if (!entry.deleteAfterDownload) return; + if (entry.deleteAfterDownload) removeBestEffort(entry.artifactPath); +} + +function removeBestEffort(targetPath: string): void { try { - fs.rmSync(entry.artifactPath, { recursive: true, force: true }); + fs.rmSync(targetPath, { recursive: true, force: true }); } catch { // best-effort cleanup only } @@ -309,13 +341,8 @@ export function cleanupDownloadableArtifact(artifactId: string): void { function cleanupDirectoryArchive(entry: ArtifactEntry): void { if (!entry.directoryArchive) return; - try { - fs.rmSync(entry.directoryArchive.tempDir, { recursive: true, force: true }); - } catch { - // best-effort cleanup only - } finally { - entry.directoryArchive = undefined; - } + removeBestEffort(entry.directoryArchive.tempDir); + entry.directoryArchive = undefined; } // --- Upload artifact tracking --- @@ -334,8 +361,6 @@ type UploadEntry = { timer: ReturnType; }; -const pendingUploads = new Map(); - export function trackUploadedArtifact(params: { artifactPath: string; tempDir: string; @@ -346,7 +371,7 @@ export function trackUploadedArtifact(params: { cleanupUploadedArtifact(uploadId); }, UPLOAD_CLEANUP_TIMEOUT_MS); timer.unref(); - pendingUploads.set(uploadId, { + pendingUploads().set(uploadId, { artifactPath: params.artifactPath, tempDir: params.tempDir, tenantId: params.tenantId, @@ -357,7 +382,7 @@ export function trackUploadedArtifact(params: { export function prepareUploadedArtifact(uploadId: string, tenantId?: string): string { const entry = requireTenantOwnedEntry( - pendingUploads, + pendingUploads(), uploadId, tenantId, UPLOADED_ARTIFACT_RESOURCE, @@ -367,9 +392,9 @@ export function prepareUploadedArtifact(uploadId: string, tenantId?: string): st } export function cleanupUploadedArtifact(uploadId: string): void { - const entry = pendingUploads.get(uploadId); + const entry = pendingUploads().get(uploadId); if (!entry) return; clearTimeout(entry.timer); - pendingUploads.delete(uploadId); + pendingUploads().delete(uploadId); fs.rmSync(entry.tempDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts index 532ae85cde..a102b43a0f 100644 --- a/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts +++ b/packages/daemon/src/server/__tests__/daemon-runtime-app-log.test.ts @@ -1,4 +1,5 @@ import { appLogResourceStore } from '@agent-device/capture-kit/app-log-durable-resource'; +import { teardownDaemonSessionForShutdown } from '../http/server-shutdown.ts'; import { recoverAppLogResourcesAfterDaemonLock } from '@agent-device/capture-kit/app-log-resource-recovery'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import fs from 'node:fs'; @@ -9,10 +10,7 @@ import type { ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import { createDurableResourceEnvelope } from '@agent-device/capture-kit'; import { createTestAppLogLiveHandle, mkdtempForTestSync } from '@agent-device/test-kit'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; -import { - flushDaemonStartupDiagnostics, - teardownDaemonSessionForShutdown, -} from '../http/daemon-runtime.ts'; +import { flushDaemonStartupDiagnostics } from '../http/daemon-runtime.ts'; import type { SessionState } from '../../core/session-state.ts'; import { unavailableDeviceRuntimeGateway } from '../../__tests__/support/unavailable-device-runtime-gateway.ts'; @@ -36,13 +34,12 @@ test('daemon startup configures the Apple runner owner after acquiring the lock, const source = fs.readFileSync(new URL('../http/daemon-runtime.ts', import.meta.url), 'utf8'); const acquiredLock = source.indexOf("if (acquisition.status !== 'acquired')"); const runnerOwnerConfigured = source.indexOf('await deps.ownerLifecycle.configureForDaemonLock('); - const lockFailureExit = source.indexOf('exit(', acquiredLock); + const lockFailureExit = source.indexOf("reason: 'daemon_registration_refused'", acquiredLock); expect(acquiredLock).toBeGreaterThanOrEqual(0); expect(lockFailureExit).toBeGreaterThan(acquiredLock); expect(runnerOwnerConfigured).toBeGreaterThan(acquiredLock); - // The configure call sits inside the post-lock try block, after the failure branch that exits - // for a lock held by another process. + // The configure call comes after the failure branch that refuses a lock held by another process. expect(runnerOwnerConfigured).toBeGreaterThan(lockFailureExit); }); diff --git a/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts b/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts index 5be1e1985a..0d56c8ec56 100644 --- a/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts +++ b/packages/daemon/src/server/http/daemon-lease-finalizer.test.ts @@ -42,6 +42,7 @@ test('journals and bounds a hung recoverable lease release before the final drai await vi.advanceTimersByTimeAsync(10); await expect(drain).resolves.toEqual({ pending: [lease], + unknown: [lease], released: [], }); } finally { @@ -83,7 +84,7 @@ test('final drain joins a release that completes after the lease timeout', async const drain = expiredProviderLeaseReleaser.drain(2_000); await vi.advanceTimersByTimeAsync(500); - await expect(drain).resolves.toEqual({ pending: [], released: [lease] }); + await expect(drain).resolves.toEqual({ pending: [], unknown: [], released: [lease] }); } finally { expiredProviderLeaseReleaser.shutdown(); vi.useRealTimers(); @@ -129,7 +130,11 @@ test('a hung provider release does not starve another lease during shutdown', as expect(leaseRegistry.listActiveLeases()).toEqual([]); const drain = expiredProviderLeaseReleaser.drain(2_000); await vi.advanceTimersByTimeAsync(2_000); - await expect(drain).resolves.toEqual({ pending: [hungLease], released: [releasedLease] }); + await expect(drain).resolves.toEqual({ + pending: [hungLease], + unknown: [hungLease], + released: [releasedLease], + }); } finally { expiredProviderLeaseReleaser.shutdown(); vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts index 8e31d53d8d..16560207b9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-artifacts.test.ts @@ -13,11 +13,6 @@ async function downloadTrackedArtifact(env: Record, awaitConsump const stateDir = mkdtempForTestSync('agent-device-daemon-runtime-artifacts-'); const artifactPath = path.join(stateDir, 'runtime-artifact.txt'); fs.writeFileSync(artifactPath, 'runtime-artifact'); - const artifactId = trackDownloadableArtifact({ - artifactPath, - artifactType: 'runtime-artifact', - fileName: 'runtime-artifact.txt', - }); const runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { @@ -28,14 +23,18 @@ async function downloadTrackedArtifact(env: Record, awaitConsump ...env, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture(), }); + const artifactId = runtime.run(() => + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'runtime-artifact.txt', + }), + ); try { - expect(runtime).not.toBeNull(); const response = await fetch( `http://127.0.0.1:${runtime?.httpPort}/artifacts/${encodeURIComponent(artifactId)}`, { headers: { authorization: `Bearer ${runtime?.token}`, connection: 'close' } }, @@ -48,8 +47,8 @@ async function downloadTrackedArtifact(env: Record, awaitConsump } return { status: response.status, body, onDisk: fs.existsSync(artifactPath) }; } finally { - await runtime?.shutdown(); - cleanupDownloadableArtifact(artifactId); + runtime.run(() => cleanupDownloadableArtifact(artifactId)); + await runtime.close(); } } diff --git a/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts new file mode 100644 index 0000000000..133f905e9f --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-bounded-close.test.ts @@ -0,0 +1,165 @@ +import fs from 'node:fs'; +import { expect, test, vi } from 'vitest'; +import { createTestDeviceInventoryGateways } from '@agent-device/device-selection/inventory-gateways-fixtures'; +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { clearAndroidObservationFixture } from '../../__tests__/support/android-observation-fixture.ts'; +import { fakeCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; +import { + daemonRuntimeDepsFixture, + pluginRegistryFixture, +} from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { daemonPlatformServicesFixture } from '../../__tests__/support/platform-services-fixture.ts'; +import { providerCredentialsFixture } from '../../__tests__/support/provider-credentials-fixture.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; +import { + describeUnfinishedShutdownWork, + reportDaemonClose, + type ShutdownReport, +} from './server-shutdown.ts'; + +const silent = { stderr: { write: () => {} }, stdout: { write: () => {} } }; + +function hostFor(stateDir: string) { + return createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }); +} + +test('a request that ignores cancellation is abandoned at the drain budget and keeps the registration', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-bounded-close-'); + const paths = resolveDaemonPaths(stateDir); + const reached = vi.fn(); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + shutdownBudgets: { requestDrainMs: 50 }, + composeDeps: daemonRuntimeDepsFixture((input) => ({ + requestRouter: Object.freeze({ + deviceInventoryGateways: createTestDeviceInventoryGateways({ + local: async () => { + reached(); + return await new Promise(() => {}); + }, + }), + platformServices: daemonPlatformServicesFixture(), + androidObservation: clearAndroidObservationFixture, + providerCredentials: providerCredentialsFixture(input.host.env, input.baseDir), + createCommandSurface: fakeCreateCommandSurface, + }), + })), + }); + const hung = fetch(`http://127.0.0.1:${runtime.httpPort}/rpc`, { + method: 'POST', + headers: { authorization: `Bearer ${runtime.token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 'hung-1', + method: 'agent_device.command', + params: { command: 'devices', meta: { requestId: 'hung-1' } }, + }), + }).catch(() => undefined); + await vi.waitFor(() => expect(reached).toHaveBeenCalled(), { timeout: 10_000 }); + + const report = await runtime.close(); + + expect(report.requests).toEqual({ drained: 0, abandoned: 1 }); + expect(fs.existsSync(paths.infoPath)).toBe(true); + // A close nobody scoped for diagnostics (a signal, the idle reap) still leaves its record. + expect(fs.readFileSync(paths.logPath, 'utf8')).toContain('daemon_shutdown_requests_abandoned'); + await hung; +}); + +test('a plugin that never finishes shutting down is reported pending at the budget', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-bounded-plugin-'); + const stuck = { + provider: 'stuck', + shutdown: () => new Promise(() => {}), + leaseLifecycle: {}, + }; + const done = { provider: 'done', shutdown: async () => {}, leaseLifecycle: {} }; + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + shutdownBudgets: { pluginMs: 50 }, + composeDeps: daemonRuntimeDepsFixture({ + ...pluginRegistryFixture([stuck, done] as unknown as ProviderDeviceRuntime[]), + }), + }); + + const report = await runtime.close(); + + // The registry reports once, after its slowest plugin, so every active plugin is pending. + expect(report.plugins).toEqual({ disposed: [], failed: [], pending: ['stuck', 'done'] }); + expect(report.requests).toEqual({ drained: 0, abandoned: 0 }); +}); + +const cleanReport: ShutdownReport = { + providerReleases: { completed: [], pending: [], unknown: [] }, + requests: { drained: 0, abandoned: 0 }, + runners: { stopped: [], handedOff: [] }, + sessions: { closed: 0, failed: 0 }, + plugins: { disposed: [], failed: [], pending: [] }, +}; + +test('a clean close leaves nothing to report and an unfinished one names each piece of work', () => { + expect(describeUnfinishedShutdownWork(cleanReport)).toEqual([]); + + const lines = describeUnfinishedShutdownWork({ + ...cleanReport, + providerReleases: { completed: [], pending: ['lease-a'], unknown: ['lease-b'] }, + requests: { drained: 1, abandoned: 2 }, + sessions: { closed: 0, failed: 1 }, + plugins: { + disposed: [], + pending: ['stuck'], + failed: [ + { provider: 'broken', step: 'dispose', error: { code: 'UNKNOWN', message: 'refused' } }, + ], + }, + }); + + expect(lines).toEqual([ + expect.stringContaining('2 request(s) ignored cancellation'), + expect.stringContaining('plugin stuck was still shutting down'), + expect.stringContaining('plugin broken failed to dispose: refused'), + expect.stringContaining('lease release still pending for lease-a'), + expect.stringContaining('outcome unknown for lease-b'), + expect.stringContaining('1 session(s) failed to tear down'), + ]); +}); + +test('the executable writes unfinished work to stderr after a close, and a clean close nothing', async () => { + const clean: string[] = []; + const unfinished: string[] = []; + const failed: string[] = []; + + expect( + await reportDaemonClose({ + closed: Promise.resolve(cleanReport), + stderr: { write: (chunk) => clean.push(chunk) }, + }), + ).toEqual({ failed: false }); + expect( + await reportDaemonClose({ + closed: Promise.resolve({ ...cleanReport, requests: { drained: 0, abandoned: 1 } }), + stderr: { write: (chunk) => unfinished.push(chunk) }, + }), + ).toEqual({ failed: false }); + expect( + await reportDaemonClose({ + closed: Promise.reject(new Error('close blew up')), + stderr: { write: (chunk) => failed.push(chunk) }, + }), + ).toEqual({ failed: true }); + + expect(clean).toEqual([]); + expect(unfinished).toEqual([expect.stringMatching(/1 request\(s\) ignored cancellation/)]); + expect(failed.join('')).toContain('Daemon shutdown error: close blew up'); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-close.test.ts b/packages/daemon/src/server/http/daemon-runtime-close.test.ts new file mode 100644 index 0000000000..ce3cac65ed --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-close.test.ts @@ -0,0 +1,141 @@ +import fs from 'node:fs'; +import { expect, test, vi } from 'vitest'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + applicationLifecycleResourceFixture, + daemonRuntimeDepsFixture, + pluginRegistryFixture, + platformOwnerLifecycleFixture, +} from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +function hostFor(stateDir: string) { + return createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }); +} + +function providerRuntime(provider: string, shutdown: () => Promise): ProviderDeviceRuntime { + return { provider, shutdown, leaseLifecycle: {} } as unknown as ProviderDeviceRuntime; +} + +const silent = { stderr: { write: () => {} }, stdout: { write: () => {} } }; + +test('close is idempotent and every caller receives the one report', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-close-'); + const stop = vi.fn(async () => {}); + const failing = vi.fn(async () => { + throw new Error('provider refused to stop'); + }); + const signalListeners = process.listenerCount('SIGTERM'); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + ...pluginRegistryFixture([providerRuntime('stops', stop), providerRuntime('fails', failing)]), + deviceRuntimeGateway: Object.freeze({ + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture({ + detachForDaemonShutdown: async () => ['handed-off-device'], + finalizeDaemonShutdown: async () => ['stopped-device'], + }), + }), + }), + }); + try { + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); + + const [first, second] = await Promise.all([runtime.close(), runtime.close()]); + const third = await runtime.close(); + + expect(second).toBe(first); + expect(third).toBe(first); + await expect(runtime.closed).resolves.toBe(first); + expect(first.runners).toEqual({ + stopped: ['stopped-device'], + handedOff: ['handed-off-device'], + }); + expect(first.sessions).toEqual({ closed: 0, failed: 0 }); + expect(first.providerReleases).toEqual({ completed: [], pending: [], unknown: [] }); + expect(first.plugins.disposed).toEqual(['stops']); + expect(first.plugins.failed).toEqual([ + { + provider: 'fails', + source: 'builtin', + step: 'shutdown', + error: expect.objectContaining({ message: 'provider refused to stop' }), + }, + ]); + expect(stop).toHaveBeenCalledOnce(); + expect(failing).toHaveBeenCalledOnce(); + } finally { + await runtime.close(); + } +}); + +test('a startup that fails after acquiring resources rolls them back once', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-rollback-'); + const stop = vi.fn(async () => {}); + const clearLock = vi.fn(async () => {}); + const failure = new Error('startup step failed'); + const signalListeners = process.listenerCount('SIGTERM'); + + await expect( + startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + ...pluginRegistryFixture([providerRuntime('stops', stop)]), + ownerLifecycle: platformOwnerLifecycleFixture({ + configureForDaemonLock: async () => { + throw failure; + }, + clearDaemonLockConfiguration: clearLock, + }), + }), + }), + ).rejects.toBe(failure); + + expect(stop).toHaveBeenCalledOnce(); + expect(clearLock).toHaveBeenCalledOnce(); + expect(process.listenerCount('SIGTERM')).toBe(signalListeners); +}); + +test('a shutdown step that rejects still releases the registration, lock and providers', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-close-failure-'); + const stop = vi.fn(async () => {}); + const clearLock = vi.fn(async () => {}); + const runtime = await startDaemonRuntime({ + host: hostFor(stateDir), + ...silent, + composeDeps: daemonRuntimeDepsFixture({ + ...pluginRegistryFixture([providerRuntime('stops', stop)]), + ownerLifecycle: platformOwnerLifecycleFixture({ clearDaemonLockConfiguration: clearLock }), + deviceRuntimeGateway: Object.freeze({ + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture({ + finalizeDaemonShutdown: async () => { + throw new Error('runner finalization failed'); + }, + }), + }), + }), + }); + const paths = resolveDaemonPaths(stateDir); + + await expect(runtime.close()).rejects.toThrow('runner finalization failed'); + await expect(runtime.closed).rejects.toThrow('runner finalization failed'); + + expect(stop).toHaveBeenCalledOnce(); + expect(clearLock).toHaveBeenCalledOnce(); + expect(fs.existsSync(paths.lockPath)).toBe(false); + expect(fs.existsSync(paths.infoPath)).toBe(false); +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts index 71c8a61268..789f83941f 100644 --- a/packages/daemon/src/server/http/daemon-runtime-composition.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-composition.test.ts @@ -13,12 +13,13 @@ import { WEB_DESKTOP_DEVICE, createTestHostEnvironment, } from '@agent-device/test-kit'; -import { - installInteractorResolution, - interactorResolution, -} from '../../core/interactor-resolution.ts'; +import { interactorResolution } from '../../core/interactor-resolution.ts'; import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; -import { startDaemonRuntime, type DaemonRuntimeDepsInput } from './daemon-runtime.ts'; +import { + daemonStartupExitCode, + startDaemonRuntime, + type DaemonRuntimeDepsInput, +} from './daemon-runtime.ts'; // The server builds its own store, so capture that instance: the proof is about the store the // server hands to the composition callbacks, not one the test fabricates. @@ -41,8 +42,7 @@ vi.mock('../../core/session-store.ts', async (importOriginal) => { test('daemon startup installs the interactor resolution the composition supplied', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-interactor-composition-'); const composed = Object.freeze({ resolve: vi.fn(async () => ({}) as never) }); - const previousResolution = interactorResolution(); - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ @@ -53,20 +53,19 @@ test('daemon startup installs the interactor resolution the composition supplied AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ interactorResolution: composed }), }); expect(runtime).not.toBeNull(); - expect(interactorResolution()).toBe(composed); - await interactorResolution().resolve(IOS_SIMULATOR, {}); + await runtime.run(async () => { + expect(interactorResolution()).toBe(composed); + await interactorResolution().resolve(IOS_SIMULATOR, {}); + }); expect(composed.resolve).toHaveBeenCalledOnce(); } finally { - await runtime?.shutdown(); - installInteractorResolution(previousResolution); + await runtime?.close(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); @@ -78,20 +77,16 @@ test('a daemon attempt losing the lock shuts the providers down through the comp const shutdownProviderRuntimes = vi.fn(async () => { throw new Error('cleanup failed'); }); - const exit = vi.fn(); - const runtime = await startDaemonRuntime({ + const refusal = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { AGENT_DEVICE_STATE_DIR: stateDir } }), - exit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ shutdownProviderRuntimes, }), - }); - expect(runtime).toBeNull(); + }).catch((error: unknown) => error); + expect(daemonStartupExitCode(refusal)).toBe(DAEMON_STARTUP_EXIT_CODES.busy); expect(shutdownProviderRuntimes).toHaveBeenCalledOnce(); - expect(exit).toHaveBeenCalledWith(DAEMON_STARTUP_EXIT_CODES.busy); await held.owner.finish(); }); @@ -109,8 +104,6 @@ test('the server runs on the host it was given and hands that host to the compos const fixtureDeps = daemonRuntimeDepsFixture(); const runtime = await startDaemonRuntime({ host, - registerProcessHandlers: false, - exit: () => {}, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: async (input) => { @@ -122,7 +115,7 @@ test('the server runs on the host it was given and hands that host to the compos assert.equal(composed?.host, host); assert.equal(composed?.baseDir, resolveDaemonPaths(stateDir).baseDir); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -140,10 +133,9 @@ test('the server derives the policy gate and session callbacks and hands them to JSON.stringify({ version: 1, capabilities: { deny: ['device-shutdown'] } }), ); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; let composed: DaemonRuntimeDepsInput | undefined; const fixtureDeps = daemonRuntimeDepsFixture(); - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ @@ -154,8 +146,6 @@ test('the server derives the policy gate and session callbacks and hands them to AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: async (input) => { @@ -163,8 +153,6 @@ test('the server derives the policy gate and session callbacks and hands them to return await fixtureDeps(input); }, }); - assert.notEqual(runtime, null, 'the assembly must boot'); - assert.deepEqual(exits, []); assert.ok(composed, 'the server must call the factory'); // The server resolved the paths and loaded the operator's policy before composing anything. assert.equal(composed.baseDir, paths.baseDir); @@ -196,11 +184,10 @@ test('the server derives the policy gate and session callbacks and hands them to outputPath: store.resolveAppLogPath('web-assembly'), pidPath: store.resolveAppLogPidPath('web-assembly'), }); - await runtime?.shutdown(); + await runtime?.close(); runtime = null; - assert.deepEqual(exits, [0]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -210,12 +197,9 @@ test('the server derives the policy gate and session callbacks and hands them to */ test('an unreadable policy stops the assembly before the factory is called', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-assembly-bad-policy-'); - const exits: number[] = []; - const stderr: string[] = []; let factoryCalls = 0; - let runtime: Awaited> = null; try { - runtime = await startDaemonRuntime({ + const refusal = await startDaemonRuntime({ host: createTestHostEnvironment({ env: { ...process.env, @@ -224,20 +208,17 @@ test('an unreadable policy stops the assembly before the factory is called', asy AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - stderr: { write: (chunk) => stderr.push(chunk) }, + stderr: { write: () => {} }, stdout: { write: () => {} }, - registerProcessHandlers: false, composeDeps: async (input) => { factoryCalls += 1; return await daemonRuntimeDepsFixture()(input); }, - }); - assert.equal(runtime, null); - assert.deepEqual(exits, [1]); + }).catch((error: unknown) => error); + assert.equal(daemonStartupExitCode(refusal), 1); assert.equal(factoryCalls, 0, 'no runtime may be built under an unreadable policy'); - assert.match(stderr.join(''), /Daemon error: .*/); + assert.match((refusal as Error).message, /policy/i); } finally { - await runtime?.shutdown(); + fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts index 8e860c4ed2..e56771a9c8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-device-claims.test.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync } from '@agent-device/test-kit'; @@ -16,7 +17,6 @@ vi.mock('../../sessions/session-teardown.ts', async (importOriginal) => { import { SessionStore } from '../../core/session-store.ts'; import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import type { SessionState } from '../../core/session-state.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; const mockTeardownSessionResources = vi.mocked(teardownSessionResources); const roots: string[] = []; diff --git a/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts b/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts index af6061195a..f27d9d84f7 100644 --- a/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-idle-reap.test.ts @@ -30,11 +30,7 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-rt-'); const paths = resolveDaemonPaths(stateDir); - let exitCode: number | undefined; - let resolveExit: () => void; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -47,25 +43,22 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '80', }, }), - exit: (code) => { - exitCode = code; - resolveExit(); - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); assert.ok(fs.existsSync(paths.lockPath), 'daemon lock should be held right after startup'); - assert.equal(exitCode, undefined); + assert.equal(closed, false); await vi.advanceTimersByTimeAsync(79); - assert.equal(exitCode, undefined); + assert.equal(closed, false); await vi.advanceTimersByTimeAsync(1); await vi.runOnlyPendingTimersAsync(); await exited; - assert.equal(exitCode, 0); + assert.equal(closed, true); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -76,7 +69,7 @@ test('daemon runtime self-reaps after the idle window when nothing ever uses it' test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS is 0', async () => { vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-off-'); - let exitCode: number | undefined; + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -89,21 +82,20 @@ test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS i AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); await vi.advanceTimersByTimeAsync(200); - assert.equal(exitCode, undefined); - const shutdownPromise = runtime?.shutdown(); + assert.equal(closed, false); + const shutdownPromise = runtime.close(); await vi.runOnlyPendingTimersAsync(); await shutdownPromise; - assert.equal(exitCode, 0); + await exited; + assert.equal(closed, true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -112,11 +104,7 @@ test('daemon runtime never self-reaps when AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS i test('only a retained lease keeps the daemon runtime from self-reaping', async () => { vi.useFakeTimers(); const stateDir = mkdtempForTestSync('agent-device-daemon-idle-reap-leases-'); - let exitCode: number | undefined; - let resolveExit: () => void; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); + let closed = false; try { const runtime = await startDaemonRuntime({ @@ -129,15 +117,12 @@ test('only a retained lease keeps the daemon runtime from self-reaping', async ( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '80', }, }), - exit: (code) => { - exitCode = code; - resolveExit(); - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); - assert.notEqual(runtime, null); + const exited = runtime.closed.then(() => { + closed = true; + }); const [registry] = leaseProbe.registries; registry!.allocateLease({ tenantId: 'tenant-a', runId: 'run-plain', ttlMs: 60_000 }); const retained = registry!.allocateLease({ @@ -148,13 +133,13 @@ test('only a retained lease keeps the daemon runtime from self-reaping', async ( }); await vi.advanceTimersByTimeAsync(80); - assert.equal(exitCode, undefined); + assert.equal(closed, false); registry!.releaseLease({ leaseId: retained.leaseId }); await vi.advanceTimersByTimeAsync(80); await vi.runOnlyPendingTimersAsync(); await exited; - assert.equal(exitCode, 0); + assert.equal(closed, true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts new file mode 100644 index 0000000000..cb4333b091 --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-isolation.test.ts @@ -0,0 +1,106 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { interactorResolution } from '../../core/interactor-resolution.ts'; +import { isActiveProviderDevice } from '../../core/provider-device-admission.ts'; +import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +async function startIsolatedRuntime(label: string, interactorTag: string, providerActive: boolean) { + const stateDir = mkdtempForTestSync(`agent-device-daemon-isolation-${label}-`); + const runtime = await startDaemonRuntime({ + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }), + stderr: { write: () => {} }, + stdout: { write: () => {} }, + composeDeps: daemonRuntimeDepsFixture({ + interactorResolution: Object.freeze({ + resolve: async () => { + throw new Error(interactorTag); + }, + }), + providerDeviceAdmission: Object.freeze({ isActive: () => providerActive }), + }), + }); + const { httpPort, token } = JSON.parse( + fs.readFileSync(resolveDaemonPaths(stateDir).infoPath, 'utf8'), + ) as { httpPort: number; token: string }; + return { runtime, stateDir, httpPort, token }; +} + +async function callSessionList(httpPort: number, token: string, requestId: string) { + const response = await fetch(`http://127.0.0.1:${httpPort}/rpc`, { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method: 'agent_device.command', + params: { command: 'session_list', meta: { requestId } }, + }), + }); + return (await response.json()) as { result?: unknown; error?: unknown }; +} + +test('two runtimes in one process serve, close and keep their state independently', async () => { + let a: Awaited> | undefined; + let b: Awaited> | undefined; + try { + a = await startIsolatedRuntime('a', 'interactor-a', true); + b = await startIsolatedRuntime('b', 'interactor-b', false); + expect(a.httpPort).not.toBe(b.httpPort); + expect(path.basename(a.stateDir)).not.toBe(path.basename(b.stateDir)); + + // Each runtime answers requests with its own token and its own state dir. + expect((await callSessionList(a.httpPort, a.token, 'iso-a-1')).error).toBeUndefined(); + expect((await callSessionList(b.httpPort, b.token, 'iso-b-1')).error).toBeUndefined(); + + // Per-runtime state: what one installs or tracks is invisible to the other. + expect(a.runtime.run(() => isActiveProviderDevice({} as never))).toBe(true); + expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); + await expect( + a.runtime.run(() => interactorResolution().resolve({} as never, {})), + ).rejects.toThrow('interactor-a'); + await expect( + b.runtime.run(() => interactorResolution().resolve({} as never, {})), + ).rejects.toThrow('interactor-b'); + + const artifactPath = path.join(a.stateDir, 'artifact.txt'); + fs.writeFileSync(artifactPath, 'artifact-a'); + const artifactId = a.runtime.run(() => + trackDownloadableArtifact({ + artifactPath, + artifactType: 'runtime-artifact', + fileName: 'artifact.txt', + }), + ); + const fromB = await fetch( + `http://127.0.0.1:${b.httpPort}/artifacts/${encodeURIComponent(artifactId)}`, + { headers: { authorization: `Bearer ${b.token}`, connection: 'close' } }, + ); + expect(fromB.status).not.toBe(200); + await fromB.text(); + + // Closing one runtime leaves the other serving. + const reportA = await a.runtime.close(); + expect(reportA.sessions).toEqual({ closed: 0, failed: 0 }); + await expect(callSessionList(a.httpPort, a.token, 'iso-a-2')).rejects.toThrow(); + expect((await callSessionList(b.httpPort, b.token, 'iso-b-2')).error).toBeUndefined(); + expect(b.runtime.run(() => isActiveProviderDevice({} as never))).toBe(false); + + const reportB = await b.runtime.close(); + expect(reportB.plugins).toEqual({ disposed: [], failed: [], pending: [] }); + } finally { + await a?.runtime.close(); + await b?.runtime.close(); + } +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts index 3d2892f5a8..e059b1d2f8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lease-expiry.test.ts @@ -63,8 +63,6 @@ test('an expired lease ends its session without any further request', async () = AGENT_DEVICE_LEASE_MIN_TTL_MS: '1', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -78,7 +76,7 @@ test('an expired lease ends its session without any further request', async () = await vi.advanceTimersByTimeAsync(1_000); assert.deepEqual(leaseProbe.ended, [[lease.leaseId, 'LEASE_EXPIRED']]); - const shutdown = runtime?.shutdown(); + const shutdown = runtime?.close(); await vi.runOnlyPendingTimersAsync(); await shutdown; } finally { @@ -106,14 +104,12 @@ test('shutdown cancels and drains lease ends before it tears sessions down', asy AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); assert.notEqual(runtime, null); - await runtime?.shutdown(); + await runtime?.close(); assert.deepEqual(leaseProbe.shutdownSteps, ['cancel', 'drain', 'teardown']); } finally { diff --git a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts index 7b506c5324..cc162432c2 100644 --- a/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-lifecycle-shutdown.test.ts @@ -4,6 +4,7 @@ import { mkdtempForTestSync, createTestHostEnvironment, } from '@agent-device/test-kit'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import fs from 'node:fs'; import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; @@ -100,7 +101,7 @@ import { daemonRuntimeDepsFixture, } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; -import { startDaemonRuntime, teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; import { makeIosSession } from '../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; @@ -116,7 +117,6 @@ afterEach(() => { test('daemon shutdown detaches before session teardown and force-finalizes only after gateway resources', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-shutdown-'); - const exits: number[] = []; const startupErrors: string[] = []; try { const runtime = await startDaemonRuntime({ @@ -135,9 +135,11 @@ test('daemon shutdown detaches before session teardown and force-finalizes only phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -153,17 +155,14 @@ test('daemon shutdown detaches before session teardown and force-finalizes only AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => exits.push(code), - registerProcessHandlers: false, stderr: { write: (chunk) => startupErrors.push(chunk) }, stdout: { write: () => {} }, }); expect(runtime, startupErrors.join('')).not.toBeNull(); - await Promise.all([runtime?.shutdown(), runtime?.shutdown()]); + await Promise.all([runtime?.close(), runtime?.close()]); expect(lifecycleEvents).toEqual(['detach', 'gateway-shutdown', 'finalize']); - expect(exits).toEqual([0]); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -190,9 +189,11 @@ test('a SIGTERM shutdown gives the handoff a diagnostics scope to write its reas phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -208,14 +209,12 @@ test('a SIGTERM shutdown gives the handoff a diagnostics scope to write its reas AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); expect(runtime).not.toBeNull(); - await runtime?.shutdown(); + await runtime?.close(); const daemonLog = fs.readFileSync(path.join(stateDir, 'daemon.log'), 'utf8'); expect(daemonLog).toMatch(/"phase":"detach_scope_probe"/); @@ -245,9 +244,11 @@ test('shutdown includes drain publications, releases their claims and refuses po phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -263,8 +264,6 @@ test('shutdown includes drain publications, releases their claims and refuses po AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -298,14 +297,14 @@ test('shutdown includes drain publications, releases their claims and refuses po refusal = error; } }); - await runtime!.shutdown(); + await runtime!.close(); expect(shutdownProbe.finalize).toHaveBeenCalledOnce(); expect(refusal).toMatchObject({ details: { reason: 'daemon_shutting_down' } }); expect(store.lookup('draining')).toBeUndefined(); expect(store.lookup('late')).toBeUndefined(); expect(fs.existsSync(deviceClaimFilePath(acquired.ownership.deviceKey))).toBe(false); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -327,9 +326,11 @@ test('shutdown joins dispatch completion after force-closing the client before t phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -345,8 +346,6 @@ test('shutdown joins dispatch completion after force-closing the client before t AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -363,7 +362,7 @@ test('shutdown joins dispatch completion after force-closing the client before t closed = resolve; }); let request: Promise | undefined; - let shutdown: Promise | undefined; + let shutdown: Promise | undefined; try { expect(runtime).not.toBeNull(); const store = shutdownProbe.store!; @@ -413,7 +412,7 @@ test('shutdown joins dispatch completion after force-closing the client before t shutdownProbe.drain = async () => { closed(); }; - shutdown = runtime!.shutdown(); + shutdown = runtime!.close(); await serversClosed; await new Promise((resolve) => setImmediate(resolve)); expect(lifecycleEvents).not.toContain('detach'); @@ -429,7 +428,7 @@ test('shutdown joins dispatch completion after force-closing the client before t release(); await request; await shutdown; - await runtime?.shutdown(); + await runtime?.close(); } }); test.each([false, true])( @@ -496,9 +495,11 @@ test('daemon shutdown releases a session lease whose session teardown rejects', phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -514,8 +515,6 @@ test('daemon shutdown releases a session lease whose session teardown rejects', AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -531,7 +530,7 @@ test('daemon shutdown releases a session lease whose session teardown rejects', }); shutdownProbe.finalize.mockRejectedValueOnce(new Error('teardown failed')); - await runtime?.shutdown(); + await runtime?.close(); expect(shutdownProbe.finalize).toHaveBeenCalledOnce(); expect(leaseProbe.released).toEqual([lease]); @@ -557,9 +556,11 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy phase: 'detach_scope_probe', data: { lane: 'physical_coredevice' }, }); + return []; }, finalizeDaemonShutdown: async () => { lifecycleEvents.push('finalize'); + return []; }, }), shutdown: async () => { @@ -575,8 +576,6 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -589,7 +588,7 @@ test('daemon shutdown releases a retainOnClose lease that no session holds', asy retainOnClose: true, }); - await runtime?.shutdown(); + await runtime?.close(); expect(leaseProbe.released).toEqual([lease]); expect(leaseRegistry!.listActiveLeases()).toEqual([]); diff --git a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts index 6da4e3eebb..f77c0fd0e0 100644 --- a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts @@ -33,7 +33,7 @@ type DaemonPaths = ReturnType; // anything that verifies identity against the process table. const SUCCESSOR_PID = 999_999_999; -function startRuntime(stateDir: string, exit: (code: number) => void) { +function startRuntime(stateDir: string) { return startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(() => ({ // The post-lock, pre-publication step the runtime awaits first. Making it throw lands the @@ -62,8 +62,6 @@ function startRuntime(stateDir: string, exit: (code: number) => void) { AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -97,15 +95,13 @@ test('a shutdown whose daemon.json names a successor keeps the file and logs the // daemon's metadata and leaving every later client failing with hasInfo:false + retainedLockProcess. const stateDir = mkdtempForTestSync('agent-device-daemon-info-ownership-shutdown-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); publishSuccessor(paths); - await runtime?.shutdown(); + await runtime?.close(); - expect(exits).toEqual([0]); expect(lifecycleEvents).toContain('gateway-shutdown'); expect(fs.existsSync(paths.infoPath), 'the serving daemon keeps its metadata').toBe(true); expect(fs.existsSync(paths.lockPath), 'the exiting daemon still releases its own lock').toBe( @@ -126,7 +122,7 @@ test('a shutdown removes its own metadata and reports an unverified release', as const stateDir = mkdtempForTestSync('agent-device-daemon-info-owned-shutdown-'); const paths = resolveDaemonPaths(stateDir); try { - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); const originalRmdir = fs.rmdirSync; @@ -134,7 +130,7 @@ test('a shutdown removes its own metadata and reports an unverified release', as if (target === paths.lockPath) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); return originalRmdir(target, options); }); - await runtime?.shutdown(); + await runtime?.close(); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(logEvents(stateDir).map((event) => event.phase)).not.toContain( @@ -170,7 +166,7 @@ test('a failed provider plugin is logged and published after this daemon publish credentials: { variables: [], byBackend: {} }, } as const; providerComposition.inventory = [failed]; - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); try { const event = logEvents(stateDir).find(({ phase }) => phase === 'provider_plugin_failed'); expect(event?.data).toMatchObject({ @@ -181,7 +177,7 @@ test('a failed provider plugin is logged and published after this daemon publish const published = JSON.parse(fs.readFileSync(resolveDaemonPaths(stateDir).infoPath, 'utf8')); expect(published.plugins).toEqual([failed]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -190,13 +186,9 @@ test('a startup that fails before publication removes no record and writes no lo // silent: reporting an "absent" record here would create a daemon.log on every such startup. const stateDir = mkdtempForTestSync('agent-device-daemon-info-startup-absent-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; startupFailure.active = true; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); - - expect(runtime).toBeNull(); - expect(exits).toEqual([1]); + await expect(startRuntime(stateDir)).rejects.toThrow('startup step failed'); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(fs.existsSync(paths.logPath), 'nothing was declined, so nothing is logged').toBe(false); } finally { @@ -207,14 +199,10 @@ test('a startup that fails before publication removes no record and writes no lo test('a startup that fails before publication leaves a foreign record standing', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-info-startup-foreign-'); const paths = resolveDaemonPaths(stateDir); - const exits: number[] = []; publishSuccessor(paths); startupFailure.active = true; try { - const runtime = await startRuntime(stateDir, (code) => exits.push(code)); - - expect(runtime).toBeNull(); - expect(exits).toEqual([1]); + await expect(startRuntime(stateDir)).rejects.toThrow('startup step failed'); expect(fs.existsSync(paths.infoPath)).toBe(true); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); @@ -230,7 +218,7 @@ test('a successor published during this daemon lifetime is recorded at detection const stateDir = mkdtempForTestSync('agent-device-daemon-metadata-lost-'); const paths = resolveDaemonPaths(stateDir); try { - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); expect(runtime).not.toBeNull(); publishSuccessor(paths); @@ -244,7 +232,7 @@ test('a successor published during this daemon lifetime is recorded at detection ); vi.useRealTimers(); - await runtime?.shutdown(); + await runtime?.close(); } finally { vi.useRealTimers(); fs.rmSync(stateDir, { recursive: true, force: true }); @@ -266,14 +254,15 @@ test('both exits tear the watch down before they touch daemon.json', () => { // A watch still armed when the removal runs can report a loss for a record its own process just // deleted. The startup-failure half of this is unreachable from a test — everything after // publication needs a real toolchain — so the invariant is read off the source, the same way the - // arming order above is. + // arming order above is. Startup rollback runs its steps in reverse, so the registration that + // stops the watch comes after the one that finishes the registration; the close path stops first. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); const stops = [...source.matchAll(/stopMetadataLossWatch\(\);/g)].map((m) => m.index); const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); expect(stops).toHaveLength(2); expect(finishes).toHaveLength(2); - expect(finishes[0]).toBeGreaterThan(stops[0]!); - expect(finishes[0]).toBeLessThan(stops[1]!); + expect(stops[0]).toBeGreaterThan(finishes[0]!); + expect(stops[1]).toBeGreaterThan(stops[0]!); expect(finishes[1]).toBeGreaterThan(stops[1]!); }); @@ -293,9 +282,9 @@ test('a plugin whose cleanup failed at shutdown is recorded in daemon.log', asyn dispose: { status: 'ok' }, }, ]; - const runtime = await startRuntime(stateDir, () => {}); + const runtime = await startRuntime(stateDir); try { - await runtime?.shutdown(); + await runtime?.close(); const events = logEvents(stateDir).filter( ({ phase }) => phase === 'provider_plugin_cleanup_failed', ); diff --git a/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts b/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts index a838601426..21e1c4f71e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-owner-lifecycle.test.ts @@ -33,8 +33,6 @@ function startDaemon( AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({ @@ -70,7 +68,7 @@ test('daemon startup configures its platform owners before it publishes readines }), ]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); @@ -94,6 +92,6 @@ test('a platform owner diagnostic about a refused restore is logged and the daem }), ]); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-png-worker.test.ts b/packages/daemon/src/server/http/daemon-runtime-png-worker.test.ts index d481c30442..d420f462a9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-png-worker.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-png-worker.test.ts @@ -45,8 +45,6 @@ async function pngWorkerEventsAfter(spawn: 'start' | 'throw', act: () => void): AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: daemonRuntimeDepsFixture({}), @@ -64,7 +62,7 @@ async function pngWorkerEventsAfter(spawn: 'start' | 'throw', act: () => void): await new Promise((resolve) => setTimeout(resolve, 50)); return readEvents(); } finally { - await runtime?.shutdown(); + await runtime?.close(); fs.rmSync(stateDir, { recursive: true, force: true }); } } diff --git a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts index de8c21c769..110b06d9c5 100644 --- a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts @@ -99,10 +99,6 @@ type Booted = { async function bootDaemon(options: { serverMode?: 'socket' | 'dual' } = {}): Promise { const stateDir = mkdtempForTestSync('agent-device-daemon-quiesce-'); const paths = resolveDaemonPaths(stateDir); - let resolveExit: (code: number) => void = () => {}; - const exit = new Promise((resolve) => { - resolveExit = resolve; - }); const runtime = await startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(), host: createTestHostEnvironment({ @@ -113,12 +109,11 @@ async function bootDaemon(options: { serverMode?: 'socket' | 'dual' } = {}): Pro AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: resolveExit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); if (!runtime?.socketPort) throw new Error('daemon did not start'); + const exit = runtime.closed.then(() => 0); const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { pid: number; processStartTime?: string; @@ -217,7 +212,7 @@ test('a daemon with a request in flight refuses to quiesce and keeps serving', a } finally { release(); await running.catch(() => {}); - await daemon.runtime.shutdown(); + await daemon.runtime.close(); } }); @@ -253,7 +248,7 @@ test('a daemon holding a session, a repair session or a live lease refuses to qu expect((await daemon.quiesce()).ok).toBe(true); expect(await daemon.exit).toBe(0); } finally { - await daemon.runtime.shutdown(); + await daemon.runtime.close(); } }); @@ -271,7 +266,7 @@ test('a quiesce with the wrong token or for another daemon instance leaves the d expect((await daemon.sessionList()).ok).toBe(true); } finally { - await daemon.runtime.shutdown(); + await daemon.runtime.close(); } }); @@ -311,7 +306,7 @@ test('a quiesce that arrives as a remote-origin HTTP request is refused', async expect(JSON.stringify(body)).toContain('"UNAUTHORIZED"'); expect((await daemon.sessionList()).ok).toBe(true); } finally { - await daemon.runtime.shutdown(); + await daemon.runtime.close(); } }); @@ -329,10 +324,6 @@ test('a quiesce asked while the daemon is still starting up is refused as busy, probe.startupGate = new Promise((resolve) => { openGate = resolve; }); - let resolveExit: (code: number) => void = () => {}; - const exit = new Promise((resolve) => { - resolveExit = resolve; - }); const starting = startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(), host: createTestHostEnvironment({ @@ -343,8 +334,6 @@ test('a quiesce asked while the daemon is still starting up is refused as busy, AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: resolveExit, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -376,13 +365,14 @@ test('a quiesce asked while the daemon is still starting up is refused as busy, }); openGate(); - await starting; + const runtime = await starting; + const exit = runtime.closed.then(() => 0); const again = await sendSocketRequest(info.port, quiesceRequest); expect(again).toEqual({ ok: true, data: { outcome: 'draining' } }); expect(await exit).toBe(0); expect(fs.existsSync(paths.infoPath)).toBe(false); } finally { openGate(); - await starting.then((runtime) => runtime?.shutdown()); + await starting.then((runtime) => runtime?.close()); } }); diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts index 8304b4c083..ed04ed819e 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-reaper.test.ts @@ -35,8 +35,6 @@ test('daemon startup reaps orphaned simctl recorders with a graceful finalize wi AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -50,7 +48,7 @@ test('daemon startup reaps orphaned simctl recorders with a graceful finalize wi // so it releases the host recording lock instead of leaving it dangling for the next session. expect(Number(recordingReap!.termTimeoutMs)).toBeGreaterThanOrEqual(5_000); - await runtime?.shutdown(); + await runtime?.close(); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts index f8b95f5e83..1e790f0321 100644 --- a/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-recording-teardown.test.ts @@ -1,11 +1,11 @@ import path from 'node:path'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { afterEach, expect, test, vi } from 'vitest'; import type { ScreenRecordingCompletion } from '@agent-device/contracts/screen-recording-runtime'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { SessionStore } from '../../core/session-store.ts'; import { makeRecordingSession } from '../../__tests__/support/session-teardown.fixtures.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts b/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts index ec814ad7e9..1526f796b8 100644 --- a/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-rpc-flow-failure.test.ts @@ -52,7 +52,6 @@ test('a /rpc flow failure the server refused itself reaches the runtime stderr a expect(observed.events).toHaveLength(1); expect(observed.events[0]?.data).toEqual({ code: 'UNKNOWN', message: 'Unknown error' }); expect(observed.rejections).toEqual([]); - expect(observed.exitCode).toBeUndefined(); expect(observed.nextCall).toEqual({ status: 400, code: -32600 }); }); @@ -67,7 +66,6 @@ test('a /rpc flow failure still reaches daemon.log when the injected stderr thro expect(observed.events[0]?.phase).toBe('daemon_rpc_request_flow_failed'); expect(observed.stderrLines).toEqual([]); expect(observed.rejections).toEqual([]); - expect(observed.exitCode).toBeUndefined(); }); type FlowFailure = { @@ -76,7 +74,6 @@ type FlowFailure = { events: Array<{ phase: string; data?: Record }>; stderrLines: string[]; rejections: unknown[]; - exitCode: number | undefined; }; /** @@ -94,8 +91,7 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise { rejections.push(reason); }; - let exitCode: number | undefined; - let runtime: Awaited> = null; + let runtime: Awaited> | null = null; try { runtime = await startDaemonRuntime({ composeDeps: daemonRuntimeDepsFixture(), @@ -107,10 +103,6 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: (chunk: string) => { if (options.stderrThrows) throw new Error('the stream is gone'); @@ -143,14 +135,13 @@ async function driveFlowFailure(options: { stderrThrows?: boolean }): Promise ({ @@ -77,7 +77,7 @@ for (const initialSession of ['published', 'draft'] as const) { (error: unknown) => ({ error }), ); const stderr: string[] = []; - let teardown: Promise | undefined; + let teardown: Promise | undefined; try { await captureEntered.promise; vi.useFakeTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts index 46e2b2d332..12376f23d9 100644 --- a/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-web-close-teardown.test.ts @@ -1,10 +1,10 @@ import path from 'node:path'; +import { teardownDaemonSessionForShutdown } from './server-shutdown.ts'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync, WEB_DESKTOP_DEVICE } from '@agent-device/test-kit'; import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; import { makeWebSession } from '../../__tests__/support/session-teardown.fixtures.ts'; import { SessionStore } from '../../core/session-store.ts'; -import { teardownDaemonSessionForShutdown } from './daemon-runtime.ts'; afterEach(() => { vi.useRealTimers(); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index cc4b0f0ea4..9c4b1f51af 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -36,14 +36,23 @@ import { getLeaseRegistryExecutionLocks, } from '../request-execution-scope.ts'; import type { LeaseEndReason } from '../../sessions/lease-lifecycle.ts'; -import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; import { finalizeDaemonSessionApplicationLifecycle } from '../application-lifecycle-recovery.ts'; import { runtimeHintValues } from '../../sessions/session-runtime.ts'; -import { closeDaemonServers } from './server-shutdown.ts'; +import { + closeDaemonServers, + createRuntimeContext, + teardownDaemonSessionForShutdown, + type RuntimeContext, + type WritableOutput, + createPluginClose, + createStartupRollback, + type ShutdownReport, + type StartupRollback, +} from './server-shutdown.ts'; import type { DaemonInvokeFn } from '../../core/daemon-request.ts'; -import type { SessionRef, SessionState } from '../../core/session-state.ts'; -import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; +import type { SessionRef } from '../../core/session-state.ts'; import { createDaemonIdleReap, createDaemonQuiesce, @@ -63,6 +72,8 @@ import { createAudioProbeAdmissionLedger } from '@agent-device/capture-kit/audio import { createPerfCaptureAdmissionLedger } from '@agent-device/capture-kit/perf-capture-admission-ledger'; import { createScreenRecordingAdmissionLedger } from '@agent-device/capture-kit/screen-recording-admission-ledger'; import { withCommandHost } from '@agent-device/host-kit/command'; +import { cancelAllRequests } from '@agent-device/host-kit/request'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import { emitDiagnostic, type ResourceDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import { @@ -90,12 +101,11 @@ import { listenNetServer, type DaemonServer, } from './transport.ts'; -import { prewarmPngWorker, terminatePngWorker } from '@agent-device/capture-kit/png-worker-client'; +import { retainPngWorker } from '@agent-device/capture-kit/png-worker-client'; import { openWebSessionNames } from '../../sessions/web-session-names.ts'; import { createAppLogAdmissionLedger } from '../../resources/app-log-admission-ledger.ts'; import type { AppLogSessionArtifacts } from '@agent-device/contracts/app-log-runtime'; import type { PlatformOwnerLifecycle } from '../../core/platform-owner-lifecycle.ts'; -import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; import type { LeaseLifecycleProvider, ProviderExpiredLeaseRecovery, @@ -110,15 +120,14 @@ const DAEMON_LEASE_RELEASE_TIMEOUT_MS = 1_000; const LEASE_EXPIRY_SWEEP_INTERVAL_MS = 1_000; const DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS = 1_000; const DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS = 2_000; +// A handler that ignores cancellation must not hold `close()` forever; past this it is reported +// abandoned and the registration stays held while the process lives. +const DAEMON_REQUEST_DRAIN_TIMEOUT_MS = 5_000; // An orphaned `simctl recordVideo` releases the host-wide recording lock only after it finishes // finalizing on SIGINT; force-killing it sooner leaves every later recording failing with EBUSY // (#2170). Bound the grace to the recorder purpose, so daemon startup stays under the client budget. const DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS = 5_000; -type WritableOutput = { - write: (chunk: string) => unknown; -}; - /** * Every capability the daemon runtime cannot construct for itself: the platform and provider * runtimes, host diagnostics, credential fingerprinting, resource cleanup, the platform lifecycle @@ -129,6 +138,8 @@ type WritableOutput = { * same factory, so the production assembly is the one every runtime test boots. */ export type DaemonRuntimeDeps = Readonly<{ + /** State the composition's platform modules keep per daemon runtime (the Apple runner's). */ + runtimeStates: readonly RuntimeStateHandle[]; ownedProcessRecords: OwnedProcessRecordStore; hostDiagnostics: HostDiagnostics; ownerLifecycle: PlatformOwnerLifecycle; @@ -210,19 +221,36 @@ export type DaemonRuntimeOptions = { host: HostEnvironment; stdout?: WritableOutput; stderr?: WritableOutput; - exit?: (code: number) => void; - registerProcessHandlers?: boolean; /** The root's runtime composition, as a factory so the server owns paths, policy, and store. */ composeDeps: ComposeDaemonRuntimeDeps; + /** Shutdown time budgets, for hosts and tests that need other than the defaults. */ + shutdownBudgets?: { requestDrainMs?: number; pluginMs?: number }; }; export type DaemonRuntimeController = { httpPort?: number; socketPort?: number; - shutdown: (options?: { exitCode?: number; cause?: unknown }) => Promise; + /** + * Closes the runtime. Concurrent and repeated calls share one close, whose report every caller + * receives. The runtime never exits the process: the executable owns signals and exit codes. + */ + close: (options?: { cause?: unknown }) => Promise; + /** Settles once the runtime has closed, whoever closed it (an idle reap closes it unasked). */ + closed: Promise; + /** Runs `fn` as this runtime, for code that serves it from outside its own callbacks. */ + run: (fn: () => T) => T; token: string; }; +/** + * Thrown by {@link startDaemonRuntime} when the daemon must not run: its policy is unusable or + * another daemon owns the state dir. `details.exitCode` is the code the executable exits with. + */ +export function daemonStartupExitCode(error: unknown): number { + const exitCode = asAppError(error).details?.exitCode; + return typeof exitCode === 'number' ? exitCode : 1; +} + export async function flushDaemonStartupDiagnostics( logPath: string, diagnostics: readonly ResourceDiagnostic[], @@ -300,126 +328,34 @@ function armDaemonMetadataLossWatch( }); } -async function settleDaemonTeardownStep(params: { - session: SessionState; - stderr: WritableOutput; - resource: 'app-log' | 'session' | 'lifecycle'; - teardown: () => Promise; -}): Promise { - const { session, stderr, resource, teardown } = params; - try { - await teardown(); - return true; - } catch (error) { - stderr.write( - `Daemon ${resource} teardown error (${session.name}): ${ - error instanceof Error ? error.message : String(error) - }\n`, - ); - return false; - } -} - -/** - * Daemon-shutdown teardown of one session: bounded resource cleanup (budget - * from {@link resolveDaemonSessionTeardownTimeoutMs}, resolved BEFORE cleanup - * starts since finalizing the recording detaches `session.screenRecording`), then the - * repair-commit finalization and session deletion. Cleanup failures — including - * a recorder that could not be finalized — surface on stderr instead of being - * silently swallowed. - */ -export async function teardownDaemonSessionForShutdown(params: { - ref: SessionRef; - sessionStore: SessionStore; - stderr: WritableOutput; - platformCleanup: PlatformResourceCleanup; - finalizeApplicationLifecycle?: ( - session: SessionState, - runtimeHints: RuntimeHintValues, - ) => Promise; - afterSuccessfulTeardown?: (session: SessionState) => Promise; -}): Promise { - const { - ref, - sessionStore, - stderr, - platformCleanup, - finalizeApplicationLifecycle, - afterSuccessfulTeardown, - } = params; - const current = sessionStore.resolveCurrent(ref); - const session = current ?? ref.session; - const runtimeHints = runtimeHintValues( - current ? sessionStore.getRuntimeHints(ref.address) : undefined, - ); - const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); - // The ownership-fenced app-log side effect must settle while this process - // still owns the daemon lock. It is intentionally outside the generic - // teardown race so lock release and runtime shutdown cannot overtake it. - - const appLogTeardownSucceeded = await settleDaemonTeardownStep({ - session, - stderr, - resource: 'app-log', - teardown: async () => await stopSessionAppLog({ ref, sessionStore }), - }); - const sessionAfterAppLog = sessionStore.resolveCurrent(ref) ?? session; - const teardown = (async () => { - const genericTeardownSucceeded = await settleDaemonTeardownStep({ - session, - stderr, - resource: 'session', - teardown: async () => - await teardownSessionResources({ - appLog: 'already-settled', - ref, - sessionStore, - platformCleanup, - }), - }); - const lifecycleTeardownSucceeded = finalizeApplicationLifecycle - ? await settleDaemonTeardownStep({ - session, - stderr, - resource: 'lifecycle', - teardown: async () => - await finalizeApplicationLifecycle(sessionAfterAppLog, runtimeHints), - }) - : true; - return genericTeardownSucceeded && lifecycleTeardownSucceeded; - })(); - const genericTeardownSucceeded = await Promise.race([ - teardown, - sleep(timeoutMs).then(() => { - stderr.write(`Daemon session teardown timed out (${session.name}).\n`); - return false; - }), - ]); - const teardownSucceeded = appLogTeardownSucceeded && genericTeardownSucceeded; - // ADR 0012 decision 6, R7 + commit semantics (C2/C5a): commit the healed - // `.ad` iff the repair transaction completed, else leave a bounded - // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. - sessionStore.finalizeRepairTeardown(ref); - if (teardownSucceeded) await afterSuccessfulTeardown?.(session); - sessionStore.retire(ref); -} - export async function startDaemonRuntime( options: DaemonRuntimeOptions, -): Promise { - return await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { - return await startDaemonRuntimeOnHost(options); - }); +): Promise { + const runtimeContext = createRuntimeContext(); + return await runtimeContext.run( + async () => + await withCommandHost({ host: options.host, requestEnv: undefined }, async () => { + const rollback = createStartupRollback(); + rollback.onRollback(() => runtimeContext.dispose()); + try { + return await startDaemonRuntimeOnHost(options, rollback, runtimeContext); + } catch (error) { + await rollback.run(); + throw error; + } + }), + ); } async function startDaemonRuntimeOnHost( options: DaemonRuntimeOptions, -): Promise { + rollback: StartupRollback, + runtimeContext: RuntimeContext, +): Promise { const { host } = options; const { env } = host; const stdout = options.stdout ?? process.stdout; const stderr = options.stderr ?? process.stderr; - const exit = options.exit ?? ((code: number) => process.exit(code)); const daemonPaths = resolveDaemonPaths(env.AGENT_DEVICE_STATE_DIR, { env, cwd: host.cwd }); const { baseDir, infoPath, logPath, sessionsDir } = daemonPaths; const daemonServerMode = resolveDaemonServerMode(env.AGENT_DEVICE_DAEMON_SERVER_MODE); @@ -427,14 +363,7 @@ async function startDaemonRuntimeOnHost( // ADR 0029: a policy that cannot be read or validated stops startup; the daemon never runs // with a weaker policy than its operator named. The server loads it before composing any // runtime, so no capability is ever built under a policy that failed validation. - let daemonPolicy: DaemonPolicy | undefined; - try { - daemonPolicy = loadDaemonPolicy(env, host.cwd); - } catch (error) { - stderr.write(`Daemon error: ${asAppError(error).message}\n`); - exit(1); - return null; - } + const daemonPolicy: DaemonPolicy | undefined = loadDaemonPolicy(env, host.cwd); // Derived here, beside the policy it reads: the server hands the same refusal to the root's // composition, so a runtime can never be built without the operator's shutdown gate. const assertShutdownAllowed = daemonPolicy @@ -454,6 +383,7 @@ async function startDaemonRuntimeOnHost( }), openWebSessionNames: () => openWebSessionNames(sessionStore), }); + runtimeContext.extend(deps.runtimeStates); const appLogAdmissionLedger = createAppLogAdmissionLedger(); const audioProbeAdmissionLedger = createAudioProbeAdmissionLedger(); const perfCaptureAdmissionLedger = createPerfCaptureAdmissionLedger(); @@ -463,26 +393,20 @@ async function startDaemonRuntimeOnHost( const daemonIdentity = readCurrentOwnerIdentity(); const daemonCodeOrigin = resolveDaemonCodeOrigin(); const daemonCodeSignature = resolveDaemonCodeSignature(); + const requestDrainTimeoutMs = + options.shutdownBudgets?.requestDrainMs ?? DAEMON_REQUEST_DRAIN_TIMEOUT_MS; + // The registry is the one plugin cleanup owner: closing it shuts each runtime down, then disposes + // the plugin, and it is idempotent, so the final close, a startup rollback and the last release + // all go through the same call. + const pluginClose = createPluginClose({ + close: deps.shutdownProviderRuntimes, + inventory: deps.pluginInventory, + timeoutMs: options.shutdownBudgets?.pluginMs, + }); + rollback.onRollback(async () => { + await pluginClose.close(); + }); const deviceRuntimeGateway = deps.deviceRuntimeGateway; - // Provider cleanup is best effort for the daemon, but a plugin whose cleanup failed is recorded. - // A daemon that lost the registration does not own `daemon.log`, so it reports on stderr instead. - const closeProviders = async (report: 'daemon-log' | 'stderr'): Promise => { - const outcomes = await deps.shutdownProviderRuntimes().catch(() => []); - for (const { provider, source, shutdown, dispose } of outcomes) { - if (shutdown.status === 'ok' && dispose.status === 'ok') continue; - const data = { - provider, - source, - shutdown: shutdown.status === 'failed' ? shutdown.error : undefined, - dispose: dispose.status === 'failed' ? dispose.error : undefined, - }; - if (report === 'daemon-log') { - await emitDaemonDiagnostic(logPath, 'provider_plugin_cleanup_failed', data); - } else { - stderr.write(`Provider plugin cleanup failed: ${JSON.stringify(data)}\n`); - } - } - }; const applicationLifecycle = deviceRuntimeGateway.applicationLifecycle; if (!applicationLifecycle) { throw new AppError('COMMAND_FAILED', 'Platform lifecycle gateway is not configured.', { @@ -498,6 +422,7 @@ async function startDaemonRuntimeOnHost( stateDir: baseDir, recoverableProviderIds: deps.recoverableProviderIds, }); + rollback.onRollback(() => expiredProviderLeaseReleaser.shutdown()); void expiredProviderLeaseReleaser.retryPending(); const leaseRegistry = new LeaseRegistry({ maxActiveSimulatorLeases: parseIntegerEnv(env.AGENT_DEVICE_MAX_SIMULATOR_LEASES), @@ -516,6 +441,7 @@ async function startDaemonRuntimeOnHost( LEASE_EXPIRY_SWEEP_INTERVAL_MS, ); leaseExpirySweep.unref(); + rollback.onRollback(() => clearInterval(leaseExpirySweep)); const dispatchRequest = createRequestHandler({ host, logPath, @@ -556,10 +482,10 @@ async function startDaemonRuntimeOnHost( const shutdownClaimLedger = createDaemonShutdownClaimLedger(env); - const teardownDaemonSession = async (ref: SessionRef): Promise => { + const teardownDaemonSession = async (ref: SessionRef): Promise => { const session = sessionStore.resolveCurrent(ref) ?? ref.session; try { - await teardownDaemonSessionForShutdown({ + return await teardownDaemonSessionForShutdown({ ref, sessionStore, stderr, @@ -579,10 +505,14 @@ async function startDaemonRuntimeOnHost( } }; - const teardownDaemonSessions = async (): Promise => { + const teardownDaemonSessions = async (): Promise => { sessionStore.closeAdmission(); const sessionsToStop = sessionStore.listRefs(); - await Promise.all(sessionsToStop.map(teardownDaemonSession)); + const outcomes = await Promise.all( + sessionsToStop.map(async (ref) => await teardownDaemonSession(ref).catch(() => false)), + ); + const closedCount = outcomes.filter(Boolean).length; + return { closed: closedCount, failed: outcomes.length - closedCount }; }; // #2833: settles the resources of a session this daemon expires for idleness. Deliberately NOT @@ -654,7 +584,7 @@ async function startDaemonRuntimeOnHost( // Reaps this daemon process when it sits fully idle (no open sessions, no // in-flight requests, no active recording) past AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS. - // `shutdown` is defined below but only invoked asynchronously by the timer, + // `close` is defined below but only invoked asynchronously by the timer, // well after this closure captures it. let shuttingDown = false; const inFlightRequests = new Set>(); @@ -663,11 +593,17 @@ async function startDaemonRuntimeOnHost( getInFlightRequestCount: () => inFlightRequests.size, hasRetainedLeases: () => leaseRegistry.hasRetainedLeases(), onIdleReap: () => { - void shutdown(); + void close(); }, env, }); + rollback.onRollback(() => { + idleReap.cancel(); + sessionIdleExpiry.cancel(); + leaseSessionEnd.cancel(); + }); + // The quiesce request is answered before dispatch and never joins `inFlightRequests`, so the // count it reads needs no allowance for it. The servers listen and `daemon.json` is published // before `shutdown` exists, and a daemon answers `draining` only once it has a drain path to run. @@ -680,14 +616,14 @@ async function startDaemonRuntimeOnHost( inFlightRequestCount: () => inFlightRequests.size, startingUp: () => startingUp, drain: () => { - void shutdown(); + void close(); }, }); const handleRequest: DaemonInvokeFn = quiesce.admit(async (req) => { if (shuttingDown) return daemonShuttingDownResponse(); idleReap.cancel(); - const dispatch = dispatchRequest(req); + const dispatch = runtimeContext.run(() => dispatchRequest(req)); inFlightRequests.add(dispatch); try { return await dispatch; @@ -784,137 +720,160 @@ async function startDaemonRuntimeOnHost( const acquisition = await tryAcquireDaemonRegistration(daemonPaths); if (acquisition.status !== 'acquired') { - await closeProviders('stderr'); - stderr.write(`Daemon registration ${acquisition.status}; exiting.\n`); - exit(DAEMON_STARTUP_EXIT_CODES[acquisition.status]); - return null; + throw new AppError('COMMAND_FAILED', `Daemon registration ${acquisition.status}.`, { + reason: 'daemon_registration_refused', + status: acquisition.status, + exitCode: DAEMON_STARTUP_EXIT_CODES[acquisition.status], + }); } const registration = acquisition.owner; - + rollback.onRollback(async () => { + await deps.ownerLifecycle.clearDaemonLockConfiguration(); + }); + rollback.onRollback(async () => { + await finishDaemonRegistration({ registration, infoPath, logPath }); + }); let servers: DaemonServer[] = []; - let socketPort: number | undefined; - let httpPort: number | undefined; + rollback.onRollback(async () => { + closeServersBestEffort(servers); + stopMetadataLossWatch(); + }); const startupDiagnostics: ResourceDiagnostic[] = []; - try { - await deps.ownerLifecycle.configureForDaemonLock({ - stateDir: baseDir, - hasDeviceClaimAuthority: (device) => processOwnsActiveDeviceClaim(device, env), - onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), - }); - const legacyMarkerRecovery = await deps.ownerLifecycle.recoverLegacyAppLogMarkers(sessionsDir); - appLogAdmissionLedger.retainLegacyMarkers(legacyMarkerRecovery.retained); - for (const markerPath of legacyMarkerRecovery.recovered) { - startupDiagnostics.push({ - phase: 'app_log_legacy_marker_recovered', - resourcePath: markerPath, - data: {}, - }); - } - for (const retained of legacyMarkerRecovery.retained) { - startupDiagnostics.push({ - phase: 'app_log_legacy_marker_retained', - resourcePath: retained.markerPath, - data: { - reason: retained.reason, - ...(retained.message === undefined ? {} : { message: retained.message }), - }, - }); - } - await recoverAppLogResourcesAfterDaemonLock({ - sessionsDir, - gateway: deviceRuntimeGateway, - scope: createRecoveryPlatformScope(), - onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + await deps.ownerLifecycle.configureForDaemonLock({ + stateDir: baseDir, + hasDeviceClaimAuthority: (device) => processOwnsActiveDeviceClaim(device, env), + onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + }); + const legacyMarkerRecovery = await deps.ownerLifecycle.recoverLegacyAppLogMarkers(sessionsDir); + appLogAdmissionLedger.retainLegacyMarkers(legacyMarkerRecovery.retained); + for (const markerPath of legacyMarkerRecovery.recovered) { + startupDiagnostics.push({ + phase: 'app_log_legacy_marker_recovered', + resourcePath: markerPath, + data: {}, }); - await reapOwnedProcessRecordsAtStartup(deps.ownedProcessRecords, { - openWebSessionNames: openWebSessionNames(sessionStore), - purposes: ['simctl-screen-recording'], - termTimeoutMs: DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS, + } + for (const retained of legacyMarkerRecovery.retained) { + startupDiagnostics.push({ + phase: 'app_log_legacy_marker_retained', + resourcePath: retained.markerPath, + data: { + reason: retained.reason, + ...(retained.message === undefined ? {} : { message: retained.message }), + }, }); - await deps.cleanupManagedWebOrphans({ - stateDir: baseDir, - openWebSessionNames: openWebSessionNames(sessionStore), - ownedProcessRecords: deps.ownedProcessRecords, + } + await recoverAppLogResourcesAfterDaemonLock({ + sessionsDir, + gateway: deviceRuntimeGateway, + scope: createRecoveryPlatformScope(), + onDiagnostic: (diagnostic) => startupDiagnostics.push(diagnostic), + }); + await reapOwnedProcessRecordsAtStartup(deps.ownedProcessRecords, { + openWebSessionNames: openWebSessionNames(sessionStore), + purposes: ['simctl-screen-recording'], + termTimeoutMs: DAEMON_RECORDING_REAP_TERM_TIMEOUT_MS, + }); + await deps.cleanupManagedWebOrphans({ + stateDir: baseDir, + openWebSessionNames: openWebSessionNames(sessionStore), + ownedProcessRecords: deps.ownedProcessRecords, + }); + // Marker-gated lifecycle recovery owns test-IME orphan repair. Its implementation remains + // lazy until the marker exists, so a normal daemon startup does not load or probe adb. + void applicationLifecycle.recoverStartupResources({ stateDir: baseDir }).catch((error) => { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_lifecycle_startup_recovery_failed', + data: { error: error instanceof Error ? error.message : String(error) }, }); - // Marker-gated lifecycle recovery owns test-IME orphan repair. Its implementation remains - // lazy until the marker exists, so a normal daemon startup does not load or probe adb. - void applicationLifecycle.recoverStartupResources({ stateDir: baseDir }).catch((error) => { - emitDiagnostic({ - level: 'warn', - phase: 'daemon_lifecycle_startup_recovery_failed', - data: { error: error instanceof Error ? error.message : String(error) }, - }); + }); + const opened = await openDaemonServers(); + servers = opened.servers; + const { socketPort, httpPort } = opened; + publishDaemonInfo(socketPort, httpPort); + stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); + await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); + for (const plugin of deps.pluginInventory.filter(({ state }) => state === 'failed')) { + await emitDaemonDiagnostic(logPath, 'provider_plugin_failed', { + provider: plugin.provider, + package: plugin.package, + source: plugin.source, + code: plugin.error?.code, + reason: plugin.error?.reason, + message: plugin.error?.message, + hint: plugin.error?.hint, }); - const opened = await openDaemonServers(); - servers = opened.servers; - socketPort = opened.socketPort; - httpPort = opened.httpPort; - publishDaemonInfo(socketPort, httpPort); - stopMetadataLossWatch = armDaemonMetadataLossWatch(baseDir, infoPath, logPath, daemonIdentity); - await flushDaemonStartupDiagnostics(logPath, startupDiagnostics); - for (const plugin of deps.pluginInventory.filter(({ state }) => state === 'failed')) { - await emitDaemonDiagnostic(logPath, 'provider_plugin_failed', { - provider: plugin.provider, - package: plugin.package, - source: plugin.source, - code: plugin.error?.code, - reason: plugin.error?.reason, - message: plugin.error?.message, - hint: plugin.error?.hint, - }); - } - // After publication: publishDaemonInfo truncates daemon.log, so anything - // written before it is lost — including reconciliation diagnostics. - await reconcileDeviceClaimsForDaemonStartup( - logPath, - createOwnerScopedDeviceClaimReconciler({ - scope: createRecoveryPlatformScope(), - composeGateway: deps.claimRecoveryGateway, - }), - baseDir, - env, - ); - // Arms the initial idle-reap timer: a daemon that starts and never - // receives a request must still be able to reap itself. - idleReap.noteActivity(); - // The #2833 deadline needs no equivalent arm: the store is empty until a request puts a session - // in it, and that request's own completion re-arms the reaper. - } catch (error) { - const appErr = asAppError(error); - stderr.write(`Daemon error: ${appErr.message}\n`); - closeServersBestEffort(servers); - stopMetadataLossWatch(); - await closeProviders('daemon-log'); - await finishDaemonRegistration({ registration, infoPath, logPath }); - await deps.ownerLifecycle.clearDaemonLockConfiguration(); - exit(1); - return null; } + // After publication: publishDaemonInfo truncates daemon.log, so anything + // written before it is lost — including reconciliation diagnostics. + await reconcileDeviceClaimsForDaemonStartup( + logPath, + createOwnerScopedDeviceClaimReconciler({ + scope: createRecoveryPlatformScope(), + composeGateway: deps.claimRecoveryGateway, + }), + baseDir, + env, + ); + // Arms the initial idle-reap timer: a daemon that starts and never + // receives a request must still be able to reap itself. + idleReap.noteActivity(); + // The #2833 deadline needs no equivalent arm: the store is empty until a request puts a session + // in it, and that request's own completion re-arms the reaper. // Spawn the PNG worker ahead of the first screenshot request so its // cold-start cost is not paid on a user-visible call. Best effort: when it // cannot start or later stops, PNG processing falls back to the in-process sync path and the // daemon log records why, once, outside any request. - prewarmPngWorker((reason) => { + const releasePngWorker = retainPngWorker((reason) => { void emitDaemonDiagnostic(logPath, 'png_worker_unavailable', { reason: reason.kind, detail: reason.detail, }).catch(() => {}); }); - const shutdownOnHost = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => { + let registrationFinished = false; + let lockConfigurationCleared = false; + // Work that ignored cancellation may still be running, so this process keeps the registration + // and lock it holds: releasing them would let a second daemon drive the same devices. + let abandonedRequests = 0; + const drainInFlightRequests = async (): Promise => { + const total = inFlightRequests.size; + if (total === 0) return { drained: 0, abandoned: 0 }; + cancelAllRequests(); + let deadlineTimer: ReturnType | undefined; + const outcome = await Promise.race([ + Promise.allSettled(inFlightRequests).then(() => 'drained' as const), + new Promise<'deadline'>((resolve) => { + deadlineTimer = setTimeout(() => resolve('deadline'), requestDrainTimeoutMs); + }), + ]); + clearTimeout(deadlineTimer); + abandonedRequests = outcome === 'drained' ? 0 : inFlightRequests.size; + if (abandonedRequests > 0) { + await emitDaemonDiagnostic(logPath, 'daemon_shutdown_requests_abandoned', { + abandoned: abandonedRequests, + budgetMs: requestDrainTimeoutMs, + }); + } + return { drained: total - abandonedRequests, abandoned: abandonedRequests }; + }; + const closeSteps = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); leaseSessionEnd.cancel(); clearInterval(leaseExpirySweep); - if (shuttingDown) return; shuttingDown = true; stopMetadataLossWatch(); - if (shutdownOptions.cause) { - await emitFatalDiagnostic(shutdownOptions.cause); + if (closeOptions.cause) { + await emitFatalDiagnostic(closeOptions.cause); } - await closeDaemonServers(servers); - await Promise.allSettled(inFlightRequests); + // Cancellation and the listener close run together: a connection held by a request that ignores + // cancellation is force-closed at the same budget the drain gives it. + const serversClosed = closeDaemonServers(servers, requestDrainTimeoutMs); + const requests = await drainInFlightRequests(); + await serversClosed; await leaseSessionEnd.drain(); // Hand healthy runners off before durable session teardown. The lifecycle gateway later // terminates only still-owned generations once all resources have finalized. @@ -923,13 +882,14 @@ async function startDaemonRuntimeOnHost( // no diagnostics scope, so `emitDiagnostic` would drop every detach reason. Daemon debug level is // forced on here because the declines are the point of the record — a handoff that silently // skipped is indistinguishable from a rebuild (#2681). + let handedOffRunners: readonly string[] = []; try { await withDaemonDiagnosticsScope({ logPath }, async () => { - await applicationLifecycle.detachForDaemonShutdown(); + handedOffRunners = await applicationLifecycle.detachForDaemonShutdown(); }); } catch {} expiredProviderLeaseReleaser.beginShutdown(); - await teardownDaemonSessions(); + const sessions = await teardownDaemonSessions(); await withDaemonDiagnosticsScope({ logPath }, async () => { await finalizeDaemonLeases({ leaseRegistry, @@ -961,58 +921,97 @@ async function startDaemonRuntimeOnHost( }); expiredProviderLeaseReleaser.shutdown(); await deviceRuntimeGateway.shutdown(); - await closeProviders('daemon-log'); - await applicationLifecycle.finalizeDaemonShutdown(); + const plugins = await pluginClose.close(); + for (const failure of plugins.failed) { + await emitDaemonDiagnostic(logPath, 'provider_plugin_cleanup_failed', { + provider: failure.provider, + source: failure.source, + [failure.step]: failure.error, + }); + } + const stoppedRunners = await applicationLifecycle.finalizeDaemonShutdown(); // Best effort: stop the PNG worker so an in-flight job cannot delay exit. await Promise.race([ - terminatePngWorker().catch(() => {}), + releasePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); - await finishDaemonRegistration({ - registration, - infoPath, - logPath, - outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, - }); - await deps.ownerLifecycle.clearDaemonLockConfiguration(); - exit(shutdownOptions.exitCode ?? 0); + if (abandonedRequests === 0) { + registrationFinished = true; + await finishDaemonRegistration({ + registration, + infoPath, + logPath, + outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, + }); + lockConfigurationCleared = true; + await deps.ownerLifecycle.clearDaemonLockConfiguration(); + } + const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); + return { + providerReleases: { + completed: providerReleaseDrain.released.map((lease) => lease.leaseId), + pending: providerReleaseDrain.pending + .filter((lease) => !unknownLeaseIds.has(lease.leaseId)) + .map((lease) => lease.leaseId), + unknown: [...unknownLeaseIds], + }, + requests, + runners: { stopped: [...stoppedRunners], handedOff: [...handedOffRunners] }, + sessions, + plugins, + }; }; - // Signal handlers and external callers run outside the startup scope, so shutdown re-enters it. - const shutdown = async (shutdownOptions: { exitCode?: number; cause?: unknown } = {}) => - await withCommandHost( - { host, requestEnv: undefined }, - async () => await shutdownOnHost(shutdownOptions), - ); + // A step that rejects must not strand what the later steps release: the PNG worker hold, the + // provider runtimes, the registration and lock, and the runtime's own state. + const closeOnHost = async (closeOptions: { cause?: unknown }): Promise => { + try { + return await closeSteps(closeOptions); + } finally { + await Promise.allSettled([ + releasePngWorker(), + pluginClose.close(), + registrationFinished || abandonedRequests > 0 + ? undefined + : finishDaemonRegistration({ registration, infoPath, logPath }), + lockConfigurationCleared || abandonedRequests > 0 + ? undefined + : deps.ownerLifecycle.clearDaemonLockConfiguration(), + ]); + runtimeContext.dispose(); + } + }; - startingUp = false; + // Idle reap, external callers and the executable's signal handlers all close through here, and + // none of them runs inside the startup scope, so the close re-enters it. + let closing: Promise | undefined; + let settleClosed: { resolve(report: ShutdownReport): void; reject(error: unknown): void }; + const closed = new Promise((resolve, reject) => { + settleClosed = { resolve, reject }; + }); + // A close nobody awaits (an idle reap) still reports through `closed`, which must not become an + // unhandled rejection when no caller observes it. + closed.catch(() => {}); + const close = (closeOptions: { cause?: unknown } = {}): Promise => { + if (!closing) { + closing = runtimeContext.run(() => + withCommandHost( + { host, requestEnv: undefined }, + async () => await closeOnHost(closeOptions), + ), + ); + closing.then(settleClosed.resolve, settleClosed.reject); + } + return closing; + }; - if (options.registerProcessHandlers !== false) { - process.on('SIGINT', () => { - void shutdown(); - }); - process.on('SIGTERM', () => { - void shutdown(); - }); - process.on('SIGHUP', () => { - void shutdown(); - }); - process.on('uncaughtException', (err) => { - const appErr = err instanceof AppError ? err : asAppError(err); - stderr.write(`Daemon error: ${appErr.message}\n`); - void shutdown({ exitCode: 1, cause: err }); - }); - process.on('unhandledRejection', (reason) => { - const err = reason instanceof Error ? reason : new Error(String(reason)); - const appErr = err instanceof AppError ? err : asAppError(err); - stderr.write(`Daemon error: ${appErr.message}\n`); - void shutdown({ exitCode: 1, cause: err }); - }); - } + startingUp = false; return { httpPort, - shutdown, + close, + closed, + run: runtimeContext.run, socketPort, token, }; diff --git a/packages/daemon/src/server/http/http-server.ts b/packages/daemon/src/server/http/http-server.ts index e5e5d878f6..6259b0df7d 100644 --- a/packages/daemon/src/server/http/http-server.ts +++ b/packages/daemon/src/server/http/http-server.ts @@ -6,6 +6,7 @@ // in `http-rpc-envelope.ts`, the authorization decisions in // `http-authorization.ts`, and the RPC method vocabulary in `http-rpc-methods.ts`. import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { AsyncResource } from 'node:async_hooks'; import http from 'node:http'; import { randomUUID } from 'node:crypto'; import type { DiagnosticsRecordRef, NormalizedError } from '@agent-device/kernel/errors'; @@ -104,7 +105,7 @@ export async function createDaemonHttpServer(options: { resolveRequestDiagnosticsPath, onUnownedRequestFailure, } = options; - return http.createServer((req, res) => { + const onRequest = AsyncResource.bind((req: http.IncomingMessage, res: http.ServerResponse) => { if (req.method === 'GET' && req.url === '/health') { res.statusCode = 200; res.setHeader('content-type', 'application/json'); @@ -212,6 +213,7 @@ export async function createDaemonHttpServer(options: { }); }); }); + return http.createServer(onRequest); } /** Accumulates one `/rpc` body and hands it to the flow once the stream ends. */ @@ -235,9 +237,12 @@ function readRpcRequestBody( } }); - req.on('end', () => { - onBody(body); - }); + req.on( + 'end', + AsyncResource.bind(() => { + onBody(body); + }), + ); } // The `/rpc` flow, in the order it runs: the envelope decides whether this body is a @@ -471,8 +476,9 @@ function beginRpcRequestLifetime(params: { }; }, watchForClientDisconnect() { - req.on('aborted', markCanceledIfResponseIncomplete); - res.on('close', markCanceledIfResponseIncomplete); + const onClientGone = AsyncResource.bind(markCanceledIfResponseIncomplete); + req.on('aborted', onClientGone); + res.on('close', onClientGone); if (req.aborted || res.destroyed) { markCanceledIfResponseIncomplete(); } diff --git a/packages/daemon/src/server/http/server-shutdown.ts b/packages/daemon/src/server/http/server-shutdown.ts index b86f71a670..2f60dd0421 100644 --- a/packages/daemon/src/server/http/server-shutdown.ts +++ b/packages/daemon/src/server/http/server-shutdown.ts @@ -1,6 +1,27 @@ +import type { + PluginCleanupOutcome, + PluginInventoryEntry, +} from '@agent-device/contracts/plugin-inventory'; +import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import type { DaemonServer } from './transport.ts'; +import { sleep } from '@agent-device/host-kit/retry'; +import { createRequestCancelRuntimeState } from '@agent-device/host-kit/request'; +import type { RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; +import type { RuntimeHintValues } from '@agent-device/contracts/application-lifecycle-runtime'; +import { SessionStore } from '../../core/session-store.ts'; +import type { SessionRef, SessionState } from '../../core/session-state.ts'; +import type { PlatformResourceCleanup } from '../../core/platform-resource-cleanup.ts'; +import { createProviderDeviceAdmissionRuntimeState } from '../../core/provider-device-admission.ts'; +import { createInteractorResolutionRuntimeState } from '../../core/interactor-resolution.ts'; +import { createArtifactTrackingRuntimeState } from '../../resources/artifact-tracking.ts'; +import { createResumableUploadRuntimeState } from '../resumable-upload.ts'; +import { createMaterializedPathRuntimeState } from '../../sessions/materialized-path-registry.ts'; +import { stopSessionAppLog, teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; +import { runtimeHintValues } from '../../sessions/session-runtime.ts'; const DAEMON_SHUTDOWN_TIMEOUT_MS = 5_000; +const DAEMON_PLUGIN_SHUTDOWN_TIMEOUT_MS = 5_000; function forceCloseServer(server: DaemonServer): void { server.destroyConnections?.(); @@ -37,3 +58,344 @@ export async function closeDaemonServers( }), ); } + +/** What a closed daemon runtime leaves behind, by kind of resource. */ +export type ShutdownReport = { + /** Provider lease releases, by lease id. `unknown` was still in flight when the budget ran out. */ + providerReleases: { completed: string[]; pending: string[]; unknown: string[] }; + /** Device runners, by device id: `handedOff` was left running for the next daemon. */ + runners: { stopped: string[]; handedOff: string[] }; + /** + * Requests in flight at close. `abandoned` ones ignored cancellation past the drain budget and may + * still be running, so the registration stays held while this process lives. + */ + requests: { drained: number; abandoned: number }; + sessions: { closed: number; failed: number }; + /** `pending` plugins were still shutting down when the budget ran out. */ + plugins: { + disposed: string[]; + failed: { + provider: string; + source?: PluginCleanupOutcome['source']; + step: 'shutdown' | 'dispose'; + error: NormalizedError; + }[]; + pending: string[]; + }; +}; + +/** + * The lines that say what a close left unfinished, for the executable to write where the operator + * looks (stderr is the daemon log for a detached daemon). A clean close yields none. + */ +export function describeUnfinishedShutdownWork(report: ShutdownReport): string[] { + const lines: string[] = []; + if (report.requests.abandoned > 0) { + lines.push( + `Daemon shutdown: ${report.requests.abandoned} request(s) ignored cancellation and may still be running; the daemon registration stays held until this process exits.`, + ); + } + for (const provider of report.plugins.pending) { + lines.push( + `Daemon shutdown: plugin ${provider} was still shutting down when the budget ran out.`, + ); + } + for (const failure of report.plugins.failed) { + lines.push( + `Daemon shutdown: plugin ${failure.provider} failed to ${failure.step === 'shutdown' ? 'shut down' : 'dispose'}: ${failure.error.message}`, + ); + } + const { pending, unknown } = report.providerReleases; + if (pending.length > 0) { + lines.push(`Daemon shutdown: provider lease release still pending for ${pending.join(', ')}.`); + } + if (unknown.length > 0) { + lines.push( + `Daemon shutdown: provider lease release outcome unknown for ${unknown.join(', ')}.`, + ); + } + if (report.sessions.failed > 0) { + lines.push(`Daemon shutdown: ${report.sessions.failed} session(s) failed to tear down.`); + } + return lines; +} + +/** + * Waits for the runtime to close and writes what it left unfinished to `stderr`. Whoever started the + * close (a signal, the idle reap, a quiesce), this is the one place the executable learns of it, and + * the registration may be held on purpose, so a clean close writes nothing and any other says why. + */ +export async function reportDaemonClose(params: { + closed: Promise; + stderr: WritableOutput; +}): Promise<{ failed: boolean }> { + try { + const report = await params.closed; + for (const line of describeUnfinishedShutdownWork(report)) params.stderr.write(`${line}\n`); + return { failed: false }; + } catch (error) { + params.stderr.write(`Daemon shutdown error: ${normalizeError(error).message}\n`); + return { failed: true }; + } +} + +export type PluginClose = Readonly<{ + /** Closes the plugin registry once; later calls return the first call's report. */ + close(): Promise; +}>; + +/** + * The daemon's one plugin cleanup: the registry closes each plugin (runtime shutdown, then dispose) + * and answers with a per-plugin outcome. Bounded, because one plugin that never finishes must not + * hold the whole close: past the budget every active plugin is reported pending while its cleanup + * keeps running. + */ +export function createPluginClose(params: { + close: () => Promise; + inventory: readonly PluginInventoryEntry[]; + timeoutMs?: number; +}): PluginClose { + let closing: Promise | undefined; + const activeProviders = (): string[] => + params.inventory.flatMap((entry) => + entry.state === 'active' && entry.provider !== undefined ? [entry.provider] : [], + ); + return { + close: () => + (closing ??= (async () => { + const outcome = await closeWithinBudget( + params.close, + params.timeoutMs ?? DAEMON_PLUGIN_SHUTDOWN_TIMEOUT_MS, + ); + if (outcome.kind === 'deadline') { + return { disposed: [], failed: [], pending: activeProviders() }; + } + if (outcome.kind === 'rejected') { + const error = normalizeError(outcome.error); + return { + disposed: [], + failed: activeProviders().map((provider) => ({ + provider, + step: 'shutdown' as const, + error, + })), + pending: [], + }; + } + return reportCleanupOutcomes(outcome.outcomes); + })()), + }; +} + +type PluginCloseOutcome = + | { kind: 'closed'; outcomes: readonly PluginCleanupOutcome[] } + | { kind: 'rejected'; error: unknown } + | { kind: 'deadline' }; + +async function closeWithinBudget( + close: () => Promise, + timeoutMs: number, +): Promise { + let deadlineTimer: ReturnType | undefined; + const outcome = await Promise.race([ + close().then( + (outcomes): PluginCloseOutcome => ({ kind: 'closed', outcomes }), + (error: unknown): PluginCloseOutcome => ({ kind: 'rejected', error }), + ), + new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve({ kind: 'deadline' }), timeoutMs); + }), + ]); + clearTimeout(deadlineTimer); + return outcome; +} + +function reportCleanupOutcomes( + outcomes: readonly PluginCleanupOutcome[], +): ShutdownReport['plugins'] { + const report: ShutdownReport['plugins'] = { disposed: [], failed: [], pending: [] }; + for (const { provider, source, shutdown, dispose } of outcomes) { + for (const [step, result] of [ + ['shutdown', shutdown], + ['dispose', dispose], + ] as const) { + if (result.status !== 'failed') continue; + const { code, message, hint } = result.error; + report.failed.push({ + provider, + source, + step, + error: { code, message, ...(hint === undefined ? {} : { hint }) }, + }); + } + if (shutdown.status === 'ok' && dispose.status === 'ok') report.disposed.push(provider); + } + return report; +} + +/** Undoes startup work in reverse order; a step that throws never stops the ones before it. */ +export type StartupRollback = Readonly<{ + onRollback(step: () => Promise | void): void; + run(): Promise; +}>; + +export function createStartupRollback(): StartupRollback { + const steps: Array<() => Promise | void> = []; + return { + onRollback: (step) => { + steps.push(step); + }, + run: async () => { + for (const step of steps.splice(0).reverse()) { + await Promise.resolve() + .then(step) + .catch(() => {}); + } + }, + }; +} + +export type WritableOutput = { + write: (chunk: string) => unknown; +}; + +async function settleDaemonTeardownStep(params: { + session: SessionState; + stderr: WritableOutput; + resource: 'app-log' | 'session' | 'lifecycle'; + teardown: () => Promise; +}): Promise { + const { session, stderr, resource, teardown } = params; + try { + await teardown(); + return true; + } catch (error) { + stderr.write( + `Daemon ${resource} teardown error (${session.name}): ${ + error instanceof Error ? error.message : String(error) + }\n`, + ); + return false; + } +} + +/** + * Daemon-shutdown teardown of one session: bounded resource cleanup (budget + * from {@link resolveDaemonSessionTeardownTimeoutMs}, resolved BEFORE cleanup + * starts since finalizing the recording detaches `session.screenRecording`), then the + * repair-commit finalization and session deletion. Cleanup failures — including + * a recorder that could not be finalized — surface on stderr instead of being + * silently swallowed. + */ +export async function teardownDaemonSessionForShutdown(params: { + ref: SessionRef; + sessionStore: SessionStore; + stderr: WritableOutput; + platformCleanup: PlatformResourceCleanup; + finalizeApplicationLifecycle?: ( + session: SessionState, + runtimeHints: RuntimeHintValues, + ) => Promise; + afterSuccessfulTeardown?: (session: SessionState) => Promise; +}): Promise { + const { + ref, + sessionStore, + stderr, + platformCleanup, + finalizeApplicationLifecycle, + afterSuccessfulTeardown, + } = params; + const current = sessionStore.resolveCurrent(ref); + const session = current ?? ref.session; + const runtimeHints = runtimeHintValues( + current ? sessionStore.getRuntimeHints(ref.address) : undefined, + ); + const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); + // The ownership-fenced app-log side effect must settle while this process + // still owns the daemon lock. It is intentionally outside the generic + // teardown race so lock release and runtime shutdown cannot overtake it. + + const appLogTeardownSucceeded = await settleDaemonTeardownStep({ + session, + stderr, + resource: 'app-log', + teardown: async () => await stopSessionAppLog({ ref, sessionStore }), + }); + const sessionAfterAppLog = sessionStore.resolveCurrent(ref) ?? session; + const teardown = (async () => { + const genericTeardownSucceeded = await settleDaemonTeardownStep({ + session, + stderr, + resource: 'session', + teardown: async () => + await teardownSessionResources({ + appLog: 'already-settled', + ref, + sessionStore, + platformCleanup, + }), + }); + const lifecycleTeardownSucceeded = finalizeApplicationLifecycle + ? await settleDaemonTeardownStep({ + session, + stderr, + resource: 'lifecycle', + teardown: async () => + await finalizeApplicationLifecycle(sessionAfterAppLog, runtimeHints), + }) + : true; + return genericTeardownSucceeded && lifecycleTeardownSucceeded; + })(); + const genericTeardownSucceeded = await Promise.race([ + teardown, + sleep(timeoutMs).then(() => { + stderr.write(`Daemon session teardown timed out (${session.name}).\n`); + return false; + }), + ]); + const teardownSucceeded = appLogTeardownSucceeded && genericTeardownSucceeded; + // ADR 0012 decision 6, R7 + commit semantics (C2/C5a): commit the healed + // `.ad` iff the repair transaction completed, else leave a bounded + // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. + sessionStore.finalizeRepairTeardown(ref); + if (teardownSucceeded) await afterSuccessfulTeardown?.(session); + sessionStore.retire(ref); + return teardownSucceeded; +} + +export type RuntimeContext = Readonly<{ + run(fn: () => T): T; + /** Adds state the composition contributed and binds it for the rest of the start. */ + extend(handles: readonly RuntimeStateHandle[]): void; + dispose(): void; +}>; + +/** The per-runtime state of every module that keeps some, in the order it is entered. */ +function createDaemonRuntimeStates(): RuntimeStateHandle[] { + return [ + createRequestCancelRuntimeState(), + createProviderDeviceAdmissionRuntimeState(), + createInteractorResolutionRuntimeState(), + createArtifactTrackingRuntimeState(), + createResumableUploadRuntimeState(), + createMaterializedPathRuntimeState(), + ]; +} + +/** The context one daemon runtime enters: every module's per-runtime state, composed in order. */ +export function createRuntimeContext(): RuntimeContext { + const states = createDaemonRuntimeStates(); + return { + run: (fn) => states.reduceRight((next, state) => () => state.run(next), fn)(), + extend: (handles) => { + for (const handle of handles) { + states.push(handle); + handle.enter(); + } + }, + dispose: () => { + for (const state of states) state.dispose(); + }, + }; +} diff --git a/packages/daemon/src/server/http/transport.ts b/packages/daemon/src/server/http/transport.ts index c5e22a6329..83a5dbcb0a 100644 --- a/packages/daemon/src/server/http/transport.ts +++ b/packages/daemon/src/server/http/transport.ts @@ -1,5 +1,6 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { AppError, normalizeError, createRequestCanceledError } from '@agent-device/kernel/errors'; +import { AsyncResource } from 'node:async_hooks'; import net from 'node:net'; import type { Server as HttpServer } from 'node:http'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; @@ -27,8 +28,12 @@ export type DaemonServer = (net.Server | HttpServer) & { export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer { const sockets = new Set(); const server: DaemonServer = net.createServer((socket) => { + // An accepted socket loses the runtime that is listening, so its callbacks keep it explicitly. sockets.add(socket); - socket.on('close', () => sockets.delete(socket)); + socket.on( + 'close', + AsyncResource.bind(() => sockets.delete(socket)), + ); let buffer = ''; let inFlightRequests = 0; const activeRequestIds = new Set(); @@ -52,9 +57,9 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer }); }; socket.setEncoding('utf8'); - socket.on('close', cancelInFlightRequests); - socket.on('error', cancelInFlightRequests); - socket.on('data', async (chunk) => { + socket.on('close', AsyncResource.bind(cancelInFlightRequests)); + socket.on('error', AsyncResource.bind(cancelInFlightRequests)); + const onData = AsyncResource.bind(async (chunk: string) => { const parsed = consumeTextLines(buffer, chunk); buffer = parsed.buffer; for (const line of parsed.lines) { @@ -104,6 +109,7 @@ export function createSocketServer(handleRequest: DaemonInvokeFn): DaemonServer } } }); + socket.on('data', onData); }); server.destroyConnections = () => { for (const socket of sockets) { diff --git a/packages/daemon/src/server/provider-lease-expiry.ts b/packages/daemon/src/server/provider-lease-expiry.ts index 038ef2b07d..349248ee1e 100644 --- a/packages/daemon/src/server/provider-lease-expiry.ts +++ b/packages/daemon/src/server/provider-lease-expiry.ts @@ -33,7 +33,15 @@ export type ExpiredProviderLeaseReleaser = { beginShutdown: () => void; release: (lease: DeviceLease) => Promise; retryPending: () => Promise; - drain: (timeoutMs: number) => Promise<{ pending: DeviceLease[]; released: DeviceLease[] }>; + /** + * `pending` holds every lease still unreleased; `unknown` is the part of it whose release was + * still in flight when the budget ran out, so its outcome is not known. + */ + drain: (timeoutMs: number) => Promise<{ + pending: DeviceLease[]; + unknown: DeviceLease[]; + released: DeviceLease[]; + }>; shutdown: () => void; }; @@ -206,6 +214,14 @@ export function createExpiredProviderLeaseReleaser(options: { await Promise.race([retryPending(), sleep(Math.max(0, timeoutMs))]); const result = { pending: [...pendingLiveLeases.values(), ...pendingRecoveryLeases.values()], + unknown: [ + ...[...pendingLiveLeases.values()].filter((lease) => + activeReleaseAttempts.has(`live:${lease.leaseId}`), + ), + ...[...pendingRecoveryLeases.values()].filter((lease) => + activeReleaseAttempts.has(`recovery:${lease.leaseId}`), + ), + ], released: [...shutdownReleasedLeases.values()], }; trackingShutdownReleases = false; diff --git a/packages/daemon/src/server/resumable-upload.ts b/packages/daemon/src/server/resumable-upload.ts index 74a86bd2ba..cb061c2be7 100644 --- a/packages/daemon/src/server/resumable-upload.ts +++ b/packages/daemon/src/server/resumable-upload.ts @@ -1,3 +1,4 @@ +import { defineRuntimeSlots, type RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; @@ -20,8 +21,34 @@ const RESUMABLE_UPLOAD_RESOURCE: TenantOwnedResourceKind = { label: 'Upload', expiredHint: `Resumable uploads expire ${RESUMABLE_UPLOAD_CLEANUP_TIMEOUT_MS / 60_000} minutes after the last received chunk. Start the upload again from the beginning.`, }; -const RESUMABLE_UPLOADS_BY_ID = new Map(); -const RESUMABLE_UPLOADS_BY_KEY = new Map(); +type ResumableUploadState = { + byId: Map; + byKey: Map; +}; + +const resumableUploadRuntime = defineRuntimeSlots(); +const RESUMABLE_UPLOAD_SLOT = Symbol('resumableUploads'); +const resumableUploads = (): ResumableUploadState => + resumableUploadRuntime.slot( + RESUMABLE_UPLOAD_SLOT, + (): ResumableUploadState => ({ byId: new Map(), byKey: new Map() }), + (state) => { + for (const entry of state.byId.values()) { + clearTimeout(entry.timer); + try { + fs.rmSync(entry.tempDir, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } + } + }, + ); +const RESUMABLE_UPLOADS_BY_ID = () => resumableUploads().byId; +const RESUMABLE_UPLOADS_BY_KEY = () => resumableUploads().byKey; + +/** The resumable uploads one daemon runtime holds; disposing it stops their expiry timers. */ +export const createResumableUploadRuntimeState = (): RuntimeStateHandle => + resumableUploadRuntime.createHandle(); export type BeginResumableUploadOptions = { baseUrl: string; @@ -62,8 +89,8 @@ export function beginResumableUpload(options: BeginResumableUploadOptions): { } { validateResumableUploadOptions(options); const key = buildResumableUploadKey(options); - const existingId = RESUMABLE_UPLOADS_BY_KEY.get(key); - const existing = existingId ? RESUMABLE_UPLOADS_BY_ID.get(existingId) : undefined; + const existingId = RESUMABLE_UPLOADS_BY_KEY().get(key); + const existing = existingId ? RESUMABLE_UPLOADS_BY_ID().get(existingId) : undefined; const entry = existing ?? createResumableUploadEntry(options, key); return { uploadId: entry.id, @@ -154,8 +181,8 @@ function createResumableUploadEntry( closed: false, finalized: false, }; - RESUMABLE_UPLOADS_BY_ID.set(id, entry); - RESUMABLE_UPLOADS_BY_KEY.set(key, id); + RESUMABLE_UPLOADS_BY_ID().set(id, entry); + RESUMABLE_UPLOADS_BY_KEY().set(key, id); refreshResumableUploadTimer(entry); return entry; } @@ -258,8 +285,8 @@ async function terminateEntry(entry: ResumableUploadEntry): Promise { function closeEntry(entry: ResumableUploadEntry): void { entry.closed = true; if (entry.timer) clearTimeout(entry.timer); - RESUMABLE_UPLOADS_BY_ID.delete(entry.id); - RESUMABLE_UPLOADS_BY_KEY.delete(entry.key); + RESUMABLE_UPLOADS_BY_ID().delete(entry.id); + RESUMABLE_UPLOADS_BY_KEY().delete(entry.key); } function requireResumableUpload( @@ -267,7 +294,7 @@ function requireResumableUpload( tenantId: string | undefined, ): ResumableUploadEntry { return requireTenantOwnedEntry( - RESUMABLE_UPLOADS_BY_ID, + RESUMABLE_UPLOADS_BY_ID(), uploadId, tenantId, RESUMABLE_UPLOAD_RESOURCE, diff --git a/packages/daemon/src/sessions/materialized-path-registry.ts b/packages/daemon/src/sessions/materialized-path-registry.ts index edc4d9e207..a5f6d1e459 100644 --- a/packages/daemon/src/sessions/materialized-path-registry.ts +++ b/packages/daemon/src/sessions/materialized-path-registry.ts @@ -1,4 +1,6 @@ +import { defineRuntimeSlots, type RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; import crypto from 'node:crypto'; +import { rmSync } from 'node:fs'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -33,7 +35,27 @@ export type RetainedMaterializedPaths = { expiresAt: string; }; -const retainedPaths = new Map(); +const retainedPathsRuntime = defineRuntimeSlots(); +const RETAINED_PATHS_SLOT = Symbol('materializedPaths'); +const retainedPaths = (): Map => + retainedPathsRuntime.slot( + RETAINED_PATHS_SLOT, + () => new Map(), + (entries) => { + for (const entry of entries.values()) { + clearTimeout(entry.timer); + try { + rmSync(entry.rootPath, { recursive: true, force: true }); + } catch { + // best-effort cleanup only + } + } + }, + ); + +/** The materialized paths one daemon runtime retains; disposing it stops their expiry timers. */ +export const createMaterializedPathRuntimeState = (): RuntimeStateHandle => + retainedPathsRuntime.createHandle(); export async function retainMaterializedPaths(params: { installablePath: string; @@ -57,7 +79,7 @@ export async function retainMaterializedPaths(params: { const timer = setTimeout(() => { void expireRetainedMaterializedPaths(materializationId); }, ttlMs); - retainedPaths.set(materializationId, { + retainedPaths().set(materializationId, { rootPath, installablePath: retainedInstallablePath, archivePath: retainedArchivePath, @@ -83,7 +105,7 @@ export async function cleanupRetainedMaterializedPaths( tenantId?: string, ): Promise { const entry = requireTenantOwnedEntry( - retainedPaths, + retainedPaths(), materializationId, tenantId, MATERIALIZED_PATHS_RESOURCE, @@ -94,7 +116,7 @@ export async function cleanupRetainedMaterializedPaths( export async function cleanupRetainedMaterializedPathsForSession( sessionName: string, ): Promise { - const matchingEntries = Array.from(retainedPaths.entries()).filter( + const matchingEntries = Array.from(retainedPaths().entries()).filter( ([, entry]) => entry.sessionName === sessionName, ); await Promise.all( @@ -107,7 +129,7 @@ export async function cleanupRetainedMaterializedPathsForSession( // Daemon-internal expiry: skips the tenant check (the timer has no tenant // context) and must never reject — the caller is a bare setTimeout. async function expireRetainedMaterializedPaths(materializationId: string): Promise { - const entry = retainedPaths.get(materializationId); + const entry = retainedPaths().get(materializationId); if (!entry) return; try { await removeRetainedEntry(materializationId, entry); @@ -118,7 +140,7 @@ async function expireRetainedMaterializedPaths(materializationId: string): Promi async function removeRetainedEntry(materializationId: string, entry: RetainedEntry): Promise { clearTimeout(entry.timer); - retainedPaths.delete(materializationId); + retainedPaths().delete(materializationId); await fs.rm(entry.rootPath, { recursive: true, force: true }); } diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts b/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts index 56e366da14..0a91eb6e0a 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/session-open.ts @@ -67,7 +67,17 @@ export type SessionOpenCommandInput = Readonly<{ reconcileOrphanedDeviceClaim: DeviceClaimReconciler; }>; -const firstSessionOpenLocks = new Map>(); +// One lock table per session store, which is to say per daemon runtime. +const firstSessionOpenLocksByStore = new WeakMap>>(); + +function firstSessionOpenLocks(sessionStore: SessionStore): Map> { + let locks = firstSessionOpenLocksByStore.get(sessionStore); + if (!locks) { + locks = new Map(); + firstSessionOpenLocksByStore.set(sessionStore, locks); + } + return locks; +} type OpenRuntimeAdmission = | Readonly<{ @@ -347,7 +357,7 @@ async function handleOpenCommand(params: SessionOpenCommandInput): Promise await openNewSessionWithDeviceClaim({ diff --git a/packages/host-kit/package.json b/packages/host-kit/package.json index 8e1f564f7b..f9a4d41030 100644 --- a/packages/host-kit/package.json +++ b/packages/host-kit/package.json @@ -72,6 +72,10 @@ "types": "./src/request.ts", "default": "./src/request.ts" }, + "./runtime-slots": { + "types": "./src/runtime-slots.ts", + "default": "./src/runtime-slots.ts" + }, "./retry": { "types": "./src/retry.ts", "default": "./src/retry.ts" diff --git a/packages/host-kit/src/internal/request-cancel.ts b/packages/host-kit/src/internal/request-cancel.ts index 6d0b65516c..54bf01a9a0 100644 --- a/packages/host-kit/src/internal/request-cancel.ts +++ b/packages/host-kit/src/internal/request-cancel.ts @@ -1,7 +1,29 @@ import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; +import { defineRuntimeSlots, type RuntimeStateHandle } from '../runtime-slots.ts'; -const canceledRequestIds = new Set(); -const requestAbortControllers = new Map(); +type RequestCancelState = { + canceledRequestIds: Set; + requestAbortControllers: Map; +}; + +const requestCancelRuntime = defineRuntimeSlots(); +const REQUEST_CANCEL_SLOT = Symbol('requestCancel'); +const requestCancelState = (): RequestCancelState => + requestCancelRuntime.slot( + REQUEST_CANCEL_SLOT, + (): RequestCancelState => ({ + canceledRequestIds: new Set(), + requestAbortControllers: new Map(), + }), + (state) => { + for (const controller of state.requestAbortControllers.values()) controller.abort(); + }, + ); +const canceledRequestIds = () => requestCancelState().canceledRequestIds; +const requestAbortControllers = () => requestCancelState().requestAbortControllers; + +export const createRequestCancelRuntimeState = (): RuntimeStateHandle => + requestCancelRuntime.createHandle(); export type RequestAbortRegistration = { requestId: string; @@ -55,16 +77,16 @@ export function registerRequestAbort( requestId: string | undefined, ): RequestAbortRegistration | undefined { if (!requestId) return undefined; - if (requestAbortControllers.has(requestId)) { + if (requestAbortControllers().has(requestId)) { throw new AppError('INVALID_ARGS', `Request ID is already in flight: ${requestId}`, { requestId, reason: 'duplicate_request_id', }); } - evictOldestEntries(requestAbortControllers); + evictOldestEntries(requestAbortControllers()); const controller = new AbortController(); - requestAbortControllers.set(requestId, controller); - if (canceledRequestIds.has(requestId)) { + requestAbortControllers().set(requestId, controller); + if (canceledRequestIds().has(requestId)) { controller.abort(); } return { requestId, controller }; @@ -72,13 +94,18 @@ export function registerRequestAbort( export function markRequestCanceled(requestId: string | undefined): void { if (!requestId) return; - evictOldestSetEntries(canceledRequestIds); - canceledRequestIds.add(requestId); + evictOldestSetEntries(canceledRequestIds()); + canceledRequestIds().add(requestId); // Abort WITH the typed error as the reason: every `signal.throwIfAborted()`, // fetch, and `throw signal.reason` downstream then surfaces the canceled // request as such, instead of a bare DOMException that normalizes to UNKNOWN. // No call site has to remember the factory — the signal carries it. - requestAbortControllers.get(requestId)?.abort(createRequestCanceledError()); + requestAbortControllers().get(requestId)?.abort(createRequestCanceledError()); +} + +/** Aborts every request this runtime has in flight, as a canceled request. */ +export function cancelAllRequests(): void { + for (const requestId of requestAbortControllers().keys()) markRequestCanceled(requestId); } export function clearRequestCanceled( @@ -89,12 +116,12 @@ export function clearRequestCanceled( if ( registration && (registration.requestId !== requestId || - requestAbortControllers.get(requestId) !== registration.controller) + requestAbortControllers().get(requestId) !== registration.controller) ) { return; } - canceledRequestIds.delete(requestId); - requestAbortControllers.delete(requestId); + canceledRequestIds().delete(requestId); + requestAbortControllers().delete(requestId); } export function clearRequestAbortRegistration( @@ -106,12 +133,12 @@ export function clearRequestAbortRegistration( export function isRequestCanceled(requestId: string | undefined): boolean { if (!requestId) return false; - return canceledRequestIds.has(requestId); + return canceledRequestIds().has(requestId); } export function getRequestSignal(requestId: string | undefined): AbortSignal | undefined { if (!requestId) return undefined; - return requestAbortControllers.get(requestId)?.signal; + return requestAbortControllers().get(requestId)?.signal; } export function throwIfRequestCanceled(requestId: string | undefined): void { diff --git a/packages/host-kit/src/request.ts b/packages/host-kit/src/request.ts index ee98a01e7b..8d22ba01b6 100644 --- a/packages/host-kit/src/request.ts +++ b/packages/host-kit/src/request.ts @@ -1,5 +1,7 @@ export { + cancelAllRequests, clearRequestAbortRegistration, + createRequestCancelRuntimeState, clearRequestCanceled, getRequestSignal, isRequestCanceled, diff --git a/packages/host-kit/src/runtime-slots.ts b/packages/host-kit/src/runtime-slots.ts new file mode 100644 index 0000000000..ec91863f07 --- /dev/null +++ b/packages/host-kit/src/runtime-slots.ts @@ -0,0 +1,48 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; + +/** + * What one daemon runtime owns of a kind: a module (or a package) keeps its state in slots of an + * async-local store the runtime enters, so two runtimes in one process never share it. Outside a + * runtime (a client process, a unit test) the store's own standalone state answers. + */ +export type RuntimeStateHandle = Readonly<{ + /** Runs `fn` as the runtime that owns this state. */ + run(fn: () => T): T; + /** Binds this state for the rest of the current async context, after the runtime is composed. */ + enter(): void; + /** Releases what the state holds: each slot's disposer runs, then the slots are forgotten. */ + dispose(): void; +}>; + +export type RuntimeSlots = Readonly<{ + /** One piece of state the current runtime owns, created on first use. */ + slot(key: symbol, create: () => T, dispose?: (value: T) => void): T; + /** The state of one new runtime: enter it before anything reads a slot. */ + createHandle(): RuntimeStateHandle; +}>; + +type Slot = { value: unknown; dispose: ((value: never) => void) | undefined }; + +/** The single owner of the "entered store, else standalone store" pattern; callers declare slots. */ +export function defineRuntimeSlots(): RuntimeSlots { + const storage = new AsyncLocalStorage>(); + const standalone = new Map(); + return { + slot: (key, create, dispose) => { + const slots = storage.getStore() ?? standalone; + if (!slots.has(key)) slots.set(key, { value: create(), dispose }); + return slots.get(key)!.value as never; + }, + createHandle: () => { + const owned = new Map(); + return { + run: (fn) => storage.run(owned, fn), + enter: () => storage.enterWith(owned), + dispose: () => { + for (const { value, dispose } of owned.values()) dispose?.(value as never); + owned.clear(); + }, + }; + }, + }; +} diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-runtime-isolation.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-runtime-isolation.test.ts new file mode 100644 index 0000000000..874720caa3 --- /dev/null +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-runtime-isolation.test.ts @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAndroidHelperRuntimeState } from '../adb-host.ts'; +import { recoverAndroidSnapshotHelperRetirement } from '../snapshot-helper-retirement.ts'; +import { + resetAndroidSnapshotHelperSessions, + stopAndroidSnapshotHelperSession, +} from '../snapshot-helper-session-lifecycle.ts'; +import { captureAndroidSnapshotWithHelperSession } from '../snapshot-helper-session.ts'; +import { + createSessionProvider, + type SessionProviderOptions, +} from './snapshot-helper-session.fixtures.ts'; + +// Two daemon runtimes in one process each own their helper sessions, failed starts and pending +// retirements; closing one stops its helpers and leaves the other's warm. + +function runtimeOnDevice(deviceKey: string, options: Partial = {}) { + const calls: (readonly string[])[] = []; + const spawnArgs: (readonly string[])[] = []; + const provider = createSessionProvider({ calls, spawnArgs, ...options }); + const state = createAndroidHelperRuntimeState(); + return { + calls, + spawnArgs, + provider, + state, + capture: async () => + await state.run( + async () => + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey, + }), + ), + }; +} + +test("closing one runtime's helpers leaves another runtime's warm session in use", async () => { + const a = runtimeOnDevice('android:emulator-5554'); + const b = runtimeOnDevice('android:emulator-5556'); + await a.capture(); + await b.capture(); + + await a.state.run(async () => await resetAndroidSnapshotHelperSessions()); + + const survivor = await b.capture(); + assert.equal(survivor?.metadata.sessionReused, true, "B's helper session was not stopped"); + assert.equal(b.spawnArgs.length, 1, 'B did not have to start another helper'); + const restarted = await a.capture(); + assert.equal(restarted?.metadata.sessionReused, false, "A's own session was stopped"); + assert.equal(a.spawnArgs.length, 2); +}); + +test("a runtime's pending retirement survives another runtime's close", async () => { + const deviceKey = 'android:emulator-5554'; + const a = runtimeOnDevice(deviceKey, { + quitResponseMode: 'malformed', + runtimeRelease: 'occupied', + }); + const b = runtimeOnDevice('android:emulator-5556'); + await a.capture(); + // A stops one device's helper and the device cannot confirm the release, so A holds a retirement. + await a.state.run(async () => await stopAndroidSnapshotHelperSession(deviceKey)); + const stopsBefore = forceStops(a.calls); + + // The other runtime closing must not clear what A holds pending for its device. + await b.state.run(async () => await resetAndroidSnapshotHelperSessions()); + await assert.rejects( + a.state.run( + async () => await recoverAndroidSnapshotHelperRetirement({ deviceKey, adb: a.provider.exec }), + ), + /still holding device automation ownership/, + ); + + assert.ok(forceStops(a.calls) > stopsBefore, 'A still settles the release it left pending'); +}); + +function forceStops(calls: readonly (readonly string[])[]): number { + return calls.filter( + (args) => args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper', + ).length; +} diff --git a/packages/platform-android/src/adb-host.ts b/packages/platform-android/src/adb-host.ts index dae11f25dc..05986e0980 100644 --- a/packages/platform-android/src/adb-host.ts +++ b/packages/platform-android/src/adb-host.ts @@ -1,4 +1,5 @@ import { AsyncLocalStorage } from 'node:async_hooks'; +import { defineRuntimeSlots } from '@agent-device/host-kit/runtime-slots'; import type { AndroidHelperInstallDecision, AndroidImeHelperArtifact } from './helper-artifacts.ts'; import type { AndroidAdbInvocation, @@ -115,6 +116,19 @@ export function bindAndroidAdbHost(host: AndroidAdbHost): void { boundHost = host; } +/** + * What one daemon runtime owns of the Android helper: warm sessions, failed starts, pending + * retirements and shell-protocol probes, which the helper modules keep in {@link helperRuntimeSlot}s. + * A close tears down only the runtime that closes; the helper itself is stopped by the close-time + * reset, which reads the same slots. + */ +const helperRuntime = defineRuntimeSlots(); + +export const createAndroidHelperRuntimeState = helperRuntime.createHandle; + +/** One piece of helper state this daemon runtime owns, created on first use. */ +export const helperRuntimeSlot = helperRuntime.slot; + export function requireAndroidAdbHost(): AndroidAdbHost { if (!boundHost) { throw new Error( diff --git a/packages/platform-android/src/adb-shell-protocol.ts b/packages/platform-android/src/adb-shell-protocol.ts index 38d4568e8a..d0ac4964f2 100644 --- a/packages/platform-android/src/adb-shell-protocol.ts +++ b/packages/platform-android/src/adb-shell-protocol.ts @@ -1,3 +1,4 @@ +import { helperRuntimeSlot } from './adb-host.ts'; import type { AndroidAdbExecutor } from './adb-executor.ts'; /** @@ -8,7 +9,9 @@ import type { AndroidAdbExecutor } from './adb-executor.ts'; const SHELL_PROTOCOL_V2_FEATURE = 'shell_v2'; const FEATURE_PROBE_TIMEOUT_MS = 2_000; -const deviceExitStatusForwarding = new Map(); +const deviceExitStatusForwardingSlot = Symbol('androidDeviceExitStatusForwarding'); +const deviceExitStatusForwarding = (): Map => + helperRuntimeSlot(deviceExitStatusForwardingSlot, () => new Map()); /** * Whether the exit status of a host `adb shell` child is the DEVICE command's exit status. @@ -26,7 +29,7 @@ export async function androidAdbForwardsDeviceExitStatus(params: { deviceKey: string; signal?: AbortSignal; }): Promise { - const cached = deviceExitStatusForwarding.get(params.deviceKey); + const cached = deviceExitStatusForwarding().get(params.deviceKey); if (cached !== undefined) return cached; let stdout: string; try { @@ -43,12 +46,12 @@ export async function androidAdbForwardsDeviceExitStatus(params: { return false; } const forwards = parseAndroidAdbFeatures(stdout).includes(SHELL_PROTOCOL_V2_FEATURE); - deviceExitStatusForwarding.set(params.deviceKey, forwards); + deviceExitStatusForwarding().set(params.deviceKey, forwards); return forwards; } export function resetAndroidAdbShellProtocolProbes(): void { - deviceExitStatusForwarding.clear(); + deviceExitStatusForwarding().clear(); } function parseAndroidAdbFeatures(stdout: string): string[] { diff --git a/packages/platform-android/src/snapshot-helper-retirement.ts b/packages/platform-android/src/snapshot-helper-retirement.ts index 7985922a88..7c9200a220 100644 --- a/packages/platform-android/src/snapshot-helper-retirement.ts +++ b/packages/platform-android/src/snapshot-helper-retirement.ts @@ -1,3 +1,4 @@ +import { helperRuntimeSlot } from './adb-host.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { shellFragment } from '@agent-device/kernel/device-shell'; @@ -37,7 +38,9 @@ type PendingRetirement = { cause: string; }; -const pendingRetirements = new Map(); +const pendingRetirementsSlot = Symbol('androidPendingRetirements'); +const pendingRetirements = (): Map => + helperRuntimeSlot(pendingRetirementsSlot, () => new Map()); export function getAndroidSnapshotHelperSessionDeviceKey( device: Pick, @@ -76,7 +79,7 @@ export async function recoverAndroidSnapshotHelperRetirement(params: { adb: AndroidAdbExecutor; signal?: AbortSignal; }): Promise { - const retirement = pendingRetirements.get(params.deviceKey); + const retirement = pendingRetirements().get(params.deviceKey); if (!retirement) return; await stopAndroidSnapshotHelperRuntime({ adb: params.adb, @@ -109,7 +112,7 @@ export async function recoverAndroidSnapshotHelperRetirement(params: { // A device that could not be read leaves the retirement pending: the next acquire asks again, and // this command answers for its own transport instead of for ownership. if (release === 'unknown') return; - pendingRetirements.delete(params.deviceKey); + pendingRetirements().delete(params.deviceKey); } /** @@ -132,11 +135,11 @@ export async function recordAndroidSnapshotHelperRelease(params: { packageName: params.packageName, })); if (release === 'released') { - pendingRetirements.delete(params.deviceKey); + pendingRetirements().delete(params.deviceKey); return release; } const causeMessage = params.cause instanceof Error ? params.cause.message : String(params.cause); - pendingRetirements.set(params.deviceKey, { + pendingRetirements().set(params.deviceKey, { packageName: params.packageName, cause: causeMessage, }); @@ -334,11 +337,11 @@ export async function stopAndroidSnapshotHelperHostProcess(params: { * process that may not be the only one, and the pending entry has to stay. */ export function settleAndroidSnapshotHelperRetirement(deviceKey: string): void { - pendingRetirements.delete(deviceKey); + pendingRetirements().delete(deviceKey); } export function resetAndroidSnapshotHelperRetirements(): void { - pendingRetirements.clear(); + pendingRetirements().clear(); } /** diff --git a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts index 0a85984bca..7a5c501000 100644 --- a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts +++ b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts @@ -7,7 +7,7 @@ * `snapshot-helper-session.ts`; they acquire through here and never reach the registry themselves. */ import type { AndroidAdbProcess } from './adb-executor.ts'; -import { requireAndroidAdbHost } from './adb-host.ts'; +import { helperRuntimeSlot, requireAndroidAdbHost } from './adb-host.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; @@ -94,12 +94,16 @@ export type AndroidSnapshotHelperSessionAcquisition = { deviceKey: string; }; -const sessions = new Map(); +const sessionsSlot = Symbol('androidSnapshotHelperSessions'); +const sessions = (): Map => + helperRuntimeSlot(sessionsSlot, () => new Map()); /** * Capture identity → when this process may spawn that helper build again after a failed start or a * failed session capture. */ -const failedSessions = new Map(); +const failedSessionsSlot = Symbol('androidSnapshotHelperFailedSessions'); +const failedSessions = (): Map => + helperRuntimeSlot(failedSessionsSlot, () => new Map()); /** * Starts (or reuses) the session without capturing, so a helper-backed read that is not a snapshot @@ -145,7 +149,7 @@ export async function acquireAndroidSnapshotHelperSession( export function getLiveAndroidSnapshotHelperSession( deviceKey: string, ): AndroidSnapshotHelperSession | undefined { - return sessions.get(deviceKey); + return sessions().get(deviceKey); } async function resolveAndroidSnapshotHelperSession(params: { @@ -157,7 +161,7 @@ async function resolveAndroidSnapshotHelperSession(params: { }): Promise { if (isAndroidSnapshotHelperSessionBackedOff(params.identity)) return undefined; await retireUnusableAndroidSnapshotHelperSession(params.deviceKey, params.identity); - return sessions.get(params.deviceKey) ?? (await tryStartAndroidSnapshotHelperSession(params)); + return sessions().get(params.deviceKey) ?? (await tryStartAndroidSnapshotHelperSession(params)); } /** Drops a cached session this command cannot write to, so only its forward is left behind. */ @@ -165,7 +169,7 @@ async function retireUnusableAndroidSnapshotHelperSession( deviceKey: string, identity: string, ): Promise { - const cached = sessions.get(deviceKey); + const cached = sessions().get(deviceKey); if (!cached || isReusableAndroidSnapshotHelperSession(cached, identity)) return; // A process that already exited cannot answer the forwarded port, so there is nothing left to ask // it to quit gracefully. @@ -191,7 +195,7 @@ async function tryStartAndroidSnapshotHelperSession(params: { return await startAndroidSnapshotHelperSession(params); } catch (error) { params.options.signal?.throwIfAborted(); - failedSessions.set( + failedSessions().set( params.identity, Date.now() + androidSnapshotHelperSessionRetryAfterMs(Date.now() - startedAtMs), ); @@ -210,10 +214,10 @@ async function tryStartAndroidSnapshotHelperSession(params: { /** A helper build whose last start or capture failed is left alone until its backoff has run out. */ function isAndroidSnapshotHelperSessionBackedOff(identity: string): boolean { - const retryAtMs = failedSessions.get(identity); + const retryAtMs = failedSessions().get(identity); if (retryAtMs === undefined) return false; if (retryAtMs > Date.now()) return true; - failedSessions.delete(identity); + failedSessions().delete(identity); return false; } @@ -250,7 +254,7 @@ export async function retireFailedAndroidSnapshotHelperSession(params: { cause: params.cause, }); if (params.signal?.aborted) return; - failedSessions.set( + failedSessions().set( params.identity, Date.now() + params.fallbackBudgetMs + @@ -318,8 +322,8 @@ async function startAndroidSnapshotHelperSession(params: { params.startBudgetMs, params.options.signal, ); - sessions.set(params.deviceKey, session); - failedSessions.delete(params.identity); + sessions().set(params.deviceKey, session); + failedSessions().delete(params.identity); // `am instrument` force-stops whatever is already instrumenting this package, so a helper that // reported itself ready is the only helper process the device has left, and the release the // previous teardown could not prove went away with the process that owed it. Leaving the entry @@ -448,9 +452,9 @@ export async function stopAndroidSnapshotHelperSession( cause?: unknown; } = {}, ): Promise { - const session = sessions.get(deviceKey); + const session = sessions().get(deviceKey); if (!session) return false; - sessions.delete(deviceKey); + sessions().delete(deviceKey); const processExit = observeAndroidSnapshotHelperProcessExit(session.process); const force = options.force === true || options.signal?.aborted === true; const graceful = await requestGracefulSessionExit(session, processExit, force, options.signal); @@ -564,15 +568,15 @@ export async function stopAndroidSnapshotHelperSessionForDevice( export async function resetAndroidSnapshotHelperSessions(): Promise { try { await Promise.allSettled( - [...sessions.keys()].map(async (deviceKey) => { + [...sessions().keys()].map(async (deviceKey) => { await stopAndroidSnapshotHelperSession(deviceKey); }), ); } finally { // One teardown that throws must not leave the next caller believing a session, a pending // retirement, or a failed start is still standing. - sessions.clear(); - failedSessions.clear(); + sessions().clear(); + failedSessions().clear(); resetAndroidSnapshotHelperRetirements(); resetAndroidAdbShellProtocolProbes(); } diff --git a/packages/platform-apple/src/core/runner-host.ts b/packages/platform-apple/src/core/runner-host.ts index c6a5534137..349d85ec19 100644 --- a/packages/platform-apple/src/core/runner-host.ts +++ b/packages/platform-apple/src/core/runner-host.ts @@ -43,6 +43,7 @@ import { visitXmlPlistEntries } from './plist-xml.ts'; import { getRunnerDeviceClaimAuthorityProbe, getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, } from './runner-owner-state.ts'; import { buildSimctlArgsForDevice, simulatorAddressFor } from './simctl.ts'; import { @@ -110,5 +111,6 @@ export const appleRunnerHost: AppleRunnerHost = { resolveIosPhysicalDeviceControl, visitXmlPlistEntries, leaseOwnerStateDir: getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, hasDeviceClaimAuthority: (device) => getRunnerDeviceClaimAuthorityProbe()?.(device) ?? false, }; diff --git a/packages/platform-apple/src/core/runner-owner-state.test.ts b/packages/platform-apple/src/core/runner-owner-state.test.ts new file mode 100644 index 0000000000..fa4de52b7a --- /dev/null +++ b/packages/platform-apple/src/core/runner-owner-state.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from 'vitest'; +import { + createAppleRunnerRuntimeState, + getRunnerLeaseOwnerStateDir, + runnerRuntimeSlot, + setRunnerLeaseOwnerStateDir, +} from './runner-owner-state.ts'; + +const SLOT = Symbol('test-slot'); + +test('each runtime keeps its own owner state and registries', () => { + const first = createAppleRunnerRuntimeState(); + const second = createAppleRunnerRuntimeState(); + + first.run(() => { + setRunnerLeaseOwnerStateDir(' /state/first '); + runnerRuntimeSlot(SLOT, () => new Map()).set('device', 'first'); + }); + second.run(() => { + setRunnerLeaseOwnerStateDir('/state/second'); + }); + + expect(first.run(() => getRunnerLeaseOwnerStateDir())).toBe('/state/first'); + expect(second.run(() => getRunnerLeaseOwnerStateDir())).toBe('/state/second'); + expect( + first.run(() => runnerRuntimeSlot(SLOT, () => new Map()).get('device')), + ).toBe('first'); + expect(second.run(() => runnerRuntimeSlot(SLOT, () => new Map()).size)).toBe(0); + // Nothing leaked into the standalone state the code outside any runtime reads. + expect(getRunnerLeaseOwnerStateDir()).toBeUndefined(); +}); + +test('disposing a runtime drops its registries', () => { + const runtime = createAppleRunnerRuntimeState(); + runtime.run(() => runnerRuntimeSlot(SLOT, () => new Map([['device', 'owned']]))); + + runtime.dispose(); + + expect(runtime.run(() => runnerRuntimeSlot(SLOT, () => new Map()).size)).toBe(0); +}); diff --git a/packages/platform-apple/src/core/runner-owner-state.ts b/packages/platform-apple/src/core/runner-owner-state.ts index e44dddd94e..d94303d549 100644 --- a/packages/platform-apple/src/core/runner-owner-state.ts +++ b/packages/platform-apple/src/core/runner-owner-state.ts @@ -1,20 +1,5 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; - -/** - * Daemon-owned lease-owner state directory for the Apple runner. Kept in a - * dedicated tiny module so daemon startup can set it without loading the - * runner client's implementation closure; the runner package reads it through - * its host port at lease time. - */ -let runnerLeaseOwnerStateDir: string | undefined; - -export function setRunnerLeaseOwnerStateDir(stateDir: string | undefined): void { - runnerLeaseOwnerStateDir = stateDir?.trim() || undefined; -} - -export function getRunnerLeaseOwnerStateDir(): string | undefined { - return runnerLeaseOwnerStateDir; -} +import { defineRuntimeSlots, type RuntimeStateHandle } from '@agent-device/host-kit/runtime-slots'; /** * Daemon-owned device-claim arbitration probe: does the embedding process hold @@ -26,14 +11,45 @@ export function getRunnerLeaseOwnerStateDir(): string | undefined { */ export type RunnerDeviceClaimAuthorityProbe = (device: DeviceInfo) => boolean; -let runnerDeviceClaimAuthorityProbe: RunnerDeviceClaimAuthorityProbe | undefined; +/** + * What one daemon runtime owns of the Apple runner: the lease-owner state directory and claim probe + * its daemon set, and the registries (sessions, locks, recycle ledger) the runner modules keep in + * {@link runnerRuntimeSlot}s. Kept in a dedicated tiny module so daemon startup can reach it + * without loading the runner client's implementation closure; the runner package reads it through + * its host port. + */ +const runnerRuntime = defineRuntimeSlots(); + +export const createAppleRunnerRuntimeState = (): RuntimeStateHandle => runnerRuntime.createHandle(); + +/** One piece of runner state this daemon runtime owns, created on first use. */ +export const runnerRuntimeSlot = runnerRuntime.slot; + +type RunnerOwner = { + leaseOwnerStateDir: string | undefined; + deviceClaimAuthorityProbe: RunnerDeviceClaimAuthorityProbe | undefined; +}; +const RUNNER_OWNER_SLOT = Symbol('appleRunnerOwner'); +const runnerOwner = (): RunnerOwner => + runnerRuntimeSlot(RUNNER_OWNER_SLOT, (): RunnerOwner => ({ + leaseOwnerStateDir: undefined, + deviceClaimAuthorityProbe: undefined, + })); + +export function setRunnerLeaseOwnerStateDir(stateDir: string | undefined): void { + runnerOwner().leaseOwnerStateDir = stateDir?.trim() || undefined; +} + +export function getRunnerLeaseOwnerStateDir(): string | undefined { + return runnerOwner().leaseOwnerStateDir; +} export function setRunnerDeviceClaimAuthorityProbe( probe: RunnerDeviceClaimAuthorityProbe | undefined, ): void { - runnerDeviceClaimAuthorityProbe = probe; + runnerOwner().deviceClaimAuthorityProbe = probe; } export function getRunnerDeviceClaimAuthorityProbe(): RunnerDeviceClaimAuthorityProbe | undefined { - return runnerDeviceClaimAuthorityProbe; + return runnerOwner().deviceClaimAuthorityProbe; } diff --git a/packages/platform-apple/src/runner-owner-facade.ts b/packages/platform-apple/src/runner-owner-facade.ts index 9b683958d8..123e115748 100644 --- a/packages/platform-apple/src/runner-owner-facade.ts +++ b/packages/platform-apple/src/runner-owner-facade.ts @@ -1,4 +1,5 @@ export { + createAppleRunnerRuntimeState, setRunnerDeviceClaimAuthorityProbe, setRunnerLeaseOwnerStateDir, type RunnerDeviceClaimAuthorityProbe, diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts index a5ecc5270f..e8436b5f97 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-close-finalization.test.ts @@ -381,7 +381,7 @@ test('graceful shutdown stops idle retained generations and hands off active rea const active = await ensureReadyRunner(activeDevice); await closeFinalizationLifecycle(idleDevice, []).finalizeApplicationClose(closeInput(true)); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(active.state, 'stopped'); assert.equal(idle.state, 'ready'); await stopAllIosRunnerSessions(); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts index ab7a494392..2251d23ec9 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-session-lifecycle.test.ts @@ -343,7 +343,7 @@ test('shutdown detach moves a handed-off session to stopped without killing its assert.ok(runnerPid); runnerStateTransitions.length = 0; - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(session.state, 'stopped'); assert.deepEqual(runnerStateTransitions, ['stopped']); @@ -361,7 +361,7 @@ test('a scoped simulator-set session hands off like one in the default set', asy const session = await ensureRunnerSession(device, {}); await serveOneCommand(device, session); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(session.state, 'stopped'); assert.match(leaseRaw(device.id), /"ownerToken": "detached-owner-/); @@ -393,7 +393,7 @@ test('a runner that served a command is handed off on the physical lane', async runnerStateTransitions.length = 0; const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); }); assert.equal(session.state, 'stopped'); @@ -418,7 +418,7 @@ test('a runner that never served a command is torn down instead of handed off', assert.equal(session.state, 'starting'); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.deepEqual(readRunnerSessionLiveness(device.id), { @@ -449,7 +449,7 @@ test('a runner reporting main-thread work still draining is not handed off', asy assert.equal(session.state, 'ready'); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"main_thread_occupied"/); @@ -484,7 +484,7 @@ test('a session that still owes a response is not handed off', async () => { assert.equal(session.commandCharges.outstandingChargeCount, 1); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"command_in_flight"/); @@ -521,7 +521,7 @@ test('a command abandoned by a cancelled transport keeps the runner occupied', a assert.equal(session.commandCharges.hasAbandonedCharges, true); const refused = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(refused, /"reason":"command_in_flight"/); // The refusal this issue is about: nothing is awaited any more, and only terminal evidence for that @@ -535,7 +535,7 @@ test('a command abandoned by a cancelled transport keeps the runner occupied', a // `runner-client-recovery-wiring.test.ts`, where the probe reaches a real session. await serveOneCommand(device, session); assert.equal(session.commandCharges.hasOutstandingCharges, false); - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); }); test('the macOS host runner is never handed off, although it is kind device', async () => { @@ -544,7 +544,7 @@ test('the macOS host runner is never handed off, although it is kind device', as await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"macos_host"/); @@ -563,7 +563,7 @@ test('a physical tvOS runner is never handed off, although it is kind device too await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"physical_non_ios_os"/); @@ -580,7 +580,7 @@ test('a usbmux-only xctest backend runner is never handed off', async () => { await serveOneCommand(device, session); const diagnostics = await captureDiagnostics(async () => { - assert.equal(await detachIosRunnerSessionsForShutdown(), 0); + assert.deepEqual(await detachIosRunnerSessionsForShutdown(), []); }); assert.match(diagnostics, /"reason":"xctest_backend"/); @@ -597,7 +597,7 @@ test("the handoff releases this daemon's log bookkeeping only after the lease sa tokenWhenReleased.push(leaseRaw(device.id)); }; - assert.equal(await detachIosRunnerSessionsForShutdown(), 1); + assert.equal((await detachIosRunnerSessionsForShutdown()).length, 1); assert.equal(tokenWhenReleased.length, 1); assert.match(tokenWhenReleased[0]!, /"ownerToken": "detached-owner-/); diff --git a/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts b/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts new file mode 100644 index 0000000000..6d0e3db2c7 --- /dev/null +++ b/packages/platform-apple/src/runner/__tests__/runner-session-runtime-isolation.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { beforeEach, test, vi } from 'vitest'; +import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; +import { createAppleRunnerRuntimeState } from '../../core/runner-owner-state.ts'; +import { appleRunnerTestHost } from '../test-host.ts'; +import { registerRunnerPrepProcess } from '../runner-artifact.ts'; +import { stopAllIosRunnerSessions } from '../runner-session.ts'; +import { openRunnerStartAdmission, runnerPrepProcessChildren } from '../runner-xctestrun.ts'; + +const mockSignalProcessGroupBestEffort = vi.fn(); +const mockRunAppleToolCommand = vi.fn(); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunAppleToolCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + appleRunnerTestHost.update({ + runAppleToolCommand: mockRunAppleToolCommand, + signalProcessGroupBestEffort: mockSignalProcessGroupBestEffort, + signalPidsBestEffort: vi.fn(), + }); +}); + +function prepChild(pid: number): ExecBackgroundResult['child'] { + return Object.assign(new EventEmitter(), { + pid, + exitCode: null, + }) as ExecBackgroundResult['child']; +} + +test("stopping every runner of one runtime leaves another runtime's prep build and admission alone", async () => { + const runtimeA = createAppleRunnerRuntimeState(); + const runtimeB = createAppleRunnerRuntimeState(); + const deviceId = 'runner-runtime-isolation-sim'; + const buildA = prepChild(6101); + const buildB = prepChild(6102); + + runtimeA.run(() => { + registerRunnerPrepProcess(deviceId, buildA, openRunnerStartAdmission(deviceId)); + }); + const admissionB = runtimeB.run(() => { + const admission = openRunnerStartAdmission(deviceId); + registerRunnerPrepProcess(deviceId, buildB, admission); + return admission; + }); + + await runtimeA.run(async () => await stopAllIosRunnerSessions()); + + const signaled = mockSignalProcessGroupBestEffort.mock.calls.map(([pid]) => pid as number); + assert.ok(signaled.includes(6101), 'runtime A stopped its own build'); + assert.ok(!signaled.includes(6102), "runtime A never signaled runtime B's build"); + assert.equal( + runtimeA.run(() => runnerPrepProcessChildren(deviceId).length), + 0, + ); + assert.equal( + runtimeB.run(() => runnerPrepProcessChildren(deviceId).length), + 1, + ); + assert.equal(admissionB.admitted, true, "runtime B's start admission stays open"); +}); diff --git a/packages/platform-apple/src/runner/host.ts b/packages/platform-apple/src/runner/host.ts index ce6af7a57f..c1ae78b3fb 100644 --- a/packages/platform-apple/src/runner/host.ts +++ b/packages/platform-apple/src/runner/host.ts @@ -102,6 +102,8 @@ export type AppleRunnerHost = Pick< resolveIosPhysicalDeviceControl: (device: DeviceInfo) => IosPhysicalDeviceRunnerControl; /** Daemon-owned lease owner state directory. */ leaseOwnerStateDir: typeof AppleRunnerOwnerState.getRunnerLeaseOwnerStateDir; + /** One registry the daemon runtime owns (sessions, locks, ledgers), created on first use. */ + runnerRuntimeSlot: typeof AppleRunnerOwnerState.runnerRuntimeSlot; /** * Daemon-owned device-claim arbitration probe: true only while the embedding process holds * the host-global local device claim for exactly this device (matched by canonical @@ -179,6 +181,7 @@ export const classifyOwnerLiveness = delegate('classifyOwnerLiveness'); export const readHostCpuArchSync = delegate('readHostCpuArchSync'); export const findProjectRoot = delegate('findProjectRoot'); export const readVersion = delegate('readVersion'); +export const runnerRuntimeSlot = delegate('runnerRuntimeSlot'); export const acquireProcessLock = delegate('acquireProcessLock'); export const withProcessLock = delegate('withProcessLock'); export const publishFileSync = delegate('publishFileSync'); diff --git a/packages/platform-apple/src/runner/runner-adoption.ts b/packages/platform-apple/src/runner/runner-adoption.ts index 8d7288957e..1270405ddc 100644 --- a/packages/platform-apple/src/runner/runner-adoption.ts +++ b/packages/platform-apple/src/runner/runner-adoption.ts @@ -83,9 +83,9 @@ export function isIosRunnerDetachEnabled(env: NodeJS.ProcessEnv = hostEnvironmen export async function detachRunnerSessionsForShutdown( sessions: ReadonlyMap, onDetached: (deviceId: string, session: RunnerSession) => void, -): Promise { - if (!isIosRunnerDetachEnabled()) return 0; - let detached = 0; +): Promise { + if (!isIosRunnerDetachEnabled()) return []; + const detached: string[] = []; for (const [deviceId, session] of sessions) { const outcome = detachRunnerSessionForShutdown(session); if (!outcome.detached) { @@ -106,7 +106,7 @@ export async function detachRunnerSessionsForShutdown( }); continue; } - detached += 1; + detached.push(deviceId); onDetached(deviceId, session); emitDiagnostic({ level: 'info', diff --git a/packages/platform-apple/src/runner/runner-artifact.ts b/packages/platform-apple/src/runner/runner-artifact.ts index b3301914b5..af226cc51e 100644 --- a/packages/platform-apple/src/runner/runner-artifact.ts +++ b/packages/platform-apple/src/runner/runner-artifact.ts @@ -16,6 +16,7 @@ import { findProjectRoot, isCommandTimeoutError, hostEnvironment, + runnerRuntimeSlot, } from './host.ts'; import type { ExecBackgroundResult } from '@agent-device/host-kit/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -120,7 +121,10 @@ type DeviceStartState = { inFlight: Set; }; -const deviceStartStates = new Map(); +const DEVICE_START_STATES_SLOT = Symbol('apple-runner-device-start-states'); +// Per daemon runtime: a runtime's teardown retires only the start admissions it opened. +const deviceStartStates = () => + runnerRuntimeSlot(DEVICE_START_STATES_SLOT, () => new Map()); /** * Opens the admission one start answers to, taken when a caller asks for a runner. Registration is @@ -145,7 +149,7 @@ export function openRunnerStartLoopAdmission(deviceId: string): RunnerStartAdmis /** The start settled: its token leaves the device's in-flight set. Verdicts live on the token. */ export function finishRunnerStartAdmission(admission: RunnerStartAdmission): void { (admission as StartAdmission).retryPending = false; - const state = deviceStartStates.get(admission.deviceId); + const state = deviceStartStates().get(admission.deviceId); if (!state) return; state.inFlight.delete(admission as StartAdmission); forgetDeviceStartState(admission.deviceId, state); @@ -181,7 +185,10 @@ export function fenceRunnerStartAdmissionsForTeardown(deviceId: string): () => v * build past the sweep that is about to stop it. */ export function retireAllRunnerStartAdmissions(additionalDeviceIds?: Iterable): () => void { - const deviceIds = new Set([...deviceStartStates.keys(), ...(additionalDeviceIds ?? [])]); + const deviceIds = new Set([ + ...deviceStartStates().keys(), + ...(additionalDeviceIds ?? []), + ]); const settles = [...deviceIds].map((deviceId) => fenceRunnerStartAdmissionsForTeardown(deviceId)); return () => { for (const settle of settles.splice(0)) settle(); @@ -194,7 +201,7 @@ export function retireAllRunnerStartAdmissions(additionalDeviceIds?: Iterable 0; + return (deviceStartStates().get(deviceId)?.pendingTeardowns ?? 0) > 0; } /** @@ -300,17 +307,17 @@ export function releaseRunnerStartWaiter( } function stateForDevice(deviceId: string): DeviceStartState { - const existing = deviceStartStates.get(deviceId); + const existing = deviceStartStates().get(deviceId); if (existing) return existing; const state: DeviceStartState = { pendingTeardowns: 0, inFlight: new Set() }; - deviceStartStates.set(deviceId, state); + deviceStartStates().set(deviceId, state); return state; } /** Nothing fences the device and no start is in flight on it once the last of both is gone. */ function forgetDeviceStartState(deviceId: string, state: DeviceStartState): void { if (state.pendingTeardowns <= 0 && state.inFlight.size === 0) { - deviceStartStates.delete(deviceId); + deviceStartStates().delete(deviceId); } } @@ -361,7 +368,10 @@ type RunnerPrepProcess = Readonly<{ child: ExecBackgroundResult['child']; }>; -const runnerPrepProcessLedger = new Set(); +const RUNNER_PREP_LEDGER_SLOT = Symbol('apple-runner-prep-process-ledger'); +// Per daemon runtime: a runtime's stop signals only the prep children it started. +const runnerPrepProcessLedger = () => + runnerRuntimeSlot(RUNNER_PREP_LEDGER_SLOT, () => new Set()); /** * Records a prep subprocess (`xcodebuild build-for-testing`) against the device it builds for and @@ -378,9 +388,9 @@ export function registerRunnerPrepProcess( startAdmission?: RunnerStartAdmission, ): void { const entry: RunnerPrepProcess = { deviceId, startAdmission, child }; - runnerPrepProcessLedger.add(entry); + runnerPrepProcessLedger().add(entry); child.on('close', () => { - runnerPrepProcessLedger.delete(entry); + runnerPrepProcessLedger().delete(entry); }); } @@ -427,14 +437,14 @@ export function markRunnerStartRetryPending(admission: RunnerStartAdmission): vo } function prepProcessEntries(deviceId?: string): readonly RunnerPrepProcess[] { - return [...runnerPrepProcessLedger].filter( + return [...runnerPrepProcessLedger()].filter( (entry) => deviceId === undefined || entry.deviceId === deviceId, ); } export function forgetRunnerPrepProcess(child: ExecBackgroundResult['child']): void { - for (const entry of runnerPrepProcessLedger) { - if (entry.child === child) runnerPrepProcessLedger.delete(entry); + for (const entry of runnerPrepProcessLedger()) { + if (entry.child === child) runnerPrepProcessLedger().delete(entry); } } diff --git a/packages/platform-apple/src/runner/runner-cache.ts b/packages/platform-apple/src/runner/runner-cache.ts index 9a427ab1b3..f16b744920 100644 --- a/packages/platform-apple/src/runner/runner-cache.ts +++ b/packages/platform-apple/src/runner/runner-cache.ts @@ -9,6 +9,7 @@ import { isEnvTruthy, findProjectRoot, hostEnvironment, + runnerRuntimeSlot, } from './host.ts'; import { RUNNER_CACHE_METADATA_FILE, @@ -39,7 +40,9 @@ export { requireRunnerPhaseRemainingMs, type RunnerPhaseBudget } from './runner- const RUNNER_XCTESTRUN_CACHE_LOCK_TIMEOUT_MS = 10 * 60_000; const RUNNER_XCTESTRUN_CACHE_LOCK_POLL_MS = 100; -const badRunnerArtifactsForRun = new Set(); +const BAD_ARTIFACTS_SLOT = Symbol('apple-runner-bad-artifacts'); +const badRunnerArtifactsForRun = () => + runnerRuntimeSlot(BAD_ARTIFACTS_SLOT, () => new Set()); export type RunnerXctestrunCacheKind = 'exact' | 'miss' | 'external'; @@ -103,7 +106,7 @@ export async function markRunnerXctestrunArtifactBadForRun( return; } - badRunnerArtifactsForRun.add(artifact.derived); + badRunnerArtifactsForRun().add(artifact.derived); await withProcessLock({ acquire: () => acquireRunnerXctestrunCacheLock(artifact.derived), task: async () => { @@ -140,7 +143,7 @@ function resolveRunnerXctestrunCacheLockPath(derived: string): string { } export function cleanRunnerDerivedBeforeEvaluation(derived: string, forceRebuild: boolean): void { - if (!shouldCleanDerived() && !forceRebuild && !badRunnerArtifactsForRun.has(derived)) { + if (!shouldCleanDerived() && !forceRebuild && !badRunnerArtifactsForRun().has(derived)) { return; } emitRunnerXctestrunDecision('clean', forceRebuild ? 'forced_rebuild' : 'forced_clean', { @@ -148,7 +151,7 @@ export function cleanRunnerDerivedBeforeEvaluation(derived: string, forceRebuild }); assertSafeDerivedCleanup(derived); cleanRunnerDerivedArtifacts(derived); - badRunnerArtifactsForRun.delete(derived); + badRunnerArtifactsForRun().delete(derived); } /** diff --git a/packages/platform-apple/src/runner/runner-recycle-ledger.ts b/packages/platform-apple/src/runner/runner-recycle-ledger.ts index 149e681ea3..8af8fbd867 100644 --- a/packages/platform-apple/src/runner/runner-recycle-ledger.ts +++ b/packages/platform-apple/src/runner/runner-recycle-ledger.ts @@ -1,5 +1,5 @@ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; -import { emitDiagnostic } from './host.ts'; +import { emitDiagnostic, runnerRuntimeSlot } from './host.ts'; import type { RunnerCommand } from './runner-contract.ts'; // Caps how many times one daemon request may pay for a full runner recycle @@ -19,7 +19,9 @@ type RunnerRecycleLedgerEntry = { lastAtMs: number; }; -const ledger = new Map(); +const LEDGER_SLOT = Symbol('apple-runner-recycle-ledger'); +const ledgerEntries = (): Map => + runnerRuntimeSlot(LEDGER_SLOT, () => new Map()); /** Requests are the recycle scope; direct calls without a request fall back to the command id. */ export function runnerRecycleLedgerKey( @@ -43,7 +45,7 @@ export function markRunnerRequestTouchedSession(key: string | undefined): void { export function hasRunnerRequestTouchedSession(key: string | undefined): boolean { if (!key) return false; - return ledger.get(key)?.touchedSession === true; + return ledgerEntries().get(key)?.touchedSession === true; } /** @@ -108,28 +110,28 @@ export function buildRunnerRecycleBudgetExhaustedError( * @internal Test isolation hook for process-global runner recycle accounting. */ export function resetRunnerRecycleLedgerForTests(): void { - ledger.clear(); + ledgerEntries().clear(); } function readEntry(key: string): RunnerRecycleLedgerEntry { pruneLedger(); - return ledger.get(key) ?? { touchedSession: false, recycles: 0, lastAtMs: Date.now() }; + return ledgerEntries().get(key) ?? { touchedSession: false, recycles: 0, lastAtMs: Date.now() }; } function writeEntry(key: string, entry: RunnerRecycleLedgerEntry): void { entry.lastAtMs = Date.now(); - ledger.delete(key); - ledger.set(key, entry); + ledgerEntries().delete(key); + ledgerEntries().set(key, entry); } function pruneLedger(): void { const cutoff = Date.now() - LEDGER_ENTRY_TTL_MS; - for (const [key, entry] of ledger) { - if (entry.lastAtMs < cutoff) ledger.delete(key); + for (const [key, entry] of ledgerEntries()) { + if (entry.lastAtMs < cutoff) ledgerEntries().delete(key); } - while (ledger.size > LEDGER_MAX_ENTRIES) { - const oldest = ledger.keys().next().value; + while (ledgerEntries().size > LEDGER_MAX_ENTRIES) { + const oldest = ledgerEntries().keys().next().value; if (oldest === undefined) break; - ledger.delete(oldest); + ledgerEntries().delete(oldest); } } diff --git a/packages/platform-apple/src/runner/runner-session.ts b/packages/platform-apple/src/runner/runner-session.ts index 22f3fb257e..f313c9c860 100644 --- a/packages/platform-apple/src/runner/runner-session.ts +++ b/packages/platform-apple/src/runner/runner-session.ts @@ -1,5 +1,6 @@ import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { + runnerRuntimeSlot, withKeyedLock, Deadline, emitRequestProgress, @@ -85,12 +86,17 @@ export type { RunnerSession } from './runner-session-types.ts'; export type RunnerSessionOptions = AppleRunnerLifecycleOptions; -const runnerSessions = new Map(); -const runnerSessionLocks = new Map>(); +const RUNNER_SESSIONS_SLOT = Symbol('apple-runner-sessions'); +const RUNNER_SESSION_LOCKS_SLOT = Symbol('apple-runner-session-locks'); +// The daemon runtime's registries: two runtimes in one process never see each other's runners. +const runnerSessions = () => + runnerRuntimeSlot(RUNNER_SESSIONS_SLOT, () => new Map()); +const runnerSessionLocks = () => + runnerRuntimeSlot(RUNNER_SESSION_LOCKS_SLOT, () => new Map>()); const RUNNER_STALE_BUNDLE_UNINSTALL_TIMEOUT_MS = 10_000; function withRunnerSessionLock(deviceId: string, task: () => Promise): Promise { - return withKeyedLock(runnerSessionLocks, deviceId, task); + return withKeyedLock(runnerSessionLocks(), deviceId, task); } export async function ensureRunnerSession( @@ -123,7 +129,7 @@ export async function ensureRunnerSession( // a joiner's cancellation can never outvote the live owner and stop its build (#3220). const releaseOwnerInterest = reserveRunnerStartOwnerInterest(startAdmission, options); try { - const existing = runnerSessions.get(device.id); + const existing = runnerSessions().get(device.id); if (existing) { assertExpectedRunnerSession(existing, options.expectedRunnerSessionId); const reusable = await resolveReusableRunnerSession(device, existing, budget.phase); @@ -196,7 +202,7 @@ async function startRunnerSessionWithLease( // gate: a start fenced by a teardown registers its runner into no device (#3220). The // runner itself keeps its lease for the next open to adopt. assertRunnerStartAdmitsPreparation(device.id, startAdmission); - runnerSessions.set(device.id, adopted); + runnerSessions().set(device.id, adopted); return adopted; } assertRunnerSessionMayStart(options.expectedRunnerSessionId); @@ -370,7 +376,7 @@ async function startRunnerSessionWithLease( }); throw error; } - runnerSessions.set(device.id, session); + runnerSessions().set(device.id, session); return session; } @@ -561,7 +567,7 @@ async function uninstallStaleSimulatorRunnerBundle( * registered, which is its own answer: there is no session to wait for or tear down. */ export function readRunnerSessionLiveness(deviceId: string): RunnerSessionRegistration | null { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session) return null; return { sessionId: session.sessionId, @@ -581,7 +587,7 @@ export async function invalidateRunnerSession( reason: string, ): Promise { await withRunnerSessionLock(session.deviceId, async () => { - if (runnerSessions.get(session.deviceId) !== session) return; + if (runnerSessions().get(session.deviceId) !== session) return; // A session already being torn down, or already torn down, is never disposed a second time // for a later reason; the reason-coded diagnostic below reports why this call was made. if (!canWorkWithRunnerSession(session)) return; @@ -606,15 +612,15 @@ async function stopRunnerSessionInternal( sessionOverride?: RunnerSession, options: RunnerDisposalOptions = {}, ): Promise { - const session = sessionOverride ?? runnerSessions.get(deviceId); + const session = sessionOverride ?? runnerSessions().get(deviceId); if (!session) return; // Once disposal has begun or finished, this session has no runner to wait on; a repeat stop // would only re-signal a process that is already leaving and re-emit a teardown for a reason // that has nothing left to tear down. if (!canWorkWithRunnerSession(session)) return; await disposeRunnerSession(session, options); - if (runnerSessions.get(deviceId) === session) { - runnerSessions.delete(deviceId); + if (runnerSessions().get(deviceId) === session) { + runnerSessions().delete(deviceId); } } @@ -624,7 +630,8 @@ async function stopRetainedRunner( reason: 'idle_timeout' | 'listener_lost', ): Promise { await withRunnerSessionLock(session.deviceId, async () => { - if (runnerSessions.get(session.deviceId) !== session || session.retention !== retention) return; + if (runnerSessions().get(session.deviceId) !== session || session.retention !== retention) + return; emitDiagnostic({ level: reason === 'listener_lost' ? 'warn' : 'info', phase: reason === 'listener_lost' ? 'ios_runner_warm_stop' : 'ios_runner_idle_stop', @@ -653,7 +660,7 @@ export async function releaseSpeculativeIosRunnerSession(deviceId: string): Prom // Under the session lock: a prewarm still starting holds it and registers its session only // when the start completes, so the release queues behind that start instead of missing it. return await withRunnerSessionLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session?.speculative) return false; emitDiagnostic({ level: 'debug', @@ -685,7 +692,7 @@ async function stopIosRunnerSessionDevice( try { await withRunnerSessionLock(deviceId, async () => { await withRunnerLeaseLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); await stopRunnerSessionInternal(deviceId, session, { leaseLockHeld: true, graceful: @@ -707,13 +714,13 @@ export async function releaseIosRunnerOnClose( deviceId: string, options: { retain: boolean }, ): Promise { - const candidate = runnerSessions.get(deviceId); + const candidate = runnerSessions().get(deviceId); const busyAtClose = isRunnerMainThreadOccupied(candidate); const settleFence = fenceRunnerStartAdmissionsForTeardown(deviceId); try { if (options.retain && candidate?.state === 'ready' && !busyAtClose) { const retained = await withRunnerSessionLock(deviceId, async () => { - const session = runnerSessions.get(deviceId); + const session = runnerSessions().get(deviceId); if (!session) return false; const { retainRunnerSession } = await import('./runner-retention.ts'); if ( @@ -752,15 +759,15 @@ export async function abortAllIosRunnerSessions(): Promise { // The same fence a close raises, for every device at once: the prep sweep the abort performs is // one-shot, so an in-flight start that would answer it with another build is refused before the // sweep runs, and stays refused until the sweep settles (#3220). - const activeSessions = Array.from(runnerSessions.values()); + const activeSessions = Array.from(runnerSessions().values()); const settleFences = retireAllRunnerStartAdmissions( activeSessions.map((session) => session.deviceId), ); try { await abortRunnerSessionsAndPrepProcesses(activeSessions); for (const session of activeSessions) { - if (runnerSessions.get(session.deviceId) === session) { - runnerSessions.delete(session.deviceId); + if (runnerSessions().get(session.deviceId) === session) { + runnerSessions().delete(session.deviceId); } } } finally { @@ -772,26 +779,29 @@ export async function abortAllIosRunnerSessions(): Promise { // The detach decision itself lives in runner-adoption.ts beside the adoption it hands off to // (#2681); detached active generations leave the registry. Idle retained generations stay // owned until shutdown stops them. -export async function detachIosRunnerSessionsForShutdown(): Promise { - return await detachRunnerSessionsForShutdown(runnerSessions, (deviceId) => { - runnerSessions.delete(deviceId); +export async function detachIosRunnerSessionsForShutdown(): Promise { + return await detachRunnerSessionsForShutdown(runnerSessions(), (deviceId) => { + runnerSessions().delete(deviceId); }); } -export async function stopAllIosRunnerSessions(): Promise { +/** Stops every remaining runner and names the devices it owned. */ +export async function stopAllIosRunnerSessions(): Promise { + const owned = [...runnerSessions().keys()]; // This is the daemon's teardown: the fence spans the whole sweep, not just the abort, so a // start that began while the per-session stops ran meets a closed admission at its next gate // instead of answering the final prep sweep with another build (#3220). - const settleFences = retireAllRunnerStartAdmissions(runnerSessions.keys()); + const settleFences = retireAllRunnerStartAdmissions(runnerSessions().keys()); try { await abortAllIosRunnerSessions(); - const pending = Array.from(runnerSessions.keys()); + const pending = Array.from(runnerSessions().keys()); await Promise.allSettled( pending.map(async (deviceId) => { await stopIosRunnerSession(deviceId); }), ); await stopRunnerPrepProcesses(); + return owned; } finally { settleFences(); } @@ -853,7 +863,7 @@ export async function executeRunnerCommandWithSession( (reason) => invalidateRunnerSession(session, reason), signal, ); - if (runnerSessions.get(device.id) === session && session.state === 'ready') { + if (runnerSessions().get(device.id) === session && session.state === 'ready') { const { observeRunnerListener } = await import('./runner-retention.ts'); observeRunnerListener(session, () => { const retention = session.retention; diff --git a/scripts/__tests__/closure-growth-rule.test.ts b/scripts/__tests__/closure-growth-rule.test.ts index cfba02f6c6..16833b2dd2 100644 --- a/scripts/__tests__/closure-growth-rule.test.ts +++ b/scripts/__tests__/closure-growth-rule.test.ts @@ -50,6 +50,45 @@ test('no-growth fails growth with both counts and passes an equal or smaller clo ).toMatch(/evaluates 43 modules.*merge-base evaluated 42/); }); +test('a module a maintainer approved is excused from the count, and only that module', () => { + const grown = { + baseCount: 42, + headCount: 43, + baseWeight: 200, + headWeight: 230, + preservedBaseClosure: true, + addedModulesAreNew: true, + }; + expect(classifyGrowth('x.ts', { ...grown, approvedAddedCount: 1 })).toBeNull(); + expect( + classifyGrowth('x.ts', { ...grown, headCount: 44, approvedAddedCount: 1 }), + 'a second, unapproved addition still fails', + ).toMatch(/evaluates 44 modules.*merge-base evaluated 42/); + expect(classifyGrowth('x.ts', grown), 'without approval the same growth fails').toMatch( + /evaluates 43 modules/, + ); +}); + +test('an approved module that only renames a base module excuses nothing', () => { + const graph = (...files: string[]) => new Map(files.map((file) => [`/r/${file}`, null])); + const weights = (...files: string[]) => new Map(files.map((file) => [`/r/${file}`, 1])); + const evidence = closureGrowthEvidence({ + baseGraph: graph('entry.ts', 'old.ts'), + headGraph: graph('entry.ts', 'moved.ts', 'extra.ts'), + baseWeights: weights('entry.ts', 'old.ts'), + headWeights: weights('entry.ts', 'moved.ts', 'extra.ts'), + headStatementTexts: new Map(), + baseTree: { exists: () => false }, + renamedBaseToHead: new Map([['/r/old.ts', '/r/moved.ts']]), + approvedModules: new Set(['/r/moved.ts']), + }); + + expect(evidence.approvedAddedCount, 'moved.ts was already in the base closure as old.ts').toBe(0); + expect(classifyGrowth('x.ts', evidence), 'the unrelated addition still fails').toMatch( + /evaluates 3 modules.*merge-base evaluated 2/, + ); +}); + // --- the split tolerance (#2469) ------------------------------------------------------------- // The no-growth rule counts modules, and a split of a hub module is count growth with no new // eager work -- which made extracting any module inside a gated closure impossible (ADR 0027). diff --git a/scripts/__tests__/closure-growth-rule.ts b/scripts/__tests__/closure-growth-rule.ts index 3ae97697e3..f9ce30a234 100644 --- a/scripts/__tests__/closure-growth-rule.ts +++ b/scripts/__tests__/closure-growth-rule.ts @@ -60,6 +60,11 @@ export type ClosureGrowthEvidence = { * pre-existing code newly made eager, under its own name or a renamed one. */ addedModulesAreNew: boolean; + /** + * How many of the added modules a maintainer approved by name (`APPROVED_CLOSURE_ADDITIONS`). + * Each is excused from the count comparison; an unapproved addition is judged as before. + */ + approvedAddedCount?: number; }; /** @@ -87,6 +92,8 @@ export function closureGrowthEvidence(params: { * pre-existing code, not a new file. */ deletedSourceTexts?: readonly (readonly string[])[]; + /** Absolute paths of the modules a maintainer approved as additions to existing closures. */ + approvedModules?: ReadonlySet; }): ClosureGrowthEvidence { const { baseGraph, @@ -97,6 +104,7 @@ export function closureGrowthEvidence(params: { baseTree, renamedBaseToHead, deletedSourceTexts = [], + approvedModules = new Set(), } = params; const sum = (files: Iterable, weights: ReadonlyMap) => [...files].reduce((total, file) => total + (weights.get(file) ?? 0), 0); @@ -107,6 +115,13 @@ export function closureGrowthEvidence(params: { return { baseCount: baseGraph.size, headCount: headGraph.size, + // Renames are canonicalized first: an approved module that is a rename of a base module is + // not an addition. + approvedAddedCount: [...headGraph.keys()].filter( + (headFile) => + approvedModules.has(headFile) && + !baseGraph.has(baseOfRenamedHead.get(headFile) ?? headFile), + ).length, baseWeight: sum(baseGraph.keys(), baseWeights), headWeight: sum(headGraph.keys(), headWeights), preservedBaseClosure: [...baseGraph.keys()].every((baseFile) => @@ -160,7 +175,7 @@ function movedFromDeletedSource( export function classifyGrowth(id: string, evidence: ClosureGrowthEvidence): string | null { const { baseCount, headCount, baseWeight, headWeight, preservedBaseClosure, addedModulesAreNew } = evidence; - if (headCount <= baseCount) return null; + if (headCount - (evidence.approvedAddedCount ?? 0) <= baseCount) return null; if (preservedBaseClosure && addedModulesAreNew && headWeight <= baseWeight) return null; const reasons: string[] = []; if (!preservedBaseClosure) { diff --git a/scripts/__tests__/eager-closure-budgets.test.ts b/scripts/__tests__/eager-closure-budgets.test.ts index ea1296cc88..2a013fdbf5 100644 --- a/scripts/__tests__/eager-closure-budgets.test.ts +++ b/scripts/__tests__/eager-closure-budgets.test.ts @@ -23,6 +23,7 @@ import { } from './closure-growth-rule.ts'; import { addedModules, + APPROVED_CLOSURE_ADDITIONS, APPROVED_OVER_CEILING, classifyNewEntry, describeClosureGrowth, @@ -725,6 +726,10 @@ const deletedSourceTexts = [ * cross-entry shared-homes note after it read the same graphs instead of walking each closure * twice. */ +const approvedAdditionPaths = new Set( + APPROVED_CLOSURE_ADDITIONS.map((row) => absolute(row.module)), +); + const carriedGrowth = carried.map((entry) => { const entryPath = absolute(entry.entryFile); const graph = eagerClosureGraphOf(entryPath); @@ -746,10 +751,20 @@ const carriedGrowth = carried.map((entry) => { baseTree, renamedBaseToHead, deletedSourceTexts, + approvedModules: approvedAdditionPaths, }), }; }); +test('every approved closure addition names an existing module with its issue and owner', () => { + for (const row of APPROVED_CLOSURE_ADDITIONS) { + expect(fs.existsSync(absolute(row.module)), `${row.module} no longer exists`).toBe(true); + expect(row.issue).toMatch(/^#\d+$/); + expect(row.reason.trim().length).toBeGreaterThan(0); + expect(row.owner.trim().length).toBeGreaterThan(0); + } +}); + test('every hub exists and is not also a discovered façade', () => { const discovered = new Set(discoverFacadeEntryFiles(repoRoot)); expect(HUB_ENTRY_FILES.filter((file) => !fs.existsSync(absolute(file)))).toEqual([]); diff --git a/scripts/__tests__/eager-closure-budgets.ts b/scripts/__tests__/eager-closure-budgets.ts index c16ba57f23..73851ada96 100644 --- a/scripts/__tests__/eager-closure-budgets.ts +++ b/scripts/__tests__/eager-closure-budgets.ts @@ -134,6 +134,26 @@ export const APPROVED_OVER_CEILING: Readonly< Record > = Object.freeze({}); +/** + * Modules a maintainer approved as additions to the closures of entries that exist at the merge-base: + * each is excused from those entries' no-growth count, and only that module is. One row per module, + * naming the issue, the reason and the owner who approved it. A row whose module is gone fails. + */ +export const APPROVED_CLOSURE_ADDITIONS: readonly Readonly<{ + module: string; + issue: string; + reason: string; + owner: string; +}>[] = Object.freeze([ + { + module: 'packages/host-kit/src/runtime-slots.ts', + issue: '#3752', + reason: + 'The one owner of per-runtime async-local state: the store, its standalone fallback and its disposal, so Apple, request-cancel and the root declare only slot keys instead of each keeping a copy.', + owner: 'thymikee', + }, +]); + /** The category is a function of the path, never a hand-written column. */ export function entryCategoryOf(entryFile: string): EntryCategory { if (/^packages\/platform-[^/]+\/src\/index\.ts$/.test(entryFile)) return 'platform-facade'; diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index bc3c73aac1..fd50f8ba44 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -614,6 +614,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/host-kit/project-runtime', '@agent-device/host-kit/request', '@agent-device/host-kit/retry', + '@agent-device/host-kit/runtime-slots', '@agent-device/host-kit/runtime-transport-hints', '@agent-device/host-kit/session-paths', '@agent-device/host-kit/status-markers', diff --git a/scripts/layering/platform-package-policy.test.ts b/scripts/layering/platform-package-policy.test.ts index a25f962e3b..1e67b22946 100644 --- a/scripts/layering/platform-package-policy.test.ts +++ b/scripts/layering/platform-package-policy.test.ts @@ -389,7 +389,7 @@ test('platform packages may import the xml vocabulary package', () => { assert.deepEqual(checkPlatformPackagePolicy(sources, declarations()), []); }); -test('the Android mechanics facet is the named consumer seam and adb host wiring is root-only', () => { +test('the Android mechanics facet is the named consumer seam and adb host wiring is limited to the root binding and daemon composition', () => { const mechanicsImport = "import { resolveAndroidAdbExecutor } from '@agent-device/platform-android/mechanics';"; const hostImport = @@ -399,6 +399,7 @@ test('the Android mechanics facet is the named consumer seam and adb host wiring allowed.set('src/sdk/android-adb.ts', mechanicsImport); allowed.set('packages/daemon/src/sessions/handlers/__tests__/session.test.ts', mechanicsImport); allowed.set('src/platform-runtime-android-adb-host.ts', hostImport); + allowed.set('src/daemon-composition.ts', hostImport); assert.deepEqual( messages(allowed).filter((message) => message.includes('may import')), [], diff --git a/scripts/layering/platform-package-policy.ts b/scripts/layering/platform-package-policy.ts index eb77969eb1..3e561f4974 100644 --- a/scripts/layering/platform-package-policy.ts +++ b/scripts/layering/platform-package-policy.ts @@ -71,7 +71,12 @@ const APPLE_RUNNER_TEST_HOST_FILE = 'packages/platform-apple/src/runner/test-hos const APPLE_RUNNER_FACADE = '@agent-device/platform-apple/runner'; const ANDROID_MECHANICS_FACADE = '@agent-device/platform-android/mechanics'; const ANDROID_HOST_FACET = '@agent-device/platform-android/adb-host'; -const ANDROID_HOST_BINDING = 'src/platform-runtime-android-adb-host.ts'; +// The host port is bound by the root; daemon composition registers the helper's runtime state. +const ANDROID_HOST_FACET_IMPORTERS: ReadonlySet = new Set([ + 'src/platform-runtime-android-adb-host.ts', + 'src/daemon-composition.ts', + 'src/daemon-composition.test.ts', +]); const ANDROID_DEVICE_BOOT_FACADE = '@agent-device/platform-android/device-boot'; const MECHANICS_FACET_SUBPATHS: Readonly>> = { apple: [ @@ -115,7 +120,7 @@ function isInteractorSubpathImport( } function isAndroidFacetSubpathImport(file: string, specifier: string): boolean { - if (specifier === ANDROID_HOST_FACET) return file === ANDROID_HOST_BINDING; + if (specifier === ANDROID_HOST_FACET) return ANDROID_HOST_FACET_IMPORTERS.has(file); if (isPlatformTestOnlyImport(specifier)) return true; if (!(MECHANICS_FACET_SUBPATHS.android ?? []).includes(specifier)) return false; // The named facets are the implementation seam for root/core/SDK consumers. Daemon production diff --git a/src/__tests__/test-utils/public-daemon-runtime.ts b/src/__tests__/test-utils/public-daemon-runtime.ts index ad8c87897c..6e8c3fcf02 100644 --- a/src/__tests__/test-utils/public-daemon-runtime.ts +++ b/src/__tests__/test-utils/public-daemon-runtime.ts @@ -52,8 +52,6 @@ export async function startPublicDaemonRuntime( AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -74,12 +72,12 @@ export async function startPublicDaemonRuntime( baseUrl: `http://127.0.0.1:${port}`, runtime, close: async () => { - await runtime.shutdown(); + await runtime.close(); fs.rmSync(stateDir, { recursive: true, force: true }); }, }; } catch (error) { - await runtime.shutdown().catch(() => {}); + await runtime.close().catch(() => {}); if (ownsStateDir) fs.rmSync(stateDir, { recursive: true, force: true }); throw error; } diff --git a/src/daemon-composition-credentials.test.ts b/src/daemon-composition-credentials.test.ts index cb85ce356c..bbc5033d32 100644 --- a/src/daemon-composition-credentials.test.ts +++ b/src/daemon-composition-credentials.test.ts @@ -15,8 +15,8 @@ vi.mock('./platform-runtime.ts', () => ({ createPlatformRuntimeGateway: () => ({ applicationLifecycle: { recoverStartupResources: async () => {}, - detachForDaemonShutdown: async () => {}, - finalizeDaemonShutdown: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], }, inspectFacts: async () => { throw new Error('unused'); @@ -63,8 +63,6 @@ test('daemon startup compares lease credentials with its own startup environment AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, composeDeps: createDaemonRuntimeDeps, @@ -105,6 +103,6 @@ test('daemon startup compares lease credentials with its own startup environment ); expect(await allocate(daemonEnv)).not.toContain('provider-credentials-changed'); } finally { - await runtime?.shutdown(); + await runtime?.close(); } }); diff --git a/src/daemon-composition.test.ts b/src/daemon-composition.test.ts index e7faee68aa..629a3f2679 100644 --- a/src/daemon-composition.test.ts +++ b/src/daemon-composition.test.ts @@ -7,6 +7,7 @@ import { createTestHostEnvironment, } from '@agent-device/test-kit'; import { createDaemonRuntimeDeps } from './daemon-composition.ts'; +import { helperRuntimeSlot } from '@agent-device/platform-android/adb-host'; import { setActiveProviderDeviceRuntimes } from './provider-device-runtime.ts'; const ports = vi.hoisted(() => ({ @@ -22,8 +23,8 @@ vi.mock('./platform-runtime.ts', () => ({ createPlatformRuntimeGateway: () => ({ applicationLifecycle: { recoverStartupResources: async () => {}, - detachForDaemonShutdown: async () => {}, - finalizeDaemonShutdown: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], }, inspectFacts: async () => { throw new Error('unused'); @@ -121,3 +122,36 @@ test('a composition that succeeds leaves the plugins open for the daemon to clos await deps.shutdownProviderRuntimes(); expect(ports.shutdownProviders).toHaveBeenCalledOnce(); }); + +test('a composed daemon registers the Apple runner and Android helper state, one set per composition', async () => { + const compose = async (name: string) => { + const baseDir = mkdtempForTestSync(`agent-device-daemon-composition-states-${name}-`); + return await createDaemonRuntimeDeps({ + host: createTestHostEnvironment({ + env: { ...process.env, AGENT_DEVICE_STATE_DIR: baseDir }, + }), + baseDir, + sessionsDir: `${baseDir}/sessions`, + assertShutdownAllowed: undefined, + resolveSessionDir: (sessionId) => `${baseDir}/sessions/${sessionId}`, + resolveSessionArtifacts: () => { + throw new Error('unused'); + }, + openWebSessionNames: () => [], + }); + }; + const first = await compose('first'); + const second = await compose('second'); + const slotKey = Symbol('composition-test'); + + // Apple's runner state and Android's helper state. + expect(first.runtimeStates).toHaveLength(2); + expect(second.runtimeStates).toHaveLength(2); + // The Android handle scopes the slots the helper modules read. + const [, firstAndroid] = first.runtimeStates; + const [, secondAndroid] = second.runtimeStates; + const inFirst = firstAndroid!.run(() => helperRuntimeSlot(slotKey, () => ({}))); + const inSecond = secondAndroid!.run(() => helperRuntimeSlot(slotKey, () => ({}))); + expect(inFirst).not.toBe(inSecond); + expect(firstAndroid!.run(() => helperRuntimeSlot(slotKey, () => ({})))).toBe(inFirst); +}); diff --git a/src/daemon-composition.ts b/src/daemon-composition.ts index 13cc987ce0..94f1b5aa39 100644 --- a/src/daemon-composition.ts +++ b/src/daemon-composition.ts @@ -125,7 +125,15 @@ export const createDaemonRuntimeDeps: ComposeDaemonRuntimeDeps = async ( const { createDaemonPlatformServices } = await import('./platform-runtime-daemon-services.ts'); const { createClaimRecoveryGatewayFor } = await import('./platform-runtime-claim-recovery.ts'); + const { createAppleRunnerRuntimeState } = + await import('@agent-device/platform-apple/runner-owner'); + const { createAndroidHelperRuntimeState } = + await import('@agent-device/platform-android/adb-host'); return Object.freeze({ + runtimeStates: Object.freeze([ + createAppleRunnerRuntimeState(), + createAndroidHelperRuntimeState(), + ]), ownedProcessRecords, hostDiagnostics, ownerLifecycle: platformDaemonLifecycleOwners, diff --git a/src/daemon-entrypoint.test.ts b/src/daemon-entrypoint.test.ts index e28372b631..7e0304bf71 100644 --- a/src/daemon-entrypoint.test.ts +++ b/src/daemon-entrypoint.test.ts @@ -78,7 +78,6 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a const paths = resolveDaemonPaths(stateDir); const stdout: string[] = []; const stderr: string[] = []; - let exitCode: number | undefined; try { const runtime = await startDaemonRuntime({ @@ -90,10 +89,6 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: (chunk) => stderr.push(chunk) }, stdout: { write: (chunk) => stdout.push(chunk) }, }); @@ -111,9 +106,7 @@ test('daemon runtime starts HTTP transport in-process and shuts down cleanly', a assert.ok(fs.existsSync(paths.lockPath), 'daemon lock should be held while runtime is active'); await waitForHttpOk(`http://127.0.0.1:${runtime?.httpPort}/health`, 2_000); - await runtime?.shutdown(); - - assert.equal(exitCode, 0); + await runtime?.close(); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -125,7 +118,6 @@ test('daemon runtime publishes dual transport metadata', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-runtime-dual-'); const paths = resolveDaemonPaths(stateDir); const stdout: string[] = []; - let exitCode: number | undefined; try { const runtime = await startDaemonRuntime({ @@ -137,10 +129,6 @@ test('daemon runtime publishes dual transport metadata', async () => { AGENT_DEVICE_DAEMON_SERVER_MODE: 'dual', }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: (chunk) => stdout.push(chunk) }, }); @@ -156,9 +144,7 @@ test('daemon runtime publishes dual transport metadata', async () => { assert.equal(info.transport, 'dual'); await waitForHttpOk(`http://127.0.0.1:${runtime?.httpPort}/health`, 2_000); - await runtime?.shutdown(); - - assert.equal(exitCode, 0); + await runtime?.close(); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -193,8 +179,6 @@ test('daemon rejects unowned cloud artifacts over RPC', async () => { BROWSERSTACK_SESSION_DETAILS_ENDPOINT: `http://127.0.0.1:${providerPort}/sessions`, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -226,7 +210,7 @@ test('daemon rejects unowned cloud artifacts over RPC', async () => { assert.deepEqual(body.error?.data?.details, { reason: 'PROVIDER_SESSION_NOT_OWNED' }); assert.deepEqual(providerRequests, []); } finally { - await runtime?.shutdown(); + await runtime?.close(); await closeLoopbackServer(providerServer); fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -269,6 +253,10 @@ test('daemon entrypoint publishes HTTP metadata and cleans up on shutdown', asyn daemon.child.kill('SIGTERM'); const exited = await waitForProcessExit(pid, 5_000); assert.equal(exited, true); + // A clean signal close exits 0 and writes no unfinished-work line. + const closed = await daemon.wait; + assert.equal(closed.exitCode, 0); + assert.equal(closed.stderr, ''); assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); } finally { @@ -281,6 +269,42 @@ test('daemon entrypoint publishes HTTP metadata and cleans up on shutdown', asyn } }); +test('daemon entrypoint reaps itself when idle: exits cleanly, releases its registration and writes nothing unfinished', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-entrypoint-idle-'); + const paths = resolveDaemonPaths(stateDir); + const daemon = runCmdBackground( + process.execPath, + ['--experimental-strip-types', 'src/daemon.ts'], + { + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '300', + }, + allowFailure: true, + }, + ); + const pid = daemon.child.pid; + assert.ok(pid, 'daemon process should have a pid'); + + try { + // Nothing ever talks to it, so the idle reap, not a signal, is what closes it. + const result = await daemon.wait; + + assert.equal(result.exitCode, 0); + assert.equal(result.stderr, ''); + assert.equal(fs.existsSync(paths.infoPath), false); + assert.equal(fs.existsSync(paths.lockPath), false); + } finally { + if (isProcessAlive(pid)) { + daemon.child.kill('SIGKILL'); + await waitForProcessExit(pid, 2_000); + } + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + test('daemon runtime records startup device-claim reconciliation in daemon.log', async () => { // Regression: reconciliation ran before publishDaemonInfo, which truncates // daemon.log — so the event was written and then wiped, leaving nothing in @@ -290,7 +314,6 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', const claimsDir = mkdtempForTestSync('agent-device-daemon-prune-claims-'); const paths = resolveDaemonPaths(stateDir); const deviceKey = 'local:android:none:prune-log'; - let exitCode: number | undefined; try { fs.writeFileSync( @@ -320,10 +343,6 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', AGENT_DEVICE_CLAIMS_DIR: claimsDir, }, }), - exit: (code) => { - exitCode = code; - }, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -335,8 +354,7 @@ test('daemon runtime records startup device-claim reconciliation in daemon.log', assert.match(log, /"phase":"device_claim_reconcile"/); assert.match(log, /"reconciled":1/); - await runtime?.shutdown(); - assert.equal(exitCode, 0); + await runtime?.close(); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); fs.rmSync(claimsDir, { recursive: true, force: true }); @@ -444,8 +462,6 @@ test('startup sweep settles a foreign dead owner without touching a live same-na AGENT_DEVICE_CLAIMS_DIR: claimsDir, }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -474,7 +490,7 @@ test('startup sweep settles a foreign dead owner without touching a live same-na "the dead owner's orphaned recorder should be terminated by recovery", ); } finally { - await runtime?.shutdown().catch(() => {}); + await runtime?.close().catch(() => {}); try { orphanRecorder.kill('SIGKILL'); } catch {} diff --git a/src/daemon-runtime-limrun-public-http.test.ts b/src/daemon-runtime-limrun-public-http.test.ts index 9d44391629..d18886091f 100644 --- a/src/daemon-runtime-limrun-public-http.test.ts +++ b/src/daemon-runtime-limrun-public-http.test.ts @@ -19,8 +19,8 @@ const limrunIo = vi.hoisted(() => ({ const rootPorts = vi.hoisted(() => { const applicationLifecycle = { recoverStartupResources: vi.fn(async () => {}), - detachForDaemonShutdown: vi.fn(async () => {}), - finalizeDaemonShutdown: vi.fn(async () => {}), + detachForDaemonShutdown: vi.fn(async () => []), + finalizeDaemonShutdown: vi.fn(async () => []), }; return { providerComposition: vi.fn(async () => ({ @@ -106,8 +106,6 @@ test('public HTTP rejects Limrun uploaded-app listing and allocation before prov LIM_ANDROID_INSTANCE_ADB_URL: 'https://region.limrun.example/v1/android_test/adb', }, }), - exit: () => {}, - registerProcessHandlers: false, stderr: { write: () => {} }, stdout: { write: () => {} }, }); @@ -162,7 +160,7 @@ test('public HTTP rejects Limrun uploaded-app listing and allocation before prov assert.equal(rootPorts.createRequestPlatformProviders.mock.calls.length, 1); assert.equal(rootPorts.applicationLifecycle.recoverStartupResources.mock.calls.length, 1); } finally { - await runtime?.shutdown(); + await runtime?.close(); await limrun.shutdown(); fs.rmSync(paths.baseDir, { recursive: true, force: true }); if (runtime) diff --git a/src/daemon.ts b/src/daemon.ts index 264b1de596..5f34d5f54e 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -1,16 +1,44 @@ import { asAppError } from '@agent-device/kernel/errors'; -import { startDaemonRuntime } from '@agent-device/daemon'; +import { daemonStartupExitCode, reportDaemonClose, startDaemonRuntime } from '@agent-device/daemon'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; +// The executable owns the process: signals, fatal-error handlers and the exit code. The runtime +// only reports when it has closed, whoever closed it. // The root's composition enters as a factory behind a function-scoped `await import`: the server // calls it before it acquires the registration, so startup order is unchanged, and the eager // closure of this entry stays exactly the operational server's (ADR 0019). -void startDaemonRuntime({ - host: captureHostEnvironment(process), - composeDeps: async (input) => - (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), -}).catch((error) => { - const appErr = asAppError(error); - process.stderr.write(`Daemon error: ${appErr.message}\n`); - process.exit(1); +async function runDaemon(): Promise { + const runtime = await startDaemonRuntime({ + host: captureHostEnvironment(process), + composeDeps: async (input) => + (await import('./daemon-composition.ts')).createDaemonRuntimeDeps(input), + }); + let exitCode = 0; + const closeAndExit = (code: number, cause?: unknown): void => { + exitCode = Math.max(exitCode, code); + // A failed close is reported once, through `closed` below. + void runtime.close({ cause }).catch(() => {}); + }; + const failFatally = (error: unknown): void => { + process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); + closeAndExit(1, error); + }; + process.on('SIGINT', () => closeAndExit(0)); + process.on('SIGTERM', () => closeAndExit(0)); + process.on('SIGHUP', () => closeAndExit(0)); + process.on('uncaughtException', failFatally); + process.on('unhandledRejection', (reason) => + failFatally(reason instanceof Error ? reason : new Error(String(reason))), + ); + // Unfinished work is written for every close, whoever started it: a signal, the idle reap or a + // quiesce. The registration may be held on purpose, and this is the only record of why. + if ((await reportDaemonClose({ closed: runtime.closed, stderr: process.stderr })).failed) { + exitCode = 1; + } + process.exit(exitCode); +} + +runDaemon().catch((error) => { + process.stderr.write(`Daemon error: ${asAppError(error).message}\n`); + process.exit(daemonStartupExitCode(error)); }); diff --git a/src/platform-runtime-apple-application-tools.ts b/src/platform-runtime-apple-application-tools.ts index 35985fe3b0..b546344a86 100644 --- a/src/platform-runtime-apple-application-tools.ts +++ b/src/platform-runtime-apple-application-tools.ts @@ -114,11 +114,11 @@ export function createAppleApplicationTools(): AppleApplicationTools { dismissCloseAlerts: async (device, input) => await dismissMacOsCloseAlerts(device, input), detachRunnerSessionsForShutdown: async () => { const { detachIosRunnerSessionsForShutdown } = await loadRunnerOperations(); - await detachIosRunnerSessionsForShutdown(); + return await detachIosRunnerSessionsForShutdown(); }, finalizeRunnerSessionsForShutdown: async () => { const { stopAllIosRunnerSessions } = await loadRunnerOperations(); - await stopAllIosRunnerSessions(); + return await stopAllIosRunnerSessions(); }, }); } diff --git a/src/platform-runtime-application-resources.test.ts b/src/platform-runtime-application-resources.test.ts index 4ad3eee04e..ea728b5cdf 100644 --- a/src/platform-runtime-application-resources.test.ts +++ b/src/platform-runtime-application-resources.test.ts @@ -43,8 +43,8 @@ test('startup recovery reaches Android mechanics only after validated marker evi }); test('daemon shutdown phases delegate to the Apple runner owner', async () => { - const detachRunnerSessionsForShutdown = vi.fn(async () => {}); - const finalizeRunnerSessionsForShutdown = vi.fn(async () => {}); + const detachRunnerSessionsForShutdown = vi.fn(async () => ['ios-sim']); + const finalizeRunnerSessionsForShutdown = vi.fn(async () => ['ios-sim']); const resources = lifecycle({ hasTestImeRecoveryEvidence: async () => false, recoverTestImeStartup: async () => {}, diff --git a/src/platform-runtime-gateway.ts b/src/platform-runtime-gateway.ts index 54feedf435..867cb313ea 100644 --- a/src/platform-runtime-gateway.ts +++ b/src/platform-runtime-gateway.ts @@ -74,8 +74,8 @@ export function createComposedPlatformRuntimeGateway(options: { }; // Which durable resources exist is the host's composition problem; the gateway only owns the // once-per-process shape of each phase and keeps the host load lazy. - let lifecycleDetach: Promise | undefined; - let lifecycleFinalize: Promise | undefined; + let lifecycleDetach: Promise | undefined; + let lifecycleFinalize: Promise | undefined; const applicationLifecycle = Object.freeze({ recoverStartupResources: async (input: Readonly<{ stateDir: string }>) => { await runStartupRecoveryFence(input.stateDir, async () => { @@ -86,13 +86,13 @@ export function createComposedPlatformRuntimeGateway(options: { lifecycleDetach ??= loadHost().then( async (host) => await host.applicationResources.detachForDaemonShutdown(), ); - await lifecycleDetach; + return await lifecycleDetach; }, finalizeDaemonShutdown: async () => { lifecycleFinalize ??= loadHost().then( async (host) => await host.applicationResources.finalizeDaemonShutdown(), ); - await lifecycleFinalize; + return await lifecycleFinalize; }, }); const registerOwner = (owner: PlatformRuntimeOwner) => { diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts index 8ef20ecdb5..7269651fd5 100644 --- a/src/provider-device-runtimes.test.ts +++ b/src/provider-device-runtimes.test.ts @@ -73,7 +73,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest interval: 10, timeout: 2_000, }); - await firstDaemon.runtime.shutdown(); + await firstDaemon.runtime.close(); assert.ok(limrun.iosDisconnect.mock.calls.length > 0); const pendingPath = path.join(stateDir, 'expired-provider-leases.json'); @@ -100,7 +100,7 @@ test('Limrun retries an expired provider lease persisted by a daemon before rest if (restartedDaemon) { await restartedDaemon.close(); } else { - await firstDaemon?.runtime.shutdown(); + await firstDaemon?.runtime.close(); fs.rmSync(stateDir, { recursive: true, force: true }); } } diff --git a/test/integration/provider-scenarios/harness.ts b/test/integration/provider-scenarios/harness.ts index e9650ca16a..a0c9d400a6 100644 --- a/test/integration/provider-scenarios/harness.ts +++ b/test/integration/provider-scenarios/harness.ts @@ -212,10 +212,8 @@ export async function createProviderScenarioHarness( try { runtime = await startDaemonRuntime({ host: createTestHostEnvironment({ env }), - registerProcessHandlers: false, stdout: { write: () => undefined }, stderr: { write: () => undefined }, - exit: () => undefined, composeDeps, }); } catch (error) { @@ -289,9 +287,9 @@ export async function createProviderScenarioHarness( if (scopedAppleToolProvider) { const { withAppleToolProvider } = await import('@agent-device/platform-apple/tool-provider'); - await withAppleToolProvider(scopedAppleToolProvider, async () => await runtime?.shutdown()); + await withAppleToolProvider(scopedAppleToolProvider, async () => await runtime?.close()); } else { - await runtime?.shutdown(); + await runtime?.close(); } await removeProviderScenarioTempDir(testRoot); },