diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d35afaa3b..3a88cf2f2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -366,7 +366,7 @@ jobs: with: { gate: integration-node } - name: Run seeded concurrency torture lane (fast PR sweep) - # #1416's nightly torture lane lives under test/integration/nightly/, out + # #1416's nightly torture lane lives under packages/daemon/test/nightly/, out # of the test:integration:node glob, so this is a *deliberate* fast PR # sweep (TORTURE_RUNS default 128 seeds, ~sub-second) — not an accidental # glob inclusion. The Concurrency Torture Nightly workflow sweeps a much diff --git a/package.json b/package.json index e73efd8a74..237c95bd81 100644 --- a/package.json +++ b/package.json @@ -208,7 +208,7 @@ "test:integration:node": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test --experimental-test-isolation=none test/integration/*.test.ts", "test:integration:macos-coverage": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-macos-coverage.test.ts", "test:integration": "pnpm test:integration:node && pnpm test:integration:provider", - "test:concurrency-torture": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/nightly/concurrency-torture.test.ts", + "test:concurrency-torture": "node --experimental-strip-types scripts/node-test-tmpdir.ts --test packages/daemon/test/nightly/concurrency-torture.test.ts", "test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator", "test:replay:ios-device": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device", "test:replay:android": "node --experimental-strip-types src/bin.ts test test/integration/replays/android/emulator", diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 940463268f..0a76e7357c 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -56,34 +56,10 @@ "types": "./src/core/context.ts", "default": "./src/core/context.ts" }, - "./testing/daemon-command-registry": { - "types": "./src/core/daemon-command-registry.ts", - "default": "./src/core/daemon-command-registry.ts" - }, - "./testing/device/device-claim-conflict": { - "types": "./src/resources/device/device-claim-conflict.ts", - "default": "./src/resources/device/device-claim-conflict.ts" - }, - "./testing/handlers/session-app-deployment": { - "types": "./src/sessions/handlers/session-app-deployment.ts", - "default": "./src/sessions/handlers/session-app-deployment.ts" - }, "./testing/interaction/index": { "types": "./src/execution/interaction/index.ts", "default": "./src/execution/interaction/index.ts" }, - "./testing/interaction/internal/interaction-touch-response": { - "types": "./src/execution/interaction/internal/interaction-touch-response.ts", - "default": "./src/execution/interaction/internal/interaction-touch-response.ts" - }, - "./testing/interactor-resolution": { - "types": "./src/core/interactor-resolution.ts", - "default": "./src/core/interactor-resolution.ts" - }, - "./testing/ios-app-session-hint": { - "types": "./src/core/ios-app-session-hint.ts", - "default": "./src/core/ios-app-session-hint.ts" - }, "./testing/lease-registry": { "types": "./src/resources/lease-registry.ts", "default": "./src/resources/lease-registry.ts" @@ -100,42 +76,18 @@ "types": "./src/core/ref-frame.ts", "default": "./src/core/ref-frame.ts" }, - "./testing/request-binding": { - "types": "./src/server/request-binding.ts", - "default": "./src/server/request-binding.ts" - }, "./testing/request-dispatch-ledger": { "types": "./src/core/request-dispatch-ledger.ts", "default": "./src/core/request-dispatch-ledger.ts" }, - "./testing/request-execution-scope": { - "types": "./src/server/request-execution-scope.ts", - "default": "./src/server/request-execution-scope.ts" - }, "./testing/request-finalization": { "types": "./src/server/request-finalization.ts", "default": "./src/server/request-finalization.ts" }, - "./testing/request-router": { - "types": "./src/server/request-router.ts", - "default": "./src/server/request-router.ts" - }, "./testing/request-runtime-binding": { "types": "./src/core/request-runtime-binding.ts", "default": "./src/core/request-runtime-binding.ts" }, - "./testing/response-views": { - "types": "./src/server/response-views.ts", - "default": "./src/server/response-views.ts" - }, - "./testing/scroll-until": { - "types": "./src/execution/scroll-until.ts", - "default": "./src/execution/scroll-until.ts" - }, - "./testing/selector-match-errors": { - "types": "./src/execution/selector-match-errors.ts", - "default": "./src/execution/selector-match-errors.ts" - }, "./testing/server/http-server": { "types": "./src/server/http/http-server.ts", "default": "./src/server/http/http-server.ts" @@ -144,14 +96,6 @@ "types": "./src/core/session-action-recorder.ts", "default": "./src/core/session-action-recorder.ts" }, - "./testing/session-observability/__tests__/*": { - "types": "./src/sessions/session-observability/__tests__/*.ts", - "default": "./src/sessions/session-observability/__tests__/*.ts" - }, - "./testing/session-recovery-hints": { - "types": "./src/core/session-recovery-hints.ts", - "default": "./src/core/session-recovery-hints.ts" - }, "./testing/session-snapshot": { "types": "./src/execution/session-snapshot.ts", "default": "./src/execution/session-snapshot.ts" @@ -163,10 +107,6 @@ "./testing/session-store": { "types": "./src/core/session-store.ts", "default": "./src/core/session-store.ts" - }, - "./testing/web-session-names": { - "types": "./src/sessions/web-session-names.ts", - "default": "./src/sessions/web-session-names.ts" } } } diff --git a/packages/daemon/src/core/__tests__/session-action-recorder.test.ts b/packages/daemon/src/core/__tests__/session-action-recorder.test.ts index d690110be7..97fb0b2b6d 100644 --- a/packages/daemon/src/core/__tests__/session-action-recorder.test.ts +++ b/packages/daemon/src/core/__tests__/session-action-recorder.test.ts @@ -7,7 +7,7 @@ import { isSessionRecording } from '../session-script-publication-capability.ts' * (`selector-recording.test.ts` covers which commands set `interactiveObservation`). */ import { test, expect } from 'vitest'; -import { recordActionEntry } from '../session-action-recorder.ts'; +import { recordActionEntry, type RecordActionEntry } from '../session-action-recorder.ts'; import { authoringPublication, makeIosSession } from '../../__tests__/support/session-factories.ts'; test('an observation-only action is excluded while repair-armed and no --record is given', () => { @@ -64,6 +64,38 @@ test('an observation-only action records normally outside a repair-armed session expect(session.actions.map((a) => a.command)).toEqual(['is']); }); +test('ordinary action history preserves authored commands, positionals, and replay flags', () => { + const session = makeIosSession('default'); + const entries = [ + { command: 'hover', positionals: ['@e4'], flags: {}, result: {} }, + { command: 'click', positionals: ['@e4'], flags: {}, result: {} }, + { + command: 'fill', + positionals: ['@e3', 'qa@example.test'], + flags: { delayMs: 1 }, + result: {}, + }, + { command: 'type', positionals: [' ok'], flags: {}, result: {} }, + { command: 'pan', positionals: ['195', '320'], flags: { pointerCount: 2 }, result: {} }, + { command: 'tv-remote', positionals: ['select'], flags: {}, result: {} }, + { command: 'fold', positionals: [], flags: { keyframes: 'fold.json' }, result: {} }, + { + command: 'trigger-app-event', + positionals: ['screenshot_taken'], + flags: {}, + result: {}, + }, + { command: 'back', positionals: [], flags: {}, result: {} }, + { command: 'home', positionals: [], flags: {}, result: {} }, + ] satisfies RecordActionEntry[]; + + for (const entry of entries) recordActionEntry(session, entry); + + expect( + session.actions.map(({ command, positionals, flags }) => ({ command, positionals, flags })), + ).toEqual(entries.map(({ command, positionals, flags }) => ({ command, positionals, flags }))); +}); + test('a mutating action is never excluded, repair-armed or not', () => { const session = makeIosSession('default', { scriptPublication: { @@ -123,6 +155,19 @@ test('--no-record still takes precedence over an observation-only action, repair expect(session.actions).toHaveLength(0); }); +test('--no-record suppresses a mutating coordinate press from action history', () => { + const session = makeIosSession('default'); + const action = recordActionEntry(session, { + command: 'press', + positionals: ['50', '60'], + flags: { noRecord: true }, + result: { x: 50, y: 60 }, + }); + + expect(action).toBeUndefined(); + expect(session.actions).toHaveLength(0); +}); + test('parameterized fills keep literals out of recording state with deterministic placeholders', () => { const session = makeIosSession('default'); const password = 'literal-password-1348'; diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-response-contract.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-response-contract.test.ts new file mode 100644 index 0000000000..ebe850e02c --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-response-contract.test.ts @@ -0,0 +1,29 @@ +import { expect, test } from 'vitest'; +import type { PressCommandResult } from '@agent-device/contracts/interaction'; +import { buildInteractionResponseData } from './interaction-touch-response.ts'; + +test('the response builder keeps capture evidence out of serialized touch payloads', () => { + const node = { ref: 'e1', index: 0, type: 'Button', label: 'Continue' }; + const result = { + kind: 'selector', + point: { x: 100, y: 200 }, + target: { kind: 'selector', selector: 'label=Continue' }, + node, + selectorChain: ['label=Continue'], + preAction: { nodes: [node] }, + backendResult: { ref: 'e1' }, + targetEvidence: { id: 'target-evidence-private' }, + } as PressCommandResult & { targetEvidence: { id: string } }; + + const payloads = buildInteractionResponseData({ + source: { kind: 'runtime', result }, + referenceFrame: undefined, + }); + + expect(payloads.recordedTarget).toEqual({ node, preActionNodes: [node] }); + for (const payload of [payloads.result, payloads.responseData]) { + expect(payload).not.toHaveProperty('node'); + expect(payload).not.toHaveProperty('preAction'); + expect(payload).not.toHaveProperty('targetEvidence'); + } +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts index 885e3e960a..0f02dd504a 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-routing.test.ts @@ -13,6 +13,7 @@ import { mockHoverRef, mockLongPressPoint, mockTapPoint, + mockTypeText, resetGetRuntimeFixture, runtimeBindingSpies, } from '../../__tests__/interaction-get-runtime-fixture.ts'; @@ -237,7 +238,69 @@ test('hover @ref on web dispatches through the provider hoverRef route, not coor }); expect(mockHoverRef).toHaveBeenCalledWith(expect.objectContaining({ ref: '@e1' })); expect(mockHoverPoint).not.toHaveBeenCalled(); - expect(sessionStore.get(sessionName)?.actions[0]?.command).toBe('hover'); + const hoverAction = sessionStore.get(sessionName)?.actions[0]; + expect(hoverAction?.command).toBe('hover'); + expect(hoverAction?.result).toMatchObject({ ref: 'e1', gesture: 'hover' }); + expect(hoverAction?.result).not.toHaveProperty('x'); + expect(hoverAction?.result).not.toHaveProperty('y'); + + const typeResponse = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'type', + positionals: [' ok'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(typeResponse?.ok).toBe(true); + expect(mockTypeText).toHaveBeenCalledOnce(); + expect(sessionStore.get(sessionName)?.actions.at(-1)).toMatchObject({ + command: 'type', + positionals: [' ok'], + }); +}); + +test('click @ref on web records the ref result without fabricated coordinates', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'click-ref'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + session.snapshot = { + nodes: attachRefs([{ index: 0, role: 'link', label: 'Second message', enabled: true }]), + createdAt: Date.now(), + backend: 'web', + }; + sessionStore.publish(sessionName, session); + + const response = await handleInteractionCommands({ + createCommandSurface: touchCreateCommandSurface, + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toMatchObject({ ok: true, data: { ref: 'e1' } }); + const action = sessionStore.get(sessionName)?.actions[0]; + expect(action?.command).toBe('click'); + expect(action?.positionals).toEqual(['@e1']); + expect(action?.result).toMatchObject({ ref: 'e1' }); + expect(action?.result).not.toHaveProperty('x'); + expect(action?.result).not.toHaveProperty('y'); }); test('hover selector on web resolves the target and dispatches coordinate hover', async () => { diff --git a/packages/daemon/src/server/__tests__/response-views.test.ts b/packages/daemon/src/server/__tests__/response-views.test.ts index 97eb9e30f1..2cabe11bba 100644 --- a/packages/daemon/src/server/__tests__/response-views.test.ts +++ b/packages/daemon/src/server/__tests__/response-views.test.ts @@ -264,7 +264,7 @@ test('find get-text digest keeps ref + text, drops the verbose node', () => { expect('node' in digest).toBe(false); }); -test('a text read keeps every OTHER cheap field (e.g. warning) while dropping the node', () => { +test('selector text digest preserves a recovery warning while dropping the verbose node', () => { const digest = findView!( { ref: '@e7', diff --git a/test/integration/provider-scenarios/daemon-http-server.test.ts b/packages/daemon/src/server/http/http-server-provider-scenarios.test.ts similarity index 98% rename from test/integration/provider-scenarios/daemon-http-server.test.ts rename to packages/daemon/src/server/http/http-server-provider-scenarios.test.ts index aa4b22b579..83a9d4cfd1 100644 --- a/test/integration/provider-scenarios/daemon-http-server.test.ts +++ b/packages/daemon/src/server/http/http-server-provider-scenarios.test.ts @@ -4,7 +4,6 @@ import assert from 'node:assert/strict'; import crypto from 'node:crypto'; import fs from 'node:fs'; import http from 'node:http'; -import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; @@ -12,9 +11,9 @@ import { cleanupUploadedArtifact, prepareUploadedArtifact, trackDownloadableArtifact, -} from '@agent-device/daemon/testing/artifact-tracking'; +} from '../../resources/artifact-tracking.ts'; import { DAEMON_RPC_PROTOCOL_VERSION } from '@agent-device/contracts/daemon-http'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; +import { createDaemonHttpServer } from './http-server.ts'; import { emitRequestProgress, getRequestSignal, @@ -24,9 +23,9 @@ import { import { closeLoopbackServer, listenOnLoopback, + mkdtempForTestSync, skipWhenLoopbackUnavailable, } from '@agent-device/test-kit'; -import { restoreEnv } from './harness.ts'; type RpcResponse = { status: number; @@ -476,7 +475,7 @@ test('Provider-backed integration daemon HTTP server accepts uploads and streams return; } - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-http-server-')); + const root = mkdtempForTestSync('agent-device-http-server-'); const downloadablePath = path.join(root, 'screen.png'); fs.writeFileSync(downloadablePath, 'png-binary'); const artifactId = trackDownloadableArtifact({ @@ -526,7 +525,6 @@ test('Provider-backed integration daemon HTTP server accepts uploads and streams assert.equal(rejectedUpload.status, 401); } finally { await closeLoopbackServer(server); - fs.rmSync(root, { recursive: true, force: true }); } }); @@ -759,7 +757,7 @@ test('Provider-backed integration daemon HTTP auth hook can scope tenants and re return; } - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-http-auth-hook-')); + const root = mkdtempForTestSync('agent-device-http-auth-hook-'); const hookPath = path.join(root, 'auth-hook.mjs'); fs.writeFileSync( hookPath, @@ -817,8 +815,8 @@ test('Provider-backed integration daemon HTTP auth hook can scope tenants and re if (server) { await closeLoopbackServer(server); } - restoreEnv('AGENT_DEVICE_HTTP_AUTH_HOOK', previousHook); - fs.rmSync(root, { recursive: true, force: true }); + if (previousHook === undefined) delete process.env.AGENT_DEVICE_HTTP_AUTH_HOOK; + else process.env.AGENT_DEVICE_HTTP_AUTH_HOOK = previousHook; } }); diff --git a/packages/daemon/src/sessions/session-observability/__tests__/session-audio.coverage.ts b/packages/daemon/src/sessions/session-observability/__tests__/session-audio.coverage.ts deleted file mode 100644 index 5bbc3e24b1..0000000000 --- a/packages/daemon/src/sessions/session-observability/__tests__/session-audio.coverage.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { PUBLIC_COMMANDS as C } from '@agent-device/command-registry/catalog'; - -export const ANDROID_AUDIO_CONTRACT_EVIDENCE = { - commands: [C.audio], - owner: 'daemon/session-observability', - testName: 'audio probe starts host helper for Android emulator audio', -} as const; diff --git a/packages/daemon/src/sessions/session-observability/internal/__tests__/session-audio.test.ts b/packages/daemon/src/sessions/session-observability/internal/__tests__/session-audio.test.ts index 30a61773f8..c1f58a43bf 100644 --- a/packages/daemon/src/sessions/session-observability/internal/__tests__/session-audio.test.ts +++ b/packages/daemon/src/sessions/session-observability/internal/__tests__/session-audio.test.ts @@ -27,7 +27,6 @@ import { createAudioProbeAdmissionLedger } from '@agent-device/capture-kit/audio import { audioProbeDurableResource } from '@agent-device/capture-kit/audio-probe-session-resource'; import type { SessionStore } from '../../../../core/session-store.ts'; import { handleSessionObservabilityCommands } from '../../index.ts'; -import { ANDROID_AUDIO_CONTRACT_EVIDENCE } from '../../__tests__/session-audio.coverage.ts'; import { encodeHostAudioProbeDescriptor } from '@agent-device/capture-kit/audio-probe-descriptor-fixtures'; async function runAudio( @@ -299,7 +298,7 @@ test('audio probe starts host helper for iOS simulator audio', async () => { assert.ok(sessionStore.get('ios-sim')?.audioProbe); }); -test(ANDROID_AUDIO_CONTRACT_EVIDENCE.testName, async () => { +test('audio probe starts host helper for Android emulator audio', async () => { const sessionStore = makeSessionStore('agent-device-session-audio-'); const session = makeAndroidSession('android'); sessionStore.publish('android', session); diff --git a/test/integration/nightly/concurrency-torture.test.ts b/packages/daemon/test/nightly/concurrency-torture.test.ts similarity index 100% rename from test/integration/nightly/concurrency-torture.test.ts rename to packages/daemon/test/nightly/concurrency-torture.test.ts diff --git a/test/integration/nightly/concurrency-torture/bindings.ts b/packages/daemon/test/nightly/concurrency-torture/bindings.ts similarity index 94% rename from test/integration/nightly/concurrency-torture/bindings.ts rename to packages/daemon/test/nightly/concurrency-torture/bindings.ts index 848fd87d28..4da08c59dd 100644 --- a/test/integration/nightly/concurrency-torture/bindings.ts +++ b/packages/daemon/test/nightly/concurrency-torture/bindings.ts @@ -20,10 +20,10 @@ import type { CommandFlags, DaemonWireRequest } from '@agent-device/contracts/command'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { SessionRef } from '@agent-device/daemon/testing/session-state'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; -import { resolveRequestExecutionLockPlan } from '@agent-device/daemon/testing/request-binding'; -import { shouldLockSessionExecution } from '@agent-device/daemon/testing/daemon-command-registry'; +import type { SessionRef } from '../../../src/core/session-state.ts'; +import type { SessionStore } from '../../../src/core/session-store.ts'; +import { resolveRequestExecutionLockPlan } from '../../../src/server/request-binding.ts'; +import { shouldLockSessionExecution } from '../../../src/core/daemon-command-registry.ts'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { withTestDeviceInventoryProvider as withDeviceInventoryProvider } from '@agent-device/device-selection/inventory-gateways-fixtures'; diff --git a/test/integration/nightly/concurrency-torture/claim-registry.ts b/packages/daemon/test/nightly/concurrency-torture/claim-registry.ts similarity index 100% rename from test/integration/nightly/concurrency-torture/claim-registry.ts rename to packages/daemon/test/nightly/concurrency-torture/claim-registry.ts diff --git a/test/integration/nightly/concurrency-torture/deterministic-scheduler.ts b/packages/daemon/test/nightly/concurrency-torture/deterministic-scheduler.ts similarity index 100% rename from test/integration/nightly/concurrency-torture/deterministic-scheduler.ts rename to packages/daemon/test/nightly/concurrency-torture/deterministic-scheduler.ts diff --git a/test/integration/nightly/concurrency-torture/envelope.ts b/packages/daemon/test/nightly/concurrency-torture/envelope.ts similarity index 77% rename from test/integration/nightly/concurrency-torture/envelope.ts rename to packages/daemon/test/nightly/concurrency-torture/envelope.ts index dec5abceb6..0eab2eed33 100644 --- a/test/integration/nightly/concurrency-torture/envelope.ts +++ b/packages/daemon/test/nightly/concurrency-torture/envelope.ts @@ -1,20 +1,10 @@ -// Standard scheduled-lane artifact envelope (#1430) for the concurrency torture -// lane (#1416). Builds the shared `LaneEnvelope` from scripts/lib/lane-envelope.ts -// so the #1430 health watcher parses one dialect, not a lane-local one; the -// lane's seed range / run count ride in the typed `data` payload, and the lane -// source hash maps onto the generic `configHash`. Written when TORTURE_ENVELOPE -// names an output path (set by the nightly workflow, which uploads it as an -// artifact); a no-op otherwise. +// Standard scheduled-lane artifact envelope (#1430) for the concurrency torture lane (#1416). import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { - laneEnvelope, - type LaneEnvelope, - type LaneResult, -} from '../../../../scripts/lib/lane-envelope.ts'; +import { laneEnvelope, type LaneEnvelope, type LaneResult } from '@agent-device/test-kit'; const LANE_ID = 'concurrency-torture'; diff --git a/test/integration/nightly/concurrency-torture/harness.ts b/packages/daemon/test/nightly/concurrency-torture/harness.ts similarity index 98% rename from test/integration/nightly/concurrency-torture/harness.ts rename to packages/daemon/test/nightly/concurrency-torture/harness.ts index 13d7718bdc..4d90acfb3e 100644 --- a/test/integration/nightly/concurrency-torture/harness.ts +++ b/packages/daemon/test/nightly/concurrency-torture/harness.ts @@ -26,9 +26,9 @@ import os from 'node:os'; import path from 'node:path'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; -import { SessionStore } from '@agent-device/daemon/testing/session-store'; -import type { SessionState } from '@agent-device/daemon/testing/session-state'; +import { LeaseRegistry } from '../../../src/resources/lease-registry.ts'; +import { SessionStore } from '../../../src/core/session-store.ts'; +import type { SessionState } from '../../../src/core/session-state.ts'; import { AppError } from '@agent-device/kernel/errors'; import { makePrng, type Prng } from './prng.ts'; diff --git a/test/integration/nightly/concurrency-torture/invariants.ts b/packages/daemon/test/nightly/concurrency-torture/invariants.ts similarity index 97% rename from test/integration/nightly/concurrency-torture/invariants.ts rename to packages/daemon/test/nightly/concurrency-torture/invariants.ts index 610c8c6863..2a830e39a8 100644 --- a/test/integration/nightly/concurrency-torture/invariants.ts +++ b/packages/daemon/test/nightly/concurrency-torture/invariants.ts @@ -10,8 +10,8 @@ // - no cross-session bleed: each stored session carries exactly its own // lease/claim/device, never a concurrent session's -import type { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; +import type { LeaseRegistry } from '../../../src/resources/lease-registry.ts'; +import type { SessionStore } from '../../../src/core/session-store.ts'; import type { SessionInstance } from './bindings.ts'; diff --git a/test/integration/nightly/concurrency-torture/prng.ts b/packages/daemon/test/nightly/concurrency-torture/prng.ts similarity index 100% rename from test/integration/nightly/concurrency-torture/prng.ts rename to packages/daemon/test/nightly/concurrency-torture/prng.ts diff --git a/test/integration/nightly/concurrency-torture/real-scope-serialization.ts b/packages/daemon/test/nightly/concurrency-torture/real-scope-serialization.ts similarity index 84% rename from test/integration/nightly/concurrency-torture/real-scope-serialization.ts rename to packages/daemon/test/nightly/concurrency-torture/real-scope-serialization.ts index 8644b84ade..9b58374a1f 100644 --- a/test/integration/nightly/concurrency-torture/real-scope-serialization.ts +++ b/packages/daemon/test/nightly/concurrency-torture/real-scope-serialization.ts @@ -2,9 +2,9 @@ import type { CommandFlags, DaemonWireRequest } from '@agent-device/contracts/command'; import { - composedPlatformServicesFixture, - composedClaimRecoveryGatewayFixture, -} from '../../../../src/__tests__/daemon-composition/platform-services-fixture.ts'; + daemonPlatformServicesFixture, + daemonClaimRecoveryGatewayFixture, +} from '../../../src/__tests__/support/platform-services-fixture.ts'; // (#1416). The seeded sweep models the mutex GRANT (a seed cannot reproduce // `withKeyedLock`'s native microtask hand-off), so on its own it could stay // green if the PRODUCTION lock APPLICATION path regressed. This guard closes @@ -21,9 +21,9 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { createRequestExecutionScope } from '@agent-device/daemon/testing/request-execution-scope'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; -import { SessionStore } from '@agent-device/daemon/testing/session-store'; +import { createRequestExecutionScope } from '../../../src/server/request-execution-scope.ts'; +import { LeaseRegistry } from '../../../src/resources/lease-registry.ts'; +import { SessionStore } from '../../../src/core/session-store.ts'; import { withTestDeviceInventoryProvider as withDeviceInventoryProvider } from '@agent-device/device-selection/inventory-gateways-fixtures'; import { DEVICE_POOL } from './bindings.ts'; @@ -69,8 +69,8 @@ export async function measureRealScopeMaxOverlap(concurrency: number): Promise createRequestExecutionScope({ - platformServices: composedPlatformServicesFixture(), - claimRecoveryGateway: composedClaimRecoveryGatewayFixture(), + platformServices: daemonPlatformServicesFixture(), + claimRecoveryGateway: daemonClaimRecoveryGatewayFixture(), req: openRequest(`real-${i}`, device.id), sessionStore, leaseRegistry, diff --git a/packages/remote/package.json b/packages/remote/package.json index 8e2d265197..a9a7f4f239 100644 --- a/packages/remote/package.json +++ b/packages/remote/package.json @@ -38,6 +38,8 @@ } }, "devDependencies": { - "@agent-device/test-kit": "workspace:*" + "@agent-device/test-kit": "workspace:*", + "@types/tar-stream": "^3.1.4", + "tar-stream": "^3.2.0" } } diff --git a/test/integration/provider-scenarios/remote-test-artifacts-directory.test.ts b/packages/remote/src/__tests__/daemon-artifacts-directory.test.ts similarity index 72% rename from test/integration/provider-scenarios/remote-test-artifacts-directory.test.ts rename to packages/remote/src/__tests__/daemon-artifacts-directory.test.ts index 05adb785e5..2399f16207 100644 --- a/test/integration/provider-scenarios/remote-test-artifacts-directory.test.ts +++ b/packages/remote/src/__tests__/daemon-artifacts-directory.test.ts @@ -1,26 +1,17 @@ -/** - * #2246: the download half of the fix. The daemon's artifact transport already tar's a - * directory artifact on the fly (`artifact-tracking.ts`'s `ensureDirectoryArchive`) — this test - * is the first to exercise that path all the way through `downloadRemoteArtifact` on the client, - * which used to only ever write a response body straight to one file (screenshot/recording). - * `test`'s suite artifacts are a whole directory tree, so the client must detect the - * `application/gzip` directory-archive response and extract it instead. - */ import assert from 'node:assert/strict'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; +import { downloadRemoteArtifact } from '../daemon-artifacts.ts'; import { - cleanupDownloadableArtifact, - trackDownloadableArtifact, -} from '@agent-device/daemon/testing/artifact-tracking'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { downloadRemoteArtifact } from '@agent-device/remote/daemon-artifacts'; + createArtifactDownloadServer, + createDirectoryArtifactArchive, +} from './support/artifact-download-fixtures.ts'; import { closeLoopbackServer, listenOnLoopback, + mkdtempForTestSync, skipWhenLoopbackUnavailable, } from '@agent-device/test-kit'; @@ -30,7 +21,7 @@ test('downloadRemoteArtifact extracts a directory artifact into the caller-local if (await skipWhenLoopbackUnavailable(t, 'remote test-artifacts directory download coverage')) return; - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-remote-test-artifacts-')); + const tempDir = mkdtempForTestSync('agent-device-remote-test-artifacts-'); // Mirrors what the daemon actually produces: `//...`. const daemonSuiteDir = path.join(tempDir, 'daemon-side', SUITE_INVOCATION_ID); const attemptDir = path.join(daemonSuiteDir, 'qa-flows__open-grid-settings.ad', 'attempt-1'); @@ -50,17 +41,9 @@ test('downloadRemoteArtifact extracts a directory artifact into the caller-local fs.mkdirSync(callerArtifactsRoot, { recursive: true }); fs.writeFileSync(path.join(callerArtifactsRoot, 'previous-run-marker.txt'), 'keep me'); - const artifactId = trackDownloadableArtifact({ - artifactPath: daemonSuiteDir, - artifactType: 'test-artifacts', - fileName: SUITE_INVOCATION_ID, - }); - const server = await createDaemonHttpServer({ - token: 'daemon-token', - handleRequest: async () => { - throw new Error('not exercised: this test only hits the /artifacts route'); - }, - }); + const artifactId = 'directory-artifact'; + const archive = await createDirectoryArtifactArchive(daemonSuiteDir); + const server = createArtifactDownloadServer(new Map([[artifactId, archive]])); try { const port = await listenOnLoopback(server); @@ -100,7 +83,6 @@ test('downloadRemoteArtifact extracts a directory artifact into the caller-local 'keep me', ); } finally { - cleanupDownloadableArtifact(artifactId); await closeLoopbackServer(server); fs.rmSync(tempDir, { recursive: true, force: true }); } @@ -110,19 +92,12 @@ test('a failed directory download leaves the caller-local artifacts root untouch if (await skipWhenLoopbackUnavailable(t, 'remote test-artifacts directory download coverage')) return; - const tempDir = fs.mkdtempSync( - path.join(os.tmpdir(), 'agent-device-remote-test-artifacts-fail-'), - ); + const tempDir = mkdtempForTestSync('agent-device-remote-test-artifacts-fail-'); const callerArtifactsRoot = path.join(tempDir, 'remote-device-artifacts', 'ad-test'); fs.mkdirSync(callerArtifactsRoot, { recursive: true }); fs.writeFileSync(path.join(callerArtifactsRoot, 'previous-run-marker.txt'), 'keep me'); - const server = await createDaemonHttpServer({ - token: 'daemon-token', - handleRequest: async () => { - throw new Error('not exercised: this test only hits the /artifacts route'); - }, - }); + const server = createArtifactDownloadServer(new Map()); try { const port = await listenOnLoopback(server); @@ -157,9 +132,7 @@ test('a directory artifact containing a symlink is rejected, not extracted', asy if (await skipWhenLoopbackUnavailable(t, 'remote test-artifacts directory download coverage')) return; - const tempDir = fs.mkdtempSync( - path.join(os.tmpdir(), 'agent-device-remote-test-artifacts-evil-'), - ); + const tempDir = mkdtempForTestSync('agent-device-remote-test-artifacts-evil-'); // A remote (possibly compromised) daemon returning an archive with a symlink entry — the // real-world shape `extractArchiveSafely` guards against, since raw `tar xzf` would otherwise // follow it wherever it points on extraction. @@ -177,17 +150,9 @@ test('a directory artifact containing a symlink is rejected, not extracted', asy ); fs.mkdirSync(callerArtifactsRoot, { recursive: true }); - const artifactId = trackDownloadableArtifact({ - artifactPath: daemonSuiteDir, - artifactType: 'test-artifacts', - fileName: SUITE_INVOCATION_ID, - }); - const server = await createDaemonHttpServer({ - token: 'daemon-token', - handleRequest: async () => { - throw new Error('not exercised: this test only hits the /artifacts route'); - }, - }); + const artifactId = 'unsafe-directory-artifact'; + const archive = await createDirectoryArtifactArchive(daemonSuiteDir); + const server = createArtifactDownloadServer(new Map([[artifactId, archive]])); try { const port = await listenOnLoopback(server); @@ -207,7 +172,6 @@ test('a directory artifact containing a symlink is rejected, not extracted', asy assert.equal(fs.existsSync(path.join(callerArtifactsRoot, SUITE_INVOCATION_ID)), false); } finally { - cleanupDownloadableArtifact(artifactId); await closeLoopbackServer(server); fs.rmSync(tempDir, { recursive: true, force: true }); } diff --git a/packages/remote/src/__tests__/support/artifact-download-fixtures.ts b/packages/remote/src/__tests__/support/artifact-download-fixtures.ts new file mode 100644 index 0000000000..d6be695e1d --- /dev/null +++ b/packages/remote/src/__tests__/support/artifact-download-fixtures.ts @@ -0,0 +1,48 @@ +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import * as tar from 'tar-stream'; + +export async function createDirectoryArtifactArchive(directoryPath: string): Promise { + const pack = tar.pack(); + const rootName = path.basename(directoryPath); + const visit = async (currentPath: string, archivePath: string): Promise => { + for (const entry of await fs.promises.readdir(currentPath, { withFileTypes: true })) { + const sourcePath = path.join(currentPath, entry.name); + const name = `${archivePath}/${entry.name}`; + if (entry.isDirectory()) { + await visit(sourcePath, name); + } else if (entry.isSymbolicLink()) { + pack.entry( + { name, type: 'symlink', linkname: await fs.promises.readlink(sourcePath) }, + Buffer.alloc(0), + ); + } else if (entry.isFile()) { + pack.entry({ name, type: 'file' }, await fs.promises.readFile(sourcePath)); + } + } + }; + await visit(directoryPath, rootName); + pack.finalize(); + const chunks: Buffer[] = []; + for await (const chunk of pack) chunks.push(Buffer.from(chunk)); + return gzipSync(Buffer.concat(chunks)); +} + +export function createArtifactDownloadServer(archives: ReadonlyMap): http.Server { + return http.createServer((request, response) => { + const prefix = '/artifacts/'; + const artifactId = request.url?.startsWith(prefix) + ? decodeURIComponent(request.url.slice(prefix.length)) + : undefined; + const archive = request.method === 'GET' && artifactId ? archives.get(artifactId) : undefined; + if (!archive) { + response.statusCode = 404; + response.end('not found'); + return; + } + response.setHeader('content-type', 'application/gzip'); + response.end(archive); + }); +} diff --git a/packages/test-kit/src/index.ts b/packages/test-kit/src/index.ts index 7ca08aa991..894670ba21 100644 --- a/packages/test-kit/src/index.ts +++ b/packages/test-kit/src/index.ts @@ -76,6 +76,13 @@ export { KEYBOARD_COVERED_TAB_BAR_RECT, keyboardCoveredTabBarSnapshot, } from './keyboard-occlusion.fixtures.ts'; +export { + LANE_ENVELOPE_SCHEMA_VERSION, + laneEnvelope, + type LaneEnvelope, + type LaneEnvelopeInput, + type LaneResult, +} from './lane-envelope.ts'; export { closeLoopbackServer, listenOnLoopback, diff --git a/scripts/lib/lane-envelope.ts b/packages/test-kit/src/lane-envelope.ts similarity index 63% rename from scripts/lib/lane-envelope.ts rename to packages/test-kit/src/lane-envelope.ts index 1a03c294d0..7cc65d2d0e 100644 --- a/scripts/lib/lane-envelope.ts +++ b/packages/test-kit/src/lane-envelope.ts @@ -1,35 +1,20 @@ -// Standard artifact envelope for scheduled lanes. -// -// A scheduled lane can go dark or drift for weeks while PR CI stays green, so -// every artifact it uploads must say — without a human reading logs — which -// commit produced it, which tool/config version measured it, how long it took, -// and whether it passed. Freshness monitoring reads `finishedAt`/`result`; drift -// analysis reads `tool`/`configHash`. -// -// Lanes that adopt the envelope later should import this module rather than -// re-deriving the field names. - export const LANE_ENVELOPE_SCHEMA_VERSION = 1; export type LaneResult = 'pass' | 'fail'; export type LaneEnvelope = { schemaVersion: number; - /** Stable lane id — the metric key freshness/health jobs group by. */ lane: string; commit: string; ref: string | undefined; runUrl: string | undefined; - /** Tool versions, or content hashes where a version is not enough. */ tool: Record; configHash: string; - /** Only lanes with randomized input record a seed; `null` states "not applicable". */ seed: string | null; startedAt: string; finishedAt: string; durationMs: number; result: LaneResult; - /** Lane-specific payload; kept separate so the envelope shape stays stable. */ data: T; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9d306e95b9..c102f033bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -730,6 +730,12 @@ importers: '@agent-device/test-kit': specifier: workspace:* version: link:../test-kit + '@types/tar-stream': + specifier: ^3.1.4 + version: 3.1.4 + tar-stream: + specifier: ^3.2.0 + version: 3.2.0 packages/replay-port: dependencies: diff --git a/scripts/__tests__/help-conformance-sample-outputs.test.ts b/scripts/__tests__/help-conformance-sample-outputs.test.ts index 0d8ab5feef..4540e8da65 100644 --- a/scripts/__tests__/help-conformance-sample-outputs.test.ts +++ b/scripts/__tests__/help-conformance-sample-outputs.test.ts @@ -15,5 +15,5 @@ import { SAMPLE_PRODUCERS } from './help-conformance-sample-producers.ts'; for (const { name, producer, sample, render } of SAMPLE_PRODUCERS) { test(`${name} matches ${producer}`, async () => { assert.equal(await render(), sample.output); - }); + }, 30_000); } diff --git a/scripts/__tests__/help-conformance-sample-producers.ts b/scripts/__tests__/help-conformance-sample-producers.ts index 4014451abf..aad738e67e 100644 --- a/scripts/__tests__/help-conformance-sample-producers.ts +++ b/scripts/__tests__/help-conformance-sample-producers.ts @@ -20,16 +20,6 @@ import { interactionCliOutputFormatters } from '../../src/commands/interaction/o import { snapshotCliOutput } from '../../src/commands/capture/output.ts'; import { openCliOutput } from '../../src/commands/management/output.ts'; import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts'; -import { buildAmbiguousMatchError } from '@agent-device/daemon/testing/selector-match-errors'; -import { refMutationAdmissionResponse } from '@agent-device/daemon/testing/interaction/index'; -import { buildDeviceInUseBySessionError } from '@agent-device/daemon/testing/session-recovery-hints'; -import { buildDeviceClaimConflictError } from '@agent-device/daemon/testing/device/device-claim-conflict'; -import { - activateCompleteRefFrame, - readRefMutationFrame, -} from '@agent-device/daemon/testing/ref-frame'; -import { resolveRefStalenessWarning } from '@agent-device/daemon/testing/session-snapshot'; -import type { SessionState } from '@agent-device/daemon/testing/session-state'; import { buildAppNotInstalledError } from '@agent-device/platform-apple/app-resolution'; import { presentConnectReadiness, @@ -42,6 +32,7 @@ import type { SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import { renderSnapshotQualityWarnings } from '@agent-device/capture-kit/quality-warnings'; import { printHumanError } from '../../src/commands/output/error.ts'; import { formatSnapshotText } from '../../src/commands/output/snapshot.ts'; +import { captureHelpConformanceErrorRoutes } from '../../test/integration/provider-scenarios/help-conformance-error-routes.ts'; // The production renderer behind each captured sample in // scripts/help-conformance-sample-outputs.mjs, as data rather than as one test @@ -109,6 +100,13 @@ function assertErrorResponse( if (!response || response.ok) throw new Error(`${what} must be an error response`); } +let daemonErrorSampleResponses: ReturnType | undefined; + +function getDaemonErrorSampleResponses() { + daemonErrorSampleResponses ??= captureHelpConformanceErrorRoutes(); + return daemonErrorSampleResponses; +} + export const SAMPLE_PRODUCERS: SampleProducer[] = [ { name: 'SETTLE_TAIL_SAMPLE', @@ -361,92 +359,46 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ }, { name: 'DEVICE_IN_USE_SAMPLE', - producer: 'the real session-open by-session conflict producer', + producer: 'the session-open by-session conflict producer', sample: DEVICE_IN_USE_SAMPLE, - render: () => { - // An explicitly named session is stored under its own name, so address === name here. - const owningSession = { address: 'checkout', session: { name: 'checkout' } as SessionState }; - const device = { id: 'SIM-001', name: 'iPhone 17 Pro' } as Parameters< - typeof buildDeviceInUseBySessionError - >[1]; - const response = buildDeviceInUseBySessionError(owningSession, device); + render: async () => { + const responses = await getDaemonErrorSampleResponses(); + const response = responses.deviceInUse; assertErrorResponse(response, 'the by-session conflict'); - return renderErrorResponse(response); + return await renderErrorResponse(response); }, }, { name: 'DEVICE_CLAIM_IN_USE_SAMPLE', - producer: 'the enforced cross-daemon device-claim conflict producer', + producer: 'the enforced device-claim conflict producer', sample: DEVICE_CLAIM_IN_USE_SAMPLE, - render: () => { - const response = buildDeviceClaimConflictError( - { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel', - kind: 'emulator', - }, - { - fileName: 'claim.json', - deviceKey: 'local:android:none:emulator-5554', - classification: 'live', - claim: { - schemaVersion: 2, - deviceKey: 'local:android:none:emulator-5554', - device: { - family: 'android', - id: 'emulator-5554', - name: 'Pixel', - kind: 'emulator', - }, - session: 'checkout', - workspace: '/worktrees/checkout', - stateDir: '/state/checkout', - ownerPid: 4242, - ownerStartTime: 'start', - ownerToken: 'token', - createdAtMs: 1, - updatedAtMs: 1, - }, - }, - ); + render: async () => { + const responses = await getDaemonErrorSampleResponses(); + const response = responses.deviceClaimInUse; assertErrorResponse(response, 'the enforced device-claim conflict'); - return renderErrorResponse(response); + return await renderErrorResponse(response); }, }, { name: 'STALE_REF_SAMPLE', - producer: 'the real ADR 0014 admission rejection and staleness hint', + producer: 'the ADR 0014 stale-ref admission producer', sample: STALE_REF_SAMPLE, - render: () => { - const session = { snapshotGeneration: 7 } as SessionState; - // Pin the epoch in the frame itself, as a real published namespace does, so the - // sample exercises the frame epoch rather than the pre-frame fallback. - activateCompleteRefFrame(session); - const response = refMutationAdmissionResponse({ - session, - ref: '@e12', - mintedGeneration: 5, - staleRefsWarning: resolveRefStalenessWarning({ session, ref: '@e12', mintedGeneration: 5 }), - frame: readRefMutationFrame({ session, ref: '@e12', mintedGeneration: 5 }), - }); + render: async () => { + const responses = await getDaemonErrorSampleResponses(); + const response = responses.staleRef; assertErrorResponse(response, 'a superseded pin'); - return renderErrorResponse(response); + return await renderErrorResponse(response); }, }, { name: 'AMBIGUOUS_MATCH_SAMPLE', - producer: 'the real find producer and the default hint', + producer: 'the find ambiguous-match error producer', sample: AMBIGUOUS_MATCH_SAMPLE, - render: () => { - const matches = [ - { ref: 'e2', type: 'Button', label: 'Follow' }, - { ref: 'e5', type: 'Button', label: 'Follow' }, - { ref: 'e9', type: 'Button', label: 'Follow' }, - ] as Parameters[0]; - const response = buildAmbiguousMatchError(matches, 'text', 'Follow'); + render: async () => { + const responses = await getDaemonErrorSampleResponses(); + const response = responses.ambiguousMatch; assertErrorResponse(response, 'an ambiguous find'); - return renderErrorResponse(response); + return await renderErrorResponse(response); }, }, { diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts index 53002c300c..d604e2e0fa 100644 --- a/scripts/fuzz/envelope.ts +++ b/scripts/fuzz/envelope.ts @@ -2,8 +2,8 @@ // // A scheduled lane goes dark quietly: it can stop running, or fail for weeks, while PR CI stays // green. Freshness monitoring therefore needs one machine-readable record per run — green runs -// included. This module only maps the lane's own facts onto `scripts/lib/lane-envelope.ts`; the -// envelope shape itself is cross-lane and lives there. +// included. This module maps the lane's own facts onto the cross-lane envelope contract in +// `@agent-device/test-kit`. // // `error` (a crash, or config the harness could not parse) is reported as `result: 'fail'` with // `data.stage: 'error'`: the shared contract deliberately has two results, and a lane that could @@ -13,7 +13,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { laneEnvelope } from '../lib/lane-envelope.ts'; +import { laneEnvelope } from '@agent-device/test-kit'; import { runCmdSync } from '@agent-device/host-kit/command'; import type { FuzzFailure } from './invariant.ts'; diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts index a9c3a16938..c525064b8c 100644 --- a/scripts/fuzz/harness.test.ts +++ b/scripts/fuzz/harness.test.ts @@ -12,7 +12,7 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { LANE_ENVELOPE_SCHEMA_VERSION } from '../lib/lane-envelope.ts'; +import { LANE_ENVELOPE_SCHEMA_VERSION } from '@agent-device/test-kit'; import { CASE_GENERATION_INPUTS } from './envelope.ts'; import { runCases } from './execute.ts'; import { checkCase } from './invariant.ts'; diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index cb7d1d8396..bcccde117b 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -560,19 +560,18 @@ Use the output already shown to determine whether the feed-search UI is present, recovery: { code: 'DEVICE_IN_USE', sample: DEVICE_IN_USE_SAMPLE }, task: quiz( DEVICE_IN_USE_SAMPLE, - 'You are continuing the checkout flow that the "checkout" session was already running on this device. What command should run next?', + 'The default web session cannot open https://example.test because the "checkout" session already owns the device. Without closing that session, what command can reuse it or wait for the device to free up?', ), expectations: ['validPlanCommands', 'fullPrefix'], matchers: [ { - id: 'retriesWithOwningSession', - pattern: /(?:^|\n)agent-device\s+press\b[^\n]*--session\s+checkout\b/i, + id: 'opensWebWithOwningSessionOrWaits', + pattern: + /(?:^|\n)agent-device\s+open\s+https:\/\/example\.test\b(?=[^\n]*--platform\s+web\b)(?=[^\n]*(?:--session\s+checkout\b|--wait\s+\d+\b))[^\n]*/i, }, - { id: 'keepsSettle', pattern: /--settle\b/i }, ], forbidden: [ { id: 'noClose', pattern: /(?:^|\n)agent-device\s+close\b/i }, - { id: 'noReopen', pattern: /(?:^|\n)agent-device\s+open\b/i }, { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, ], }, @@ -604,7 +603,7 @@ Use the output already shown to determine whether the feed-search UI is present, recovery: { code: 'COMMAND_FAILED', sample: STALE_REF_SAMPLE }, task: quiz( STALE_REF_SAMPLE, - 'The Continue control this ref pointed at may have moved. What command should run next?', + 'The Place order control this ref pointed at may have moved. What command should run next?', ), expectations: ['validPlanCommands', 'fullPrefix'], matchers: [ diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index 3f53547f30..21e9768f0f 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -85,9 +85,9 @@ Detected an overly complex or slow accessibility tree. Fell back to the private- // DEVICE_IN_USE from buildDeviceInUseBySessionError // (packages/daemon/src/core/session-recovery-hints.ts) — the parity test drives that exact producer. export const DEVICE_IN_USE_SAMPLE = { - command: `agent-device press 'label="Place order"' --settle`, + command: 'agent-device open https://example.test --platform web --session default', output: `Error (DEVICE_IN_USE): Device is already in use by session "checkout". -Hint: Run agent-device session list to inspect active sessions. To reuse this device, rerun the command with --session checkout. To open a new session on this device, first run agent-device close --session checkout.`, +Hint: Run agent-device session list to inspect active sessions. To reuse this device, rerun the command with --session checkout. To open a new session on this device, first run agent-device close --session checkout. To wait for the device to free up, run the same open with --wait .`, }; // Enforced cross-daemon device claim conflict from @@ -105,9 +105,9 @@ Hint: Inspect the owner with: agent-device device status --platform android --se // `~s5` pin at the boundary (interaction-touch-targets.ts), so the message // names the plain ref; the hint is the precise resolveRefStalenessWarning. export const STALE_REF_SAMPLE = { - command: 'agent-device press @e12~s5 --settle', - output: `Error (COMMAND_FAILED): Ref @e12 was minted from a superseded snapshot generation -Hint: Ref @e12 was minted from snapshot s5 but the session's ref frame is now s7 — re-run snapshot -i.`, + command: 'agent-device press @e2~s5', + output: `Error (COMMAND_FAILED): Ref @e2 was minted from a superseded snapshot generation +Hint: Ref @e2 was minted from snapshot s5 but the session's ref frame is now s7 — re-run snapshot -i.`, }; // AMBIGUOUS_MATCH from buildAmbiguousMatchError (packages/daemon/src/execution/selector-match-errors.ts) diff --git a/scripts/mutation/envelope.test.ts b/scripts/mutation/envelope.test.ts index 99f13c7fea..954876809e 100644 --- a/scripts/mutation/envelope.test.ts +++ b/scripts/mutation/envelope.test.ts @@ -6,10 +6,13 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { + laneEnvelope, + LANE_ENVELOPE_SCHEMA_VERSION, + mkdtempForTestSync, +} from '@agent-device/test-kit'; import { test } from 'node:test'; import { runCmdSync } from '@agent-device/host-kit/command'; -import { laneEnvelope, LANE_ENVELOPE_SCHEMA_VERSION } from '../lib/lane-envelope.ts'; const repoRoot = path.resolve(import.meta.dirname, '../..'); diff --git a/scripts/mutation/run.ts b/scripts/mutation/run.ts index d661e06022..8d599dc3a5 100644 --- a/scripts/mutation/run.ts +++ b/scripts/mutation/run.ts @@ -17,7 +17,7 @@ import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { runCmdStreaming, runCmdSync } from '@agent-device/host-kit/command'; import { parseScriptArgs } from '../lib/cli-args.ts'; -import { laneEnvelope } from '../lib/lane-envelope.ts'; +import { laneEnvelope } from '@agent-device/test-kit'; import { COVERAGE_CAPTURE_ENV } from './coverage-reporter.mjs'; import { assertCoveragePolicy, diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index 7f31199b62..edd5ae2116 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -1,5 +1,4 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { ANDROID_AUDIO_CONTRACT_EVIDENCE } from '@agent-device/daemon/testing/session-observability/__tests__/session-audio.coverage'; import { ANDROID_INSTALL_SOURCE_CONTRACT_EVIDENCE } from '../../../src/__tests__/install-source.coverage.ts'; import { ANDROID_LIFECYCLE_CONTRACT_EVIDENCE } from '../provider-scenarios/android-lifecycle.coverage.ts'; import { @@ -15,6 +14,7 @@ import { } from './entries.ts'; import { ANDROID_ACTION_BUTTON_RUNTIME_CONTRACT_EVIDENCE, + ANDROID_AUDIO_CONTRACT_EVIDENCE, ANDROID_APPLICATION_LIFECYCLE_CONTRACT_EVIDENCE, ANDROID_FOLD_RUNTIME_CONTRACT_EVIDENCE, ANDROID_HOVER_RUNTIME_CONTRACT_EVIDENCE, diff --git a/test/integration/command-coverage/evidence.ts b/test/integration/command-coverage/evidence.ts index 342a1d2273..1d8d661b62 100644 --- a/test/integration/command-coverage/evidence.ts +++ b/test/integration/command-coverage/evidence.ts @@ -67,6 +67,12 @@ export const ANDROID_VIEWPORT_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence [C.viewport], 'rejects an unavailable exact-owner fact before binding', ); +export const ANDROID_AUDIO_CONTRACT_EVIDENCE: AndroidContractEvidence = + defineAndroidContractEvidence( + 'packages/daemon/src/sessions/session-observability/internal/__tests__/session-audio.test.ts', + [C.audio], + 'audio probe starts host helper for Android emulator audio', + ); export const TVOS_REMOTE_TEST_NAME = 'Provider-backed integration tvOS remote flow maps navigation commands to runner remote presses'; diff --git a/test/integration/interaction-contract/coordinate.contract.test.ts b/test/integration/interaction-contract/coordinate.contract.test.ts index 4c86080e53..6ae749819d 100644 --- a/test/integration/interaction-contract/coordinate.contract.test.ts +++ b/test/integration/interaction-contract/coordinate.contract.test.ts @@ -84,7 +84,7 @@ test(scenario('errorTaxonomy'), async () => { const press = await daemon.callCommand('press', ['100', '200']); // An unclassified backend failure gets the shared fallback classification // from normalizeError: stable code, original message, actionable hint. - const error = assertRpcError(press, 'UNKNOWN', /runner tap crashed/); + const error = assertRpcError(press, 500, 'UNKNOWN', /runner tap crashed/); assert.ok(typeof error.hint === 'string' && error.hint.length > 0); assert.ok(typeof error.diagnosticId === 'string'); }, diff --git a/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts index cab6bebe94..1170aa66a4 100644 --- a/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts +++ b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts @@ -27,6 +27,7 @@ test( await withIosContractDaemon([runnerSnapshotEntry(KEYBOARD_TREE)], async (daemon) => { const error = assertRpcError( await daemon.callCommand('find', ['label', 'Form', 'focus']), + 500, 'COMMAND_FAILED', /Ref @e2 is behind the visible keyboard and cannot be focused safely/, ); @@ -66,6 +67,7 @@ test( await withIosContractDaemon([runnerSnapshotEntry(coveredFormTree())], async (daemon) => { const error = assertRpcError( await daemon.callCommand('find', ['label', 'Form', 'focus']), + 500, 'COMMAND_FAILED', /Matched element @e2 is covered by another visible element and cannot be focused safely/, ); @@ -87,6 +89,7 @@ test( async (daemon) => { const ambiguous = assertRpcError( await daemon.callCommand('find', ['label', 'Form', 'focus']), + 500, 'AMBIGUOUS_MATCH', /find matched 2 elements for label "Form"/, ); @@ -129,6 +132,7 @@ test( await withIosContractDaemon([runnerSnapshotEntry(KEYBOARD_TREE)], async (daemon) => { assertRpcError( await daemon.callCommand('find', ['label', 'Nowhere', 'focus']), + 500, 'COMMAND_FAILED', /find did not match any element/, ); diff --git a/test/integration/interaction-contract/maestro-fallback.contract.test.ts b/test/integration/interaction-contract/maestro-fallback.contract.test.ts index 7cdf1ce222..2c335eb9c4 100644 --- a/test/integration/interaction-contract/maestro-fallback.contract.test.ts +++ b/test/integration/interaction-contract/maestro-fallback.contract.test.ts @@ -130,7 +130,7 @@ test( ], async (daemon) => { const click = await daemon.callCommand('click', ['label=Explore'], MAESTRO_FLAGS); - assertRpcError(click, 'ELEMENT_OFFSCREEN', /no tappable frame/); + assertRpcError(click, 500, 'ELEMENT_OFFSCREEN', /no tappable frame/); }, ); }, diff --git a/test/integration/interaction-contract/native-ref.contract.test.ts b/test/integration/interaction-contract/native-ref.contract.test.ts index 036167ca75..9055580fa9 100644 --- a/test/integration/interaction-contract/native-ref.contract.test.ts +++ b/test/integration/interaction-contract/native-ref.contract.test.ts @@ -4,7 +4,6 @@ import type { InteractionGuarantee } from '@agent-device/contracts/interaction-g import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { ref } from '../../../src/commands/interaction/runtime/selector-read-utils.ts'; import { scenarioName } from './coverage-manifest.ts'; -import { buildInteractionResponseData } from '@agent-device/daemon/testing/interaction/internal/interaction-touch-response'; import { NATIVE_REF_COVERAGE } from './native-ref.coverage.ts'; import { closedDrawerSnapshot, @@ -193,23 +192,6 @@ test(scenario('responseConstruction'), async () => { // record-time evidence node on the runtime result. assert.equal(result.node?.ref, 'e1'); assert.ok(Array.isArray(result.preAction?.nodes)); - - const { - result: visualization, - responseData, - recordedTarget, - } = buildInteractionResponseData({ - source: { kind: 'runtime', result }, - referenceFrame: undefined, - }); - // The construction site routes node/tree onto the typed recordedTarget - // channel only; neither serialized payload carries them. - assert.equal(recordedTarget?.node.ref, 'e1'); - for (const payload of [visualization, responseData]) { - assert.equal('node' in payload, false); - assert.equal('preAction' in payload, false); - assert.equal('targetEvidence' in payload, false); - } }); test(scenario('resolutionDisclosure'), async () => { diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index f708be01c3..510173e2c1 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -269,6 +269,7 @@ test( const refusal = await daemon.callCommand('press', ['label=Continue']); const data = assertRpcError( refusal, + 500, 'COMMAND_FAILED', /label=Continue had no confirmed resting position when the transition window ran out/, ); @@ -392,7 +393,7 @@ test( [runnerSnapshotEntry(RUNNER_CONTINUE_NODES), runnerSnapshotEntry(RUNNER_CONTINUE_NODES)], async (daemon) => { const press = await daemon.callCommand('press', ['label=Missing']); - const error = assertRpcError(press, 'COMMAND_FAILED', /Selector did not match/); + const error = assertRpcError(press, 500, 'COMMAND_FAILED', /Selector did not match/); assert.equal((error.details as Record)?.dispatched, 'no'); }, ); diff --git a/test/integration/provider-scenarios/active-session-script-publication.test.ts b/test/integration/provider-scenarios/active-session-script-publication.test.ts index a94a9fd789..d4d0b8f51c 100644 --- a/test/integration/provider-scenarios/active-session-script-publication.test.ts +++ b/test/integration/provider-scenarios/active-session-script-publication.test.ts @@ -9,14 +9,6 @@ import { assertRpcError, assertRpcOk } from './assertions.ts'; import { androidSettingsXml, createAndroidSettingsWorld } from './android-world.ts'; import { withProviderScenarioResource } from './harness.ts'; -/** The authoring lifecycle status of the world's live session, or `undefined` outside authoring. */ -function authoringPublicationStatus(world: { - daemon: { session: () => { scriptPublication?: { kind: string; status?: string } } | undefined }; -}): string | undefined { - const publication = world.daemon.session()?.scriptPublication; - return publication?.kind === 'authoring' ? publication.status : undefined; -} - test('provider route publishes and replays an open-to-destination script with a live handoff', async () => { await withProviderScenarioResource(createAndroidSettingsWorld, async (world) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-active-script-provider-')); @@ -34,13 +26,12 @@ test('provider route publishes and replays an open-to-destination script with a assert.equal(published.savedScript, scriptPath); assert.equal(published.session, 'default'); assert.equal(published.actionCount, 2); - assert.equal(authoringPublicationStatus(world), 'published'); const liveSnapshot = await client.capture.snapshot({ interactiveOnly: true }); assert.ok(liveSnapshot.nodes.some((node) => node.label === 'Search')); const flaggedClose = await world.daemon.callCommand('close', [], { saveScript: scriptPath }); - assertRpcError(flaggedClose, 'INVALID_ARGS', /cannot re-publish/); - assert.ok(world.daemon.session(), 'flagged close must preserve the published live session'); + assertRpcError(flaggedClose, 400, 'INVALID_ARGS', /cannot re-publish/); + assert.equal(await world.daemon.hasSession(), true); await client.sessions.close(); const replay = await world.daemon.callCommand('replay', [scriptPath], world.selection); @@ -52,7 +43,7 @@ test('provider route publishes and replays an open-to-destination script with a // tests in daemon-client-lifecycle.test.ts assume but cannot themselves // exercise, since they inject sessionActive via a fake HTTP fixture. assert.equal(replayData.sessionActive, true); - assert.ok(world.daemon.session(), 'a close-less replay must not tear down the session'); + assert.equal(await world.daemon.hasSession(), true); const replaySnapshot = await client.capture.snapshot({ interactiveOnly: true }); assert.ok(replaySnapshot.nodes.some((node) => node.label === 'Search')); await client.sessions.close(); @@ -76,13 +67,13 @@ test('provider route can suppress an authored terminal close for a live replay h }); assert.equal(replay.session, 'default'); assert.equal(replay.sessionActive, true); - assert.ok(world.daemon.session(), 'keepSession must preserve the replay-opened session'); + assert.equal(await world.daemon.hasSession(), true); const takeoverSnapshot = await client.capture.snapshot({ interactiveOnly: true }); assert.ok(takeoverSnapshot.nodes.some((node) => node.label === 'Search')); await client.sessions.close(); - assert.equal(world.daemon.session(), undefined); + assert.equal(await world.daemon.hasSession(), false); } finally { fs.rmSync(root, { recursive: true, force: true }); } @@ -135,7 +126,7 @@ test('replaying a published script against a reshuffled screen fails closed inst phase = 'replay'; const replay = await world.daemon.callCommand('replay', [scriptPath], world.selection); - const errorData = assertRpcError(replay, 'REPLAY_DIVERGENCE', /wait/i); + const errorData = assertRpcError(replay, 500, 'REPLAY_DIVERGENCE', /wait/i); const divergence = (errorData.details as Record | undefined) ?.divergence as | { kind?: string; targetBinding?: { classification?: string; matchCount?: number } } @@ -168,28 +159,24 @@ test('a second successful open aborts publication and terminal save flags fail b relaunch: true, saveScript: scriptPath, }); - assertRpcError(rearm, 'INVALID_ARGS', /only arm a fresh session/); - assert.equal(authoringPublicationStatus(world), 'armed'); - + assertRpcError(rearm, 400, 'INVALID_ARGS', /only arm a fresh session/); const second = await world.daemon.callCommand('open', ['settings'], { ...world.selection, relaunch: true, }); const secondData = assertRpcOk<{ warnings?: string[] }>(second); assert.match(String(secondData.warnings), /publication was aborted/i); - assert.equal(authoringPublicationStatus(world), 'aborted'); - const publication = await world.daemon.callCommand('session_save_script', [scriptPath]); - assertRpcError(publication, 'COMMAND_FAILED', /aborted by a second successful open/); + assertRpcError(publication, 500, 'COMMAND_FAILED', /aborted by a second successful open/); const flaggedClose = await world.daemon.callCommand('close', [], { saveScript: scriptPath }); - assertRpcError(flaggedClose, 'INVALID_ARGS', /terminal recording/); - assert.ok(world.daemon.session(), 'flagged close must not tear down the session'); + assertRpcError(flaggedClose, 400, 'INVALID_ARGS', /terminal recording/); + assert.equal(await world.daemon.hasSession(), true); assert.equal(fs.existsSync(scriptPath), false); const plainClose = await world.daemon.callCommand('close'); assertRpcOk(plainClose); - assert.equal(world.daemon.session(), undefined); + assert.equal(await world.daemon.hasSession(), false); } finally { fs.rmSync(root, { recursive: true, force: true }); } @@ -209,16 +196,14 @@ test('an unarmed session refuses close --save-script and closes cleanly on plain try { const opened = await world.daemon.callCommand('open', ['settings'], { ...world.selection }); assertRpcOk(opened); - assert.equal(authoringPublicationStatus(world), undefined); - const flaggedClose = await world.daemon.callCommand('close', [], { saveScript: scriptPath }); - assertRpcError(flaggedClose, 'INVALID_ARGS', /not armed/); - assert.ok(world.daemon.session(), 'flagged close must not tear down the unarmed session'); + assertRpcError(flaggedClose, 400, 'INVALID_ARGS', /not armed/); + assert.equal(await world.daemon.hasSession(), true); assert.equal(fs.existsSync(scriptPath), false); const plainClose = await world.daemon.callCommand('close'); assertRpcOk(plainClose); - assert.equal(world.daemon.session(), undefined); + assert.equal(await world.daemon.hasSession(), false); } finally { fs.rmSync(root, { recursive: true, force: true }); } @@ -239,8 +224,6 @@ test('an armed session still publishes target-v1 evidence through close --save-s saveScript: scriptPath, ...world.selection, }); - assert.equal(authoringPublicationStatus(world), 'armed'); - const snapshot = await client.capture.snapshot({ interactiveOnly: true, ...world.selection }); const search = snapshot.nodes.find((node) => node.label === 'Search'); assert.ok(search?.ref, JSON.stringify(snapshot.nodes)); @@ -251,7 +234,7 @@ test('an armed session still publishes target-v1 evidence through close --save-s assert.equal(fs.existsSync(scriptPath), true); const script = fs.readFileSync(scriptPath, 'utf8'); assert.match(script, /agent-device:target-v1/); - assert.equal(world.daemon.session(), undefined); + assert.equal(await world.daemon.hasSession(), false); } finally { fs.rmSync(root, { recursive: true, force: true }); } @@ -303,7 +286,7 @@ test('parameterized fill publishes only ${VAR} and replay resolves it immediatel ...world.selection, recordAs: 'password', }); - assertRpcError(invalidName, 'INVALID_ARGS', /Invalid --record-as variable/); + assertRpcError(invalidName, 400, 'INVALID_ARGS', /Invalid --record-as variable/); assert.equal(world.textInjectionCalls.length, 0); const contradictory = await world.daemon.callCommand('fill', [`@${search.ref}`, secret], { @@ -311,7 +294,7 @@ test('parameterized fill publishes only ${VAR} and replay resolves it immediatel recordAs: 'SEARCH_TERM', noRecord: true, }); - assertRpcError(contradictory, 'INVALID_ARGS', /cannot be combined with --no-record/); + assertRpcError(contradictory, 400, 'INVALID_ARGS', /cannot be combined with --no-record/); assert.equal(world.textInjectionCalls.length, 0); const fill = await client.interactions.fill({ @@ -332,9 +315,6 @@ test('parameterized fill publishes only ${VAR} and replay resolves it immediatel ...world.selection, }); - const recordedState = JSON.stringify(world.daemon.session()?.actions); - assert.equal(recordedState.includes(secret), false, recordedState); - assert.match(recordedState, /\$\{SEARCH_TERM\}/); await client.sessions.saveScript({ path: scriptPath }); const script = fs.readFileSync(scriptPath, 'utf8'); assert.equal(script.includes(secret), false); @@ -348,7 +328,7 @@ test('parameterized fill publishes only ${VAR} and replay resolves it immediatel [scriptPath], world.selection, ); - assertRpcError(missingValue, 'INVALID_ARGS', /Unresolved variable \$\{SEARCH_TERM\}/); + assertRpcError(missingValue, 400, 'INVALID_ARGS', /Unresolved variable \$\{SEARCH_TERM\}/); assert.equal(world.textInjectionCalls.length, callsBeforeMissingValue); await client.sessions.close(); @@ -366,9 +346,6 @@ test('parameterized fill publishes only ${VAR} and replay resolves it immediatel ); assertRpcOk(replay); assert.equal(world.textInjectionCalls.at(-1)?.text, secret); - const replayState = JSON.stringify(world.daemon.session()?.actions); - assert.equal(replayState.includes(secret), false, replayState); - assert.match(replayState, /\$\{SEARCH_TERM\}/); assert.ok(opened.sessionStateDir); const requestLog = fs.readFileSync( path.join(opened.sessionStateDir, 'requests', 'parameterized-replay-diagnostics.ndjson'), @@ -386,7 +363,7 @@ test('parameterized fill publishes only ${VAR} and replay resolves it immediatel ['id=com.android.settings:id/search', secret], { ...world.selection, recordAs: 'SEARCH_TERM' }, ); - assertRpcError(unarmed, 'INVALID_ARGS', /requires an armed script recording/); + assertRpcError(unarmed, 400, 'INVALID_ARGS', /requires an armed script recording/); assert.equal(world.textInjectionCalls.length, callsBeforeUnarmed); await client.sessions.close(); } finally { @@ -463,10 +440,6 @@ test('#1398: a read-only wait recorded after a parameterized fill never re-seria ...world.selection, }); - const stateBeforeGuard = JSON.stringify(world.daemon.session()?.actions); - assert.equal(stateBeforeGuard.includes(secret), false, stateBeforeGuard); - assert.match(stateBeforeGuard, /\$\{SEARCH_TERM\}/); - // Being the only wait so far, it cannot serve as the destination // guard: its ONLY identity was the parameterized-value echo, so // landmark evidence was dropped rather than published unverifiable. @@ -475,7 +448,7 @@ test('#1398: a read-only wait recorded after a parameterized fill never re-seria [scriptPath], world.selection, ); - assertRpcError(refusedPublish, 'COMMAND_FAILED', /destination guard/); + assertRpcError(refusedPublish, 500, 'COMMAND_FAILED', /destination guard/); // Recording a stable, non-value-bearing landmark — mirroring the // real-world fix of switching to the "Apps" landmark — recovers a diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 6bfc4adad6..d0432296a7 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -169,7 +169,6 @@ test(ANDROID_TOUCH_CONTRACT_EVIDENCE.testName, async () => { ...world.selection, }); assert.equal(oneFingerPan.pointerCount, 1); - assert.equal(world.daemon.session()?.actions.at(-1)?.flags.pointerCount, undefined); const twoFingerPan = await client.interactions.pan({ x: 195, @@ -182,8 +181,6 @@ test(ANDROID_TOUCH_CONTRACT_EVIDENCE.testName, async () => { }); assert.equal(twoFingerPan.pointerCount, 2); assert.equal(twoFingerPan.backend, 'provider-native-touch'); - assert.equal(world.daemon.session()?.actions.at(-1)?.flags.pointerCount, 2); - const pinch = await client.interactions.pinch({ scale: 2, x: 195, @@ -295,7 +292,7 @@ test('Provider-backed integration Android external ANR fails with actionable con (call) => call.slice(0, 4).join(' ') === 'shell am start -W', ).length; const press = await world.daemon.callCommand('press', ['50', '60'], world.selection); - const error = assertRpcError(press, 'COMMAND_FAILED', /com\.android\.systemui/); + const error = assertRpcError(press, 500, 'COMMAND_FAILED', /com\.android\.systemui/); const details = error.details as Record; assert.equal(details.focusedPackage, 'com.android.systemui'); @@ -355,7 +352,7 @@ async function runAndroidSetupAndInstallWorkflow( selectorTriggeredEvent.eventUrl, 'demo://agent-device/event?name=pre_open_ping&payload=%7B%22stage%22%3A%22explicit-selector%22%7D&platform=android', ); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); const keyboardDismiss = await client.command.keyboard({ action: 'dismiss', ...selection }); assert.equal(keyboardDismiss.platform, 'android'); @@ -529,15 +526,7 @@ async function runAndroidAppControlAndObservabilityWorkflow( triggeredEvent.eventUrl, 'demo://agent-device/event?name=screenshot_taken&payload=%7B%22source%22%3A%22provider-scenario%22%2C%22foreground%22%3Atrue%7D&platform=android', ); - const sessionAfterTriggeredEvent = daemon.session(); - assert.equal(sessionAfterTriggeredEvent?.appBundleId, 'com.example.demo'); - assert.ok( - sessionAfterTriggeredEvent?.actions.some( - (action) => - action.command === 'trigger-app-event' && action.positionals[0] === 'screenshot_taken', - ), - JSON.stringify(sessionAfterTriggeredEvent?.actions), - ); + assert.equal(await daemon.hasSession(), true); const tracePath = path.join(world.tempRoot, 'android-provider.adtrace'); const finalTracePath = path.join(world.tempRoot, 'android-provider-final.adtrace'); const traceStart = await daemon.callCommand('trace', ['start', tracePath], selection); @@ -683,7 +672,7 @@ async function runAndroidAppControlAndObservabilityWorkflow( platform: 'android', serial: PROVIDER_SCENARIO_ANDROID.id, }); - assertRpcError(invalidPerfAction, 'INVALID_ARGS', /perf action must be/i); + assertRpcError(invalidPerfAction, 400, 'INVALID_ARGS', /perf action must be/i); const logsStop = await client.observability.logs({ action: 'stop', ...selection }); assert.equal(logsStop.stopped, true); @@ -811,17 +800,6 @@ async function runAndroidCaptureInteractionAndReplayWorkflow( noRecord: true, }); assert.equal(unrecordedPress.json?.result?.data?.x, 50); - assert.equal( - daemon - .session() - ?.actions.some( - (action) => - action.command === 'press' && - action.positionals[0] === '50' && - action.positionals[1] === '60', - ), - false, - ); // ADR 0014: the ref click expired the frame — re-observe before filling. await client.capture.snapshot({ interactiveOnly: true, ...selection }); @@ -1016,7 +994,7 @@ async function runAndroidCaptureInteractionAndReplayWorkflow( // close --save-script + shutdown below. const plainCloseBeforeArm = await daemon.callCommand('close'); assert.equal(plainCloseBeforeArm.statusCode, 200, JSON.stringify(plainCloseBeforeArm.json)); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); const savedReplayPath = path.join(tempRoot, 'saved-session.ad'); const armedOpen = await client.apps.open({ @@ -1031,7 +1009,7 @@ async function runAndroidCaptureInteractionAndReplayWorkflow( }); assert.equal(close.json?.result?.data?.shutdown?.success, true); assert.equal(fs.existsSync(savedReplayPath), true); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); const testReplayPath = path.join(tempRoot, 'settings-smoke.ad'); fs.writeFileSync( diff --git a/test/integration/provider-scenarios/android-recording-fixtures.ts b/test/integration/provider-scenarios/android-recording-fixtures.ts index 96ad68d843..0b251b894f 100644 --- a/test/integration/provider-scenarios/android-recording-fixtures.ts +++ b/test/integration/provider-scenarios/android-recording-fixtures.ts @@ -28,7 +28,7 @@ export async function stopAndroidRecording( export async function createAndroidRecordingScenarioHarness( deps: Parameters[0], ): Promise { - return await createProviderScenarioHarness({ ...deps, platformRuntime: true }); + return await createProviderScenarioHarness({ ...deps }); } export async function withAndroidRecordingScenario( diff --git a/test/integration/provider-scenarios/android-recording.test.ts b/test/integration/provider-scenarios/android-recording.test.ts index dcbe82a758..9060ca21d6 100644 --- a/test/integration/provider-scenarios/android-recording.test.ts +++ b/test/integration/provider-scenarios/android-recording.test.ts @@ -287,6 +287,7 @@ test('Provider-backed integration Android record start retains completed evidenc serial: PROVIDER_SCENARIO_ANDROID.id, recordingScope: 'device', }), + 500, 'DEVICE_IN_USE', /another recorder is writing/, ); @@ -348,6 +349,7 @@ test('Provider-backed integration Android record start refuses completed evidenc serial: PROVIDER_SCENARIO_ANDROID.id, recordingScope: 'device', }), + 500, 'DEVICE_IN_USE', /its recorder is writing/, ); @@ -391,7 +393,7 @@ test('Provider-backed integration Android corrupt descriptor is retained without seedAndroidRecordingResource(daemon, manifest, { descriptor: { malformed: true } }); try { const stopped = await stopAndroidRecording(daemon); - assertRpcError(stopped, 'COMMAND_FAILED', /cannot be reattached/); + assertRpcError(stopped, 500, 'COMMAND_FAILED', /cannot be reattached/); assert.equal( calls.some((args) => args[1]?.includes('agent-device-recording-active.json')), false, @@ -429,6 +431,7 @@ test('Provider-backed integration Android cleanup-only recovery terminalizes bef try { assertRpcError( await stopAndroidRecording(daemon), + 500, 'COMMAND_FAILED', /launch was interrupted/, ); diff --git a/test/integration/provider-scenarios/android-telephony.test.ts b/test/integration/provider-scenarios/android-telephony.test.ts index 5a5bef4476..f1f5800d8a 100644 --- a/test/integration/provider-scenarios/android-telephony.test.ts +++ b/test/integration/provider-scenarios/android-telephony.test.ts @@ -1,9 +1,11 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'vitest'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { AndroidAdbProvider } from '@agent-device/platform-android/mechanics'; +import { parseReplayScriptDetailed } from '@agent-device/ad-script'; import { assertRpcError } from './assertions.ts'; -import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import { createAndroidSettingsWorld } from './android-world.ts'; import { runProviderScenario } from './scenario.ts'; const PROVIDER_SCENARIO_ANDROID_EMULATOR: DeviceInfo = { @@ -22,133 +24,117 @@ const PROVIDER_SCENARIO_ANDROID_PHONE: DeviceInfo = { kind: 'device', }; -function recordingAdbProvider(reply: (args: readonly string[]) => string = () => 'OK\n') { - const calls: string[][] = []; - const provider: AndroidAdbProvider = { - exec: async (args) => { - calls.push([...args]); - return { stdout: reply(args), stderr: '', exitCode: 0 }; - }, - }; - return { calls, provider }; -} - const SMS_TEXT = `Don't "stop"\nC:\\videos`; function emulatorConsoleCalls(calls: readonly string[][]): string[][] { return calls.map((call) => call.slice(call.indexOf('emu'))).filter((call) => call[0] === 'emu'); } -test('Provider-backed integration Android emulator telephony flow drives the emulator console', async () => { - const adb = recordingAdbProvider(); +test('Provider-backed integration Android emulator telephony flow drives the emulator console and records its commands', async () => { + const world = await createAndroidSettingsWorld({ + device: PROVIDER_SCENARIO_ANDROID_EMULATOR, + onAdbExec: (args) => + args[0] === 'emu' ? { stdout: 'OK\n', stderr: '', exitCode: 0 } : undefined, + }); - await withProviderScenarioResource( - async () => - await createProviderScenarioHarness({ - androidAdbProvider: () => adb.provider, - deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID_EMULATOR], - }), - async (daemon) => { - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_ANDROID_EMULATOR, - createdAt: Date.now(), - actions: [], - }); + try { + const client = world.daemon.client(); + const opened = await client.apps.open({ + app: 'settings', + ...world.selection, + saveScript: true, + }); + assert.equal(opened.device?.id, PROVIDER_SCENARIO_ANDROID_EMULATOR.id); - await runProviderScenario(daemon, [ - { - name: 'ring', - command: 'telephony', - positionals: ['call', '5551234'], - flags: { platform: 'android' }, - expectData: { - action: 'call', - number: '5551234', - message: 'Simulated an incoming call from 5551234', - }, - }, - { - name: 'hang up', - command: 'telephony', - positionals: ['cancel', '5551234'], - flags: { platform: 'android' }, - expectData: { action: 'cancel', number: '5551234' }, - }, - { - name: 'text', - command: 'telephony', - positionals: ['sms', '+15551234567', SMS_TEXT], - flags: { platform: 'android' }, - expectData: { action: 'sms', number: '+15551234567', textLength: SMS_TEXT.length }, + await runProviderScenario(world.daemon, [ + { + name: 'ring', + command: 'telephony', + positionals: ['call', '5551234'], + flags: world.selection, + expectData: { + action: 'call', + number: '5551234', + message: 'Simulated an incoming call from 5551234', }, - ]); + }, + { + name: 'hang up', + command: 'telephony', + positionals: ['cancel', '5551234'], + flags: world.selection, + expectData: { action: 'cancel', number: '5551234' }, + }, + { + name: 'text', + command: 'telephony', + positionals: ['sms', '+15551234567', SMS_TEXT], + flags: world.selection, + expectData: { action: 'sms', number: '+15551234567', textLength: SMS_TEXT.length }, + }, + ]); - assert.deepEqual(emulatorConsoleCalls(adb.calls), [ - ['emu', 'gsm', 'call', '5551234'], - ['emu', 'gsm', 'cancel', '5551234'], - ['emu', 'sms', 'send', '+15551234567', String.raw`Don't "stop"\nC:\\videos`], - ]); - assert.deepEqual( - daemon.session('default')?.actions.map((action) => action.command), - ['telephony', 'telephony', 'telephony'], - ); - }, - ); + assert.deepEqual(emulatorConsoleCalls(world.adbCalls), [ + ['emu', 'gsm', 'call', '5551234'], + ['emu', 'gsm', 'cancel', '5551234'], + ['emu', 'sms', 'send', '+15551234567', String.raw`Don't "stop"\nC:\\videos`], + ]); + + await client.command.wait({ selector: 'label="Apps"', ...world.selection }); + const published = await client.sessions.saveScript({ + path: path.join(world.tempRoot, 'telephony.ad'), + }); + const actions = parseReplayScriptDetailed( + fs.readFileSync(published.savedScript, 'utf8'), + ).actions; + assert.equal(published.actionCount, actions.length); + assert.deepEqual( + actions.filter((action) => action.command === 'telephony').map((action) => action.command), + ['telephony', 'telephony', 'telephony'], + ); + } finally { + await world.close(); + } }); test('Provider-backed integration Android telephony reports a console refusal with its reason', async () => { - const adb = recordingAdbProvider(() => 'KO: no call with this number\n'); - - await withProviderScenarioResource( - async () => - await createProviderScenarioHarness({ - androidAdbProvider: () => adb.provider, - deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID_EMULATOR], - }), - async (daemon) => { - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_ANDROID_EMULATOR, - createdAt: Date.now(), - actions: [], - }); + const world = await createAndroidSettingsWorld({ + onAdbExec: (args) => + args[0] === 'emu' + ? { stdout: 'KO: no call with this number\n', stderr: '', exitCode: 0 } + : undefined, + }); - const data = assertRpcError( - await daemon.callCommand('telephony', ['accept', '5551234'], { platform: 'android' }), - 'COMMAND_FAILED', - /refused telephony accept/, - ); - assert.equal((data.details as Record)?.reason, 'telephony-console-refused'); - assert.deepEqual(emulatorConsoleCalls(adb.calls), [['emu', 'gsm', 'accept', '5551234']]); - }, - ); + try { + await world.daemon.client().apps.open({ app: 'settings', ...world.selection }); + const data = assertRpcError( + await world.daemon.callCommand('telephony', ['accept', '5551234'], world.selection), + 500, + 'COMMAND_FAILED', + /refused telephony accept/, + ); + assert.equal((data.details as Record)?.reason, 'telephony-console-refused'); + assert.deepEqual(emulatorConsoleCalls(world.adbCalls), [['emu', 'gsm', 'accept', '5551234']]); + } finally { + await world.close(); + } }); -test('Provider-backed integration Android telephony refuses a physical device before adb', async () => { - const adb = recordingAdbProvider(); - - await withProviderScenarioResource( - async () => - await createProviderScenarioHarness({ - androidAdbProvider: () => adb.provider, - deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID_PHONE], - }), - async (daemon) => { - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_ANDROID_PHONE, - createdAt: Date.now(), - actions: [], - }); +test('Provider-backed integration Android telephony refuses a physical device before the emulator console', async () => { + const world = await createAndroidSettingsWorld({ device: PROVIDER_SCENARIO_ANDROID_PHONE }); - const data = assertRpcError( - await daemon.callCommand('telephony', ['call', '5551234'], { platform: 'android' }), - 'UNSUPPORTED_OPERATION', - /telephony is not supported on this device/, - ); - assert.equal((data.details as Record)?.reason, 'unsupported-device-kind'); - assert.deepEqual(emulatorConsoleCalls(adb.calls), []); - }, - ); + try { + const opened = await world.daemon.client().apps.open({ app: 'settings', ...world.selection }); + assert.equal(opened.device?.id, PROVIDER_SCENARIO_ANDROID_PHONE.id); + const data = assertRpcError( + await world.daemon.callCommand('telephony', ['call', '5551234'], world.selection), + 500, + 'UNSUPPORTED_OPERATION', + /telephony is not supported on this device/, + ); + assert.equal((data.details as Record)?.reason, 'unsupported-device-kind'); + assert.deepEqual(emulatorConsoleCalls(world.adbCalls), []); + } finally { + await world.close(); + } }); diff --git a/test/integration/provider-scenarios/android-tv-remote.test.ts b/test/integration/provider-scenarios/android-tv-remote.test.ts index bb9baccd7d..4f3fe41030 100644 --- a/test/integration/provider-scenarios/android-tv-remote.test.ts +++ b/test/integration/provider-scenarios/android-tv-remote.test.ts @@ -1,4 +1,3 @@ -import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { AndroidAdbProvider } from '@agent-device/platform-android/mechanics'; @@ -31,11 +30,10 @@ test('Provider-backed integration Android TV remote flow sends D-pad keyevents', deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID_TV], }), async (daemon) => { - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_ANDROID_TV, - createdAt: Date.now(), - actions: [], + await daemon.client().apps.open({ + app: 'com.android.settings', + platform: 'android', + serial: PROVIDER_SCENARIO_ANDROID_TV.id, }); await runProviderScenario(daemon, [ @@ -72,10 +70,6 @@ test('Provider-backed integration Android TV remote flow sends D-pad keyevents', '--longpress', 'KEYCODE_DPAD_CENTER', ]); - assert.deepEqual( - daemon.session('default')?.actions.map((action) => action.command), - ['tv-remote', 'tv-remote'], - ); }, ); }); diff --git a/test/integration/provider-scenarios/android-uninstall.test.ts b/test/integration/provider-scenarios/android-uninstall.test.ts index 4d668d8bed..943d210dbe 100644 --- a/test/integration/provider-scenarios/android-uninstall.test.ts +++ b/test/integration/provider-scenarios/android-uninstall.test.ts @@ -1,80 +1,51 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { AndroidAdbProvider } from '@agent-device/platform-android/mechanics'; import { assertCommandCall } from './assertions.ts'; -import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import { createAndroidSettingsWorld } from './android-world.ts'; import { runProviderScenario } from './scenario.ts'; -const PROVIDER_SCENARIO_ANDROID: DeviceInfo = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel', - kind: 'emulator', - target: 'mobile', - booted: true, -}; - test('Provider-backed integration Android uninstall removes an installed package and no-ops on a missing one', async () => { - const adbCalls: string[][] = []; - const adbProvider: AndroidAdbProvider = { - exec: async (args) => { - adbCalls.push([...args]); - if (args.join(' ') === 'shell pm list packages') { - return { stdout: 'package:com.example.demo\n', stderr: '', exitCode: 0 }; - } - return { stdout: args[0] === 'uninstall' ? 'Success\n' : '', stderr: '', exitCode: 0 }; - }, - }; - - await withProviderScenarioResource( - async () => - await createProviderScenarioHarness({ - androidAdbProvider: () => adbProvider, - deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID], - }), - async (daemon) => { - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_ANDROID, - createdAt: Date.now(), - actions: [], - }); + const world = await createAndroidSettingsWorld(); - await runProviderScenario(daemon, [ - { - name: 'uninstall an installed package', - command: 'uninstall', - positionals: ['com.example.demo'], - flags: { platform: 'android' }, - expectData: { - platform: 'android', - app: 'com.example.demo', - wasInstalled: true, - appId: 'com.example.demo', - message: 'Uninstalled: com.example.demo', - }, + try { + await world.daemon.client().apps.open({ app: 'settings', ...world.selection }); + await runProviderScenario(world.daemon, [ + { + name: 'uninstall an installed package', + command: 'uninstall', + positionals: ['com.example.demo'], + flags: world.selection, + expectData: { + platform: 'android', + app: 'com.example.demo', + wasInstalled: true, + appId: 'com.example.demo', + message: 'Uninstalled: com.example.demo', }, - { - name: 'uninstall a package nothing installed matches', - command: 'uninstall', - positionals: ['com.example.missing'], - flags: { platform: 'android' }, - expectData: { - platform: 'android', - app: 'com.example.missing', - wasInstalled: false, - message: 'Not installed: com.example.missing', - }, + }, + { + name: 'uninstall a package nothing installed matches', + command: 'uninstall', + positionals: ['com.example.missing'], + flags: world.selection, + expectData: { + platform: 'android', + app: 'com.example.missing', + wasInstalled: false, + message: 'Not installed: com.example.missing', }, - ]); + }, + ]); - assertCommandCall(adbCalls, ['uninstall', 'com.example.demo']); - assert.equal( - adbCalls.some((call) => call[0] === 'uninstall' && call.includes('com.example.missing')), - false, - 'a package nothing installed matches must not reach adb uninstall', - ); - }, - ); + assertCommandCall(world.adbCalls, ['uninstall', 'com.example.demo']); + assert.equal( + world.adbCalls.some( + (call) => call[0] === 'uninstall' && call.includes('com.example.missing'), + ), + false, + 'a package nothing installed matches must not reach adb uninstall', + ); + } finally { + await world.close(); + } }); diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 1a25d1f4d3..68f0c1e6e4 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -9,6 +9,7 @@ import type { AndroidAdbProvider, } from '@agent-device/platform-android/mechanics'; import type { DeviceInventoryRequest } from '@agent-device/device-selection/dispatch-resolve'; +import type { DeviceInfo } from '@agent-device/kernel/device'; import { ANDROID_IME_HELPER_FIXTURE_ARTIFACT } from '../../../src/__tests__/test-utils/android-ime-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, @@ -60,11 +61,12 @@ type AndroidSettingsWorld = { }; export async function createAndroidSettingsWorld(options?: { + device?: DeviceInfo; nativeTextInjection?: boolean; onTextInjection?: (request: AndroidSettingsWorld['textInjectionCalls'][number]) => void; snapshotXml?: () => string; dumpsysWindow?: () => string; - onAdbExec?: (args: readonly string[]) => void; + onAdbExec?: (args: readonly string[]) => void | AndroidAdbResult; }): Promise { // The world's helper version probe always reports no helper, so no install may be remembered // from an earlier world on the same serial. @@ -80,6 +82,7 @@ export async function createAndroidSettingsWorld(options?: { const shellState = createAndroidProviderShellState(); const appState = createAndroidProviderAppState(); const spawnedLogcat: AndroidAdbProcess[] = []; + const device = options?.device ?? PROVIDER_SCENARIO_ANDROID; const tempRoot = fs.mkdtempSync( path.join(os.tmpdir(), 'agent-device-provider-scenario-android-deploy-'), ); @@ -105,7 +108,8 @@ export async function createAndroidSettingsWorld(options?: { exec: async (received) => { const args = [...received]; adbCalls.push(args); - options?.onAdbExec?.([...args]); + const override = options?.onAdbExec?.([...args]); + if (override) return override; updateAndroidProviderShellState(args, shellState); const stateResult = updateAndroidProviderAppState(args, appState); if (stateResult) return stateResult; @@ -152,11 +156,10 @@ export async function createAndroidSettingsWorld(options?: { }; } const daemon = await createProviderScenarioHarness({ - platformRuntime: true, androidAdbProvider: () => adbProvider, deviceInventoryProvider: async (request) => { inventoryRequests.push({ ...request }); - return [PROVIDER_SCENARIO_ANDROID]; + return [device]; }, }); @@ -177,7 +180,7 @@ export async function createAndroidSettingsWorld(options?: { apkPath, aabPath, manifestApkPath, - selection: { platform: 'android', serial: PROVIDER_SCENARIO_ANDROID.id }, + selection: { platform: 'android', serial: device.id }, assertNoHostAdbCalls: () => { assert.deepEqual(readHostAdbCalls(hostAdbGuard.argsLogPath), []); }, diff --git a/test/integration/provider-scenarios/apple-app-log-runtime-provider.test.ts b/test/integration/provider-scenarios/apple-app-log-runtime-provider.test.ts index 19365aabec..5a5c7d032c 100644 --- a/test/integration/provider-scenarios/apple-app-log-runtime-provider.test.ts +++ b/test/integration/provider-scenarios/apple-app-log-runtime-provider.test.ts @@ -2,33 +2,37 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness } from './harness.ts'; -import { createRecordingAppleToolProvider } from './providers.ts'; +import { createRecordingAppleToolProvider, simctlListDevicesResult } from './providers.ts'; test('provider-inventory Apple logs doctor stays on the scoped Apple tool provider', async () => { const appleTool = createRecordingAppleToolProvider({ simctl: async (args) => { + const devices = simctlListDevicesResult(args, 'com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: 'iPhone 15', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + ]); + if (devices) return devices; + if (args.join(' ') === 'listapps sim-1') { + return { + stdout: '{"com.example.demo":{"CFBundleDisplayName":"Demo"}}', + stderr: '', + exitCode: 0, + }; + } + if (args[0] === 'launch') return { stdout: '', stderr: '', exitCode: 0 }; assert.deepEqual(args, ['help']); return { stdout: 'simctl help', stderr: '', exitCode: 0 }; }, }); const daemon = await createProviderScenarioHarness({ - platformRuntime: true, appleToolProvider: () => appleTool.provider, deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], }); - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_IOS_SIMULATOR, - appBundleId: 'com.example.app', - createdAt: Date.now(), - actions: [], - }); - try { const selection = { platform: 'ios' as const, udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, }; + await daemon.client().apps.open({ app: 'com.example.demo', ...selection }); const result = await daemon.client().observability.logs({ action: 'doctor', ...selection, diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index e34429a78f..e8d4b5230f 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -4,6 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; import type { AppleRunnerProvider } from '@agent-device/platform-apple/runner'; +import type { AppleRunnerScreenRecordingTransport } from '../../../src/platform-runtime-screen-recording-apple-runner-transport.ts'; import type { AppleSimulatorScreenRecordingTransport } from '../../../src/platform-runtime-screen-recording-apple-transport.ts'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { PROVIDER_SCENARIO_IOS_SIMULATOR, PROVIDER_SCENARIO_MACOS } from './fixtures.ts'; @@ -342,10 +343,23 @@ function permissiveRecording(): AppleSimulatorScreenRecordingTransport { }; } +function unavailableRunnerRecording(): AppleRunnerScreenRecordingTransport { + return { + authority: 'local-lease', + available: false, + start: async () => { + throw new Error('Apple runner recording is unavailable in this fake provider'); + }, + inspect: async () => 'missing', + stop: async () => {}, + }; +} + async function createWorldDaemon(world: World): Promise { const device = world === 'ios' ? PROVIDER_SCENARIO_IOS_SIMULATOR : PROVIDER_SCENARIO_MACOS; return await createProviderScenarioHarness({ appleRunnerProvider: () => permissiveRunner(), + appleRunnerScreenRecordingTransport: () => unavailableRunnerRecording(), appleToolProvider: () => permissiveTool(world).provider, appleSimulatorScreenRecordingTransport: () => permissiveRecording(), deviceInventoryProvider: async () => [device], @@ -353,7 +367,7 @@ async function createWorldDaemon(world: World): Promise } async function ensureSession(daemon: ProviderScenarioHarness, world: World): Promise { - if (daemon.session()) return; + if (await daemon.hasSession()) return; const { positionals, flags } = WORLDS[world].open; await daemon.callCommand('open', positionals ?? [], flags ?? {}); } diff --git a/test/integration/provider-scenarios/assertions.ts b/test/integration/provider-scenarios/assertions.ts index a1f023a1b3..31cd9ee774 100644 --- a/test/integration/provider-scenarios/assertions.ts +++ b/test/integration/provider-scenarios/assertions.ts @@ -40,10 +40,11 @@ export function assertRpcOk = Record { - assert.equal(response.statusCode, 200, JSON.stringify(response.json)); + assert.equal(response.statusCode, expectedStatus, JSON.stringify(response.json)); assert.equal(response.json?.error?.data?.code, code, JSON.stringify(response.json)); assert.match(response.json?.error?.message ?? '', message); return (response.json?.error?.data ?? {}) as Record; diff --git a/test/integration/provider-scenarios/batch-flags.test.ts b/test/integration/provider-scenarios/batch-flags.test.ts index 7654acac94..432cb7d88e 100644 --- a/test/integration/provider-scenarios/batch-flags.test.ts +++ b/test/integration/provider-scenarios/batch-flags.test.ts @@ -14,19 +14,19 @@ test('Provider-backed integration batch rejects boundary flag values through the batchOnError: 'continue' as never, batchSteps: [{ command: 'wait', positionals: ['1'] }], }); - assertRpcError(unsupportedOnError, 'INVALID_ARGS', /Unsupported batch on-error mode/); + assertRpcError(unsupportedOnError, 400, 'INVALID_ARGS', /Unsupported batch on-error mode/); const invalidMaxSteps = await daemon.callCommand('batch', [], { batchMaxSteps: 0, batchSteps: [{ command: 'wait', positionals: ['1'] }], }); - assertRpcError(invalidMaxSteps, 'INVALID_ARGS', /Invalid batch max-steps: 0/); + assertRpcError(invalidMaxSteps, 400, 'INVALID_ARGS', /Invalid batch max-steps: 0/); const nonIntegerMaxSteps = await daemon.callCommand('batch', [], { batchMaxSteps: 1.5, batchSteps: [{ command: 'wait', positionals: ['1'] }], }); - assertRpcError(nonIntegerMaxSteps, 'INVALID_ARGS', /Invalid batch max-steps: 1\.5/); + assertRpcError(nonIntegerMaxSteps, 400, 'INVALID_ARGS', /Invalid batch max-steps: 1\.5/); const exceedsMaxSteps = await daemon.callCommand('batch', [], { batchMaxSteps: 1, @@ -35,7 +35,7 @@ test('Provider-backed integration batch rejects boundary flag values through the { command: 'wait', positionals: ['1'] }, ], }); - assertRpcError(exceedsMaxSteps, 'INVALID_ARGS', /batch has 2 steps; max allowed is 1/); + assertRpcError(exceedsMaxSteps, 400, 'INVALID_ARGS', /batch has 2 steps; max allowed is 1/); }, ); }); diff --git a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts index 742e6b566d..31d576e1b0 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts @@ -54,9 +54,7 @@ test('cloud iOS snapshot captures through the provider session after open', asyn ), true, ); - // The explicit bundle id is the one identity a cloud open can adopt without - // local tooling, so the session keeps it for every appBundleId-gated command. - assert.equal(daemon.session()?.appBundleId, 'com.example.demo'); + // The app-gated capture succeeds after opening the cloud session through the public client. }); }, 15_000); @@ -291,10 +289,11 @@ async function openCloudIosSession(daemon: ProviderScenarioHarness): Promise(open); + assert.equal(opened.appBundleId, 'com.example.demo'); // The prepared cloud session chose iOS. This must become an explicit device leaf before the // runtime facts admission; a collapsed record may not borrow support from an owner label. - assert.equal(daemon.session()?.device.appleOs, 'ios'); + assert.equal((await daemon.client().sessions.list())[0]?.device.appleOs, 'ios'); return lease; } diff --git a/test/integration/provider-scenarios/cloud-webdriver-lease-http.test.ts b/test/integration/provider-scenarios/cloud-webdriver-lease-http.test.ts index 2958fc569a..3212d62642 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-lease-http.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-lease-http.test.ts @@ -3,13 +3,8 @@ import http from 'node:http'; import { test } from 'vitest'; import { createBundledCloudWebDriverRuntimes } from '@agent-device/provider-webdriver'; import { createProviderDeviceRuntimeRequestProviders } from '../../../src/provider-device-runtime.ts'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; import { buildHttpRpcPayload } from '../../../src/daemon-client/daemon-client-rpc.ts'; -import { - closeLoopbackServer, - listenOnLoopback, - skipWhenLoopbackUnavailable, -} from '@agent-device/test-kit'; +import { skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; import { CloudWebDriverTestServer, @@ -57,13 +52,8 @@ test('lease_allocate over HTTP prepares the BrowserStack session end to end', as deviceInventorySource: providers.deviceInventorySource!, platformRuntime: { providerRuntimes: runtimes, providerModules }, }); - const httpServer = await createDaemonHttpServer({ - token: harness.token, - handleRequest: harness.handleRequest, - }); try { - const port = await listenOnLoopback(httpServer); const envelope = buildHttpRpcPayload( { token: harness.token, @@ -92,7 +82,7 @@ test('lease_allocate over HTTP prepares the BrowserStack session end to end', as { includeTokenParam: false }, ); - const body = await postRpc(port, harness.token, envelope); + const body = await postRpc(harness.httpPort, harness.token, envelope); assert.equal(body.status, 200); assert.equal(body.rpc.error, undefined, `rpc error: ${JSON.stringify(body.rpc.error)}`); assert.ok( @@ -115,7 +105,6 @@ test('lease_allocate over HTTP prepares the BrowserStack session end to end', as }, }); } finally { - await closeLoopbackServer(httpServer); await harness.close(); await Promise.allSettled(runtimes.map(async (runtime) => await runtime.shutdown())); } diff --git a/test/integration/provider-scenarios/cloud-webdriver-runtime.test.ts b/test/integration/provider-scenarios/cloud-webdriver-runtime.test.ts index dedc081143..51be0b2d83 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-runtime.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-runtime.test.ts @@ -2,17 +2,12 @@ import type { DaemonWireRequest } from '@agent-device/contracts/command'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { test } from 'vitest'; -import { - clearRequestAbortRegistration, - markRequestCanceled, - registerRequestAbort, -} from '@agent-device/host-kit/request'; +import { test, vi } from 'vitest'; +import { isRequestCanceledError } from '@agent-device/kernel/errors'; import { createBundledCloudWebDriverRuntimes } from '@agent-device/provider-webdriver'; import type { CloudArtifact } from '@agent-device/contracts/observability'; import type { DeviceLease } from '@agent-device/contracts/device'; import { createProviderDeviceRuntimeRequestProviders } from '../../../src/provider-device-runtime.ts'; -import { createExpiredProviderLeaseReleaser } from '@agent-device/daemon/testing/provider-lease-expiry'; import { assertRpcOk } from './assertions.ts'; import { createProviderScenarioHarness, @@ -192,26 +187,25 @@ test('packaged Cloud WebDriver orientation surfaces a provider failure instead o }, 15_000); test('packaged Cloud WebDriver expiry releases the live provider session', async () => { - await withProviderScenarioResource(createCloudWebDriverWorld, async (world) => { - const lease = await allocateWebDriverLease(world.daemon); - const releaser = createExpiredProviderLeaseReleaser({ - leaseLifecycleProvider: world.providers.leaseLifecycleProvider, - providerRuntimeIds: world.providers.providerRuntimeIds, - recoverableProviderIds: world.providers.recoverableProviderIds, - }); - - try { - await releaser.release(lease); - assert.equal( - world.server.calls.some( - (call) => call.method === 'DELETE' && call.path === '/wd/hub/session/wd-1', - ), - true, - ); - } finally { - releaser.shutdown(); - } - }); + let now = Date.now(); + const dateSpy = vi.spyOn(Date, 'now').mockImplementation(() => now); + try { + await withProviderScenarioResource( + async () => await createCloudWebDriverWorld({ leaseTtlMs: 5_000 }), + async (world) => { + const lease = await allocateWebDriverLease(world.daemon); + now = lease.expiresAt + 1; + await waitForLeaseExpiry(world.server); + assert.ok( + world.server.calls.some( + (call) => call.method === 'DELETE' && call.path === '/wd/hub/session/wd-1', + ), + ); + }, + ); + } finally { + dateSpy.mockRestore(); + } }, 15_000); // #2509: on a screen that never goes idle — a looping onboarding video — the driver's @@ -223,17 +217,17 @@ test('packaged Cloud WebDriver expiry releases the live provider session', async test('packaged Cloud WebDriver cancels an abandoned source capture and keeps its session', async () => { await withProviderScenarioResource(createCloudWebDriverWorld, async (world) => { const { daemon, server } = world; - const requestId = 'cloud-webdriver-cancelled-source'; const lease = await openWebDriverSession(daemon); - const registration = registerRequestAbort(requestId); server.sourceBehavior = 'never'; + const controller = new AbortController(); try { - const capture = daemon.callCommand('snapshot', [], leaseFlags(lease.leaseId), { - meta: { ...leaseMeta(lease.leaseId), requestId }, + const capture = daemon.client().capture.snapshot({ + signal: controller.signal, + ...leaseClientOptions(lease.leaseId), }); await waitForSourceRequest(server); - markRequestCanceled(requestId); + controller.abort(); // Bounded on purpose: the bug was that this never returned at all, and a // regression should fail here with its reason rather than as a test timeout. @@ -242,8 +236,10 @@ test('packaged Cloud WebDriver cancels an abandoned source capture and keeps its 3_000, 'the abandoned capture was still running after its client hung up', ); - assert.notEqual(canceled.json?.error, undefined, 'an abandoned capture must not succeed'); - assert.equal(canceled.json?.error?.data?.details?.reason, 'request_canceled'); + assert.fail(`an aborted capture unexpectedly succeeded: ${JSON.stringify(canceled)}`); + } catch (error) { + assert.equal(isRequestCanceledError(error), true); + await waitForSourceAbort(server); assert.equal( sourceCalls(server)[0]?.signal?.aborted, true, @@ -262,7 +258,7 @@ test('packaged Cloud WebDriver cancels an abandoned source capture and keeps its assertRpcOk(shot); }); } finally { - clearRequestAbortRegistration(registration); + controller.abort(); } }); }, 20_000); @@ -290,7 +286,27 @@ async function waitForSourceRequest(server: FakeWebDriverServer): Promise assert.ok(sourceCalls(server).length > 0, 'the driver never received a page-source request'); } -async function createCloudWebDriverWorld() { +async function waitForSourceAbort(server: FakeWebDriverServer): Promise { + for (let attempt = 0; attempt < 200; attempt += 1) { + if (sourceCalls(server)[0]?.signal?.aborted) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail('the client abort did not reach the WebDriver page-source request'); +} + +async function waitForLeaseExpiry(server: FakeWebDriverServer): Promise { + for (let attempt = 0; attempt < 250; attempt += 1) { + if ( + server.calls.some((call) => call.method === 'DELETE' && call.path === '/wd/hub/session/wd-1') + ) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + assert.fail('expired lease did not release the live WebDriver session'); +} + +async function createCloudWebDriverWorld(params: { leaseTtlMs?: number } = {}) { const server = await FakeWebDriverServer.start(); const runtimes = createBundledCloudWebDriverRuntimes({ env: { @@ -313,11 +329,11 @@ async function createCloudWebDriverWorld() { ...providers, deviceInventorySource: providers.deviceInventorySource!, platformRuntime: { providerRuntimes: runtimes, providerModules }, + ...(params.leaseTtlMs === undefined ? {} : { leaseTtlMs: params.leaseTtlMs }), }); return { daemon, server, - providers, close: async () => { await Promise.allSettled(runtimes.map(async (runtime) => await runtime.shutdown())); await daemon.close(); @@ -644,3 +660,23 @@ function leaseMeta(leaseId?: string): DaemonWireRequest['meta'] { clientId: 'client-a', }; } + +function leaseClientOptions(leaseId: string) { + return { + platform: 'android' as const, + target: 'mobile' as const, + tenant: 'team-a', + runId: 'run-a', + leaseId, + leaseBackend: 'android-instance' as const, + leaseProvider: WEBDRIVER_PROVIDER, + deviceKey: 'webdriver-android-a', + clientId: 'client-a', + device: 'Google Pixel 8', + providerApp: 'bs://app-id', + providerOsVersion: '14.0', + providerProject: 'agent-device', + providerBuild: 'run-a', + providerSessionName: leaseId, + }; +} diff --git a/test/integration/provider-scenarios/daemon-command-policy.test.ts b/test/integration/provider-scenarios/daemon-command-policy.test.ts index 6aa1f58e8a..91bfa8a53d 100644 --- a/test/integration/provider-scenarios/daemon-command-policy.test.ts +++ b/test/integration/provider-scenarios/daemon-command-policy.test.ts @@ -27,7 +27,6 @@ test('Provider-backed integration daemon command policies gate admission and pro await withProviderScenarioResource( async () => await createProviderScenarioHarness({ - platformRuntime: true, androidAdbProvider: () => adbProvider, deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID], }), @@ -63,7 +62,7 @@ test('Provider-backed integration daemon command policies gate admission and pro { platform: 'android' }, { meta: inactiveLeaseMeta }, ); - assertRpcError(blockedSnapshot, 'UNAUTHORIZED', /Lease is not active/); + assertRpcError(blockedSnapshot, 401, 'UNAUTHORIZED', /Lease is not active/); const recordStart = await daemon.callCommand('record', ['start', '/tmp/policy-record.mp4'], { recordingScope: 'device', @@ -75,7 +74,7 @@ test('Provider-backed integration daemon command policies gate admission and pro ); }, ); -}); +}, 15_000); function androidAdbResult( args: readonly string[], diff --git a/test/integration/provider-scenarios/doctor.test.ts b/test/integration/provider-scenarios/doctor.test.ts index cfc28fbdac..5233dfcc60 100644 --- a/test/integration/provider-scenarios/doctor.test.ts +++ b/test/integration/provider-scenarios/doctor.test.ts @@ -108,7 +108,6 @@ test('Provider-backed integration doctor --app verifies an installed app without await withProviderScenarioResource( async () => await createProviderScenarioHarness({ - platformRuntime: true, androidAdbProvider: () => adbProvider, deviceInventoryProvider: async () => [PROVIDER_SCENARIO_ANDROID], }), diff --git a/test/integration/provider-scenarios/harness-proxy.ts b/test/integration/provider-scenarios/harness-proxy.ts new file mode 100644 index 0000000000..4d8bfc4a70 --- /dev/null +++ b/test/integration/provider-scenarios/harness-proxy.ts @@ -0,0 +1,294 @@ +import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import http from 'node:http'; +import { createDaemonProxyServer, type DaemonProxyAuthenticate } from '@agent-device/proxy'; +import { buildHttpRpcPayload } from '../../../src/daemon-client/daemon-client-rpc.ts'; +import { withClientReplayScriptSources } from '../../../src/__tests__/test-utils/replay-script-source.fixtures.ts'; +import { closeLoopbackServer, listenOnLoopback } from '@agent-device/test-kit'; +import { commandRequest, responseToRpcResult } from './harness-rpc.ts'; +import { type ProviderScenarioHarness, type ProviderScenarioRpcResult } from './harness.ts'; + +export type ProviderScenarioForwardedRequest = { + method: string; + route: string; + requestBody: string; + responseBody: string; + authorization: string | undefined; + tenantId: string | undefined; + statusCode: number; + upstreamErrorCode?: string; +}; + +export type ProviderScenarioProxy = { + baseUrl: string; + port: number; + forwarded: ProviderScenarioForwardedRequest[]; + requestRpc: ( + authToken: string, + payload: Record, + ) => Promise; + callCommand: ( + authToken: string, + command: string, + positionals?: string[], + flags?: DaemonWireRequest['flags'], + options?: { + input?: DaemonWireRequest['input']; + meta?: DaemonWireRequest['meta']; + runtime?: DaemonWireRequest['runtime']; + session?: string; + }, + ) => Promise; + close: () => Promise; +}; + +export type ProviderScenarioProxyOptions = + | { clientToken: string; authenticate?: never } + | { clientToken?: never; authenticate: DaemonProxyAuthenticate }; + +type JsonRpcResponseEnvelope = { + result?: DaemonResponse; + error?: { code: number; message: string; data?: Record }; +}; + +export async function createProviderScenarioProxy( + daemon: ProviderScenarioHarness, + options: ProviderScenarioProxyOptions, +): Promise { + const upstream = createRecordingUpstreamServer(daemon.httpPort); + const upstreamPort = await listenOnLoopback(upstream.server); + const proxy = createDaemonProxyServer({ + upstreamBaseUrl: `http://127.0.0.1:${upstreamPort}`, + upstreamToken: daemon.token, + ...options, + }); + let proxyPort: number; + try { + proxyPort = await listenOnLoopback(proxy); + } catch (error) { + await closeLoopbackServer(proxy); + await closeLoopbackServer(upstream.server); + throw error; + } + + const baseUrl = `http://127.0.0.1:${proxyPort}/agent-device`; + let closed = false; + const requestRpc: ProviderScenarioProxy['requestRpc'] = async (authToken, payload) => { + const response = await fetch(`${baseUrl}/rpc`, { + method: 'POST', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(payload), + }); + return { statusCode: response.status, json: await response.json() }; + }; + + return { + baseUrl, + port: proxyPort, + forwarded: upstream.forwarded, + requestRpc, + callCommand: async (authToken, command, positionals = [], flags = {}, commandOptions = {}) => { + const request = await withClientReplayScriptSources( + commandRequest(authToken, command, positionals, flags, commandOptions), + ); + const payload = buildHttpRpcPayload(request, { includeTokenParam: true }); + const result = await requestRpc(authToken, payload); + const envelope = result.json as JsonRpcResponseEnvelope; + const id = String(payload.id); + if (envelope.result) return responseToRpcResult(envelope.result, id, result.statusCode); + return { + statusCode: result.statusCode, + json: { + jsonrpc: '2.0', + id, + error: envelope.error ?? { code: result.statusCode, message: '' }, + }, + }; + }, + close: async () => { + if (closed) return; + closed = true; + await closeLoopbackServer(proxy); + await closeLoopbackServer(upstream.server); + }, + }; +} + +function createRecordingUpstreamServer(upstreamPort: number): { + server: http.Server; + forwarded: ProviderScenarioForwardedRequest[]; +} { + const forwarded: ProviderScenarioForwardedRequest[] = []; + const server = http.createServer((request, response) => { + void handleRecordingUpstreamRequest(request, response, upstreamPort, forwarded); + }); + return { server, forwarded }; +} + +type RecordingUpstreamResponse = { + statusCode: number; + headers: http.IncomingHttpHeaders; + body: Buffer; +}; + +async function handleRecordingUpstreamRequest( + request: http.IncomingMessage, + response: http.ServerResponse, + upstreamPort: number, + forwarded: ProviderScenarioForwardedRequest[], +): Promise { + const target = allowlistedUpstreamTarget(request); + if (!target) { + response.statusCode = 404; + response.end('not found'); + return; + } + + let entry: ProviderScenarioForwardedRequest | undefined; + try { + const requestBody = await readRequestBody(request); + entry = recordForwardedRequest(request, target.route, requestBody, forwarded); + const upstreamResponse = await forwardToDaemon(upstreamPort, request, target.path, requestBody); + recordUpstreamResponse(entry, upstreamResponse); + sendUpstreamResponse(response, upstreamResponse); + } catch (error) { + if (entry && error instanceof Error && 'code' in error && typeof error.code === 'string') { + entry.upstreamErrorCode = error.code; + } + response.statusCode = 502; + response.end('upstream request failed'); + } +} + +function allowlistedUpstreamTarget( + request: http.IncomingMessage, +): { path: string; route: string } | undefined { + const requestTarget = request.url ?? '/'; + if (!requestTarget.startsWith('/') || requestTarget.startsWith('//')) return undefined; + + let target: URL; + try { + target = new URL(requestTarget, 'http://provider-scenario.invalid'); + } catch { + return undefined; + } + if (target.origin !== 'http://provider-scenario.invalid' || target.search || target.hash) { + return undefined; + } + + const route = target.pathname; + const method = request.method; + const isRpc = method === 'POST' && route === '/rpc'; + const isDiagnostics = + method === 'GET' && + /^\/sessions\/(?:[A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+\/requests\/(?:[A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+\/diagnostics$/.test( + route, + ); + const isArtifactList = method === 'GET' && (route === '/artifacts' || route === '/artifacts/'); + const isArtifact = method === 'GET' && /^\/artifacts\/[0-9a-f-]+$/i.test(route); + if (!isRpc && !isDiagnostics && !isArtifactList && !isArtifact) return undefined; + return { path: route, route }; +} + +async function readRequestBody(request: http.IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +function recordForwardedRequest( + request: http.IncomingMessage, + route: string, + requestBody: Buffer, + forwarded: ProviderScenarioForwardedRequest[], +): ProviderScenarioForwardedRequest { + const tenantHeader = request.headers['x-agent-device-tenant']; + const entry: ProviderScenarioForwardedRequest = { + method: request.method ?? '', + route, + requestBody: requestBody.toString('utf8'), + responseBody: '', + authorization: request.headers.authorization, + tenantId: Array.isArray(tenantHeader) ? tenantHeader.join(', ') : tenantHeader, + statusCode: 0, + }; + forwarded.push(entry); + return entry; +} + +async function forwardToDaemon( + upstreamPort: number, + request: http.IncomingMessage, + allowedPath: string, + requestBody: Buffer, +): Promise { + const headers: Record = {}; + for (const [name, value] of Object.entries(request.headers)) { + if ( + value === undefined || + ['host', 'content-length', 'connection', 'transfer-encoding', 'accept-encoding'].includes( + name, + ) + ) { + continue; + } + headers[name] = Array.isArray(value) ? value.join(', ') : value; + } + if (requestBody.length > 0) headers['content-length'] = String(requestBody.length); + return await new Promise((resolve, reject) => { + const upstreamRequest = http.request( + { + hostname: '127.0.0.1', + port: upstreamPort, + path: allowedPath, + method: request.method, + headers, + }, + (upstreamReply) => { + const responseChunks: Buffer[] = []; + upstreamReply.on('data', (chunk: Buffer | string) => { + responseChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + upstreamReply.on('end', () => { + resolve({ + statusCode: upstreamReply.statusCode ?? 0, + headers: upstreamReply.headers, + body: Buffer.concat(responseChunks), + }); + }); + upstreamReply.on('error', reject); + }, + ); + upstreamRequest.on('error', reject); + upstreamRequest.end(requestBody); + }); +} + +function recordUpstreamResponse( + entry: ProviderScenarioForwardedRequest, + upstreamResponse: RecordingUpstreamResponse, +): void { + entry.statusCode = upstreamResponse.statusCode; + const contentTypeHeader = upstreamResponse.headers['content-type']; + const contentType = Array.isArray(contentTypeHeader) + ? contentTypeHeader.join(', ') + : contentTypeHeader; + if (contentType?.includes('json') || contentType?.startsWith('text/')) { + entry.responseBody = upstreamResponse.body.toString('utf8'); + } +} + +function sendUpstreamResponse( + response: http.ServerResponse, + upstreamResponse: RecordingUpstreamResponse, +): void { + response.statusCode = upstreamResponse.statusCode; + for (const name of ['content-type', 'content-disposition', 'content-length', 'x-request-id']) { + const value = upstreamResponse.headers[name]; + if (value) response.setHeader(name, value); + } + response.end(upstreamResponse.body); +} diff --git a/test/integration/provider-scenarios/harness-rpc.ts b/test/integration/provider-scenarios/harness-rpc.ts new file mode 100644 index 0000000000..0fbcf68b77 --- /dev/null +++ b/test/integration/provider-scenarios/harness-rpc.ts @@ -0,0 +1,52 @@ +import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import type { ProviderScenarioRpcResult } from './harness.ts'; + +export function commandRequest( + token: string, + command: string, + positionals: string[] = [], + flags: DaemonWireRequest['flags'] = {}, + options: { + input?: DaemonWireRequest['input']; + meta?: DaemonWireRequest['meta']; + runtime?: DaemonWireRequest['runtime']; + session?: string; + } = {}, +): DaemonWireRequest { + return { + token, + session: options.session ?? 'default', + command, + positionals, + input: options.input, + flags, + runtime: options.runtime, + meta: options.meta, + }; +} + +export function responseToRpcResult( + response: DaemonResponse, + id: string, + statusCode: number, +): ProviderScenarioRpcResult { + return { + statusCode, + json: response.ok + ? { + jsonrpc: '2.0', + id, + result: { data: response.data ?? {} }, + } + : { + jsonrpc: '2.0', + id, + error: { + code: -32000, + message: response.error.message, + data: response.error, + }, + }, + }; +} diff --git a/test/integration/provider-scenarios/harness.ts b/test/integration/provider-scenarios/harness.ts index 702e3a97be..9f76509ef9 100644 --- a/test/integration/provider-scenarios/harness.ts +++ b/test/integration/provider-scenarios/harness.ts @@ -2,34 +2,34 @@ import type { DaemonWireRequest } from '@agent-device/contracts/command'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; +import http from 'node:http'; import os from 'node:os'; import path from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; import { createAgentDeviceClient } from '../../../src/agent-device-client.ts'; -import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; import type { AgentDeviceClient } from '../../../src/client/client-types.ts'; +import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; +import type { AppleToolProvider } from '@agent-device/platform-apple/tool-provider'; +import { + startDaemonRuntime, + type ComposeDaemonRuntimeDeps, + type DaemonRuntimeDeps, +} from '@agent-device/daemon'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { resolveSessionDir } from '@agent-device/daemon-contracts/session-artifact-paths'; import { - createRequestHandler, - type RequestRouterDeps, -} from '@agent-device/daemon/testing/request-router'; + buildHttpRpcPayload, + handleDaemonHttpResponseBody, +} from '../../../src/daemon-client/daemon-client-rpc.ts'; +import { commandRequest, responseToRpcResult } from './harness-rpc.ts'; +import { createDaemonRuntimeDeps } from '../../../src/daemon-composition.ts'; import { createPlatformRuntimeGateway, createRequestPlatformProviders, - androidObservation, type PlatformProviderResolvers, } from '../../../src/platform-runtime.ts'; -import { createClaimRecoveryGateway } from '../../../src/platform-runtime-claim-recovery.ts'; -import { createDaemonPlatformServices } from '../../../src/platform-runtime-daemon-services.ts'; -import { platformResourceCleanup } from '../../../src/platform-runtime-resource-cleanup.ts'; -import { readDaemonProviderCredentials } from '../../../src/provider-credential-fingerprint.ts'; import type { AppleSimulatorScreenRecordingProcess } from '../../../src/platform-runtime-screen-recording-apple-transport.ts'; -import { trackDownloadableArtifact } from '@agent-device/daemon/testing/artifact-tracking'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; -import { SessionStore } from '@agent-device/daemon/testing/session-store'; -import type { DaemonInvokeFn } from '@agent-device/daemon'; -import type { SessionState } from '@agent-device/daemon/testing/session-state'; import { runCmdBackground } from '@agent-device/host-kit/command'; -import { createOwnedProcessRecordStore } from '@agent-device/host-kit/process'; import { withClientReplayScriptSources } from '../../../src/__tests__/test-utils/replay-script-source.fixtures.ts'; import type { DeviceInventoryProvider, @@ -40,24 +40,9 @@ import { createTestDeviceInventoryGateways, createTestDeviceInventoryGatewaysFromProvider, } from '@agent-device/device-selection/inventory-gateways-fixtures'; -import { createHostDiagnostics } from '../../../src/platform-runtime-host-diagnostics.ts'; import type { PlatformRuntimeProviderRegistration } from '../../../src/platform-runtime-gateway.ts'; import { createProviderPlatformRuntimeRegistrations } from '../../../src/provider-device-runtimes.ts'; -import { isActiveProviderDevice } from '../../../src/provider-device-runtime.ts'; -import { installProviderDeviceAdmission } from '@agent-device/daemon/testing/provider-device-admission'; -import { installInteractorResolution } from '@agent-device/daemon/testing/interactor-resolution'; -import { getInteractor } from '../../../src/core/interactors.ts'; -import { unavailableDeviceRuntimeGateway } from '@agent-device/daemon/testing/__tests__/support/unavailable-device-runtime-gateway'; -import { openWebSessionNames } from '@agent-device/daemon/testing/web-session-names'; -import { testCreateCommandSurface } from '../../../src/__tests__/daemon-composition/command-surface-fixture.ts'; - -// Match daemon composition (src/daemon-composition.ts): the daemon decides on provider -// ownership through its own admission seam, which root composition installs. -installProviderDeviceAdmission({ isActive: (device) => isActiveProviderDevice(device) }); -installInteractorResolution({ resolve: getInteractor }); - -const PROVIDER_SCENARIO_TOKEN = 'provider-scenario-token'; const PROVIDER_SCENARIO_TEMP_REMOVE_OPTIONS = { recursive: true, force: true, @@ -67,6 +52,11 @@ const PROVIDER_SCENARIO_TEMP_REMOVE_OPTIONS = { export type ProviderScenarioRpcResult = { statusCode: number; json: any }; +type JsonRpcResponseEnvelope = { + result?: DaemonResponse; + error?: { code: number; message: string; data?: Record }; +}; + export type ProviderScenarioHarness = { callCommand: ( command: string, @@ -80,12 +70,11 @@ export type ProviderScenarioHarness = { }, ) => Promise; client: () => AgentDeviceClient; - /** The scenario daemon's request boundary, for mounting it behind a real HTTP server. */ - handleRequest: DaemonInvokeFn; + hasSession: (session?: string) => Promise; + baseUrl: string; + httpPort: number; token: string; - session: (name?: string) => SessionState | undefined; sessionDir: (name?: string) => string; - publishSession: (name: string, session: SessionState) => void; close: () => Promise; }; @@ -93,43 +82,45 @@ export type ClosableProviderScenarioResource = { close: () => Promise | void; }; -export type ProviderScenarioPlatformRuntime = - | boolean - | Readonly<{ - providerRuntimes: readonly ProviderDeviceRuntime[]; - providerModules: readonly PlatformRuntimeProviderRegistration[]; - }>; +export type ProviderScenarioPlatformRuntime = Readonly<{ + providerRuntimes: readonly ProviderDeviceRuntime[]; + providerModules: readonly PlatformRuntimeProviderRegistration[]; +}>; + +type ProviderScenarioDependencies = Partial> & + Partial & + Partial & + ( + | { deviceInventoryProvider: DeviceInventoryProvider; deviceInventorySource?: never } + | { deviceInventorySource: ProviderDeviceInventorySource; deviceInventoryProvider?: never } + ) & { + providerRuntimes?: readonly ProviderDeviceRuntime[]; + platformRuntime?: ProviderScenarioPlatformRuntime; + leaseTtlMs?: number; + }; export async function createProviderScenarioHarness( - deps: Partial> & - Partial & - ( - | { deviceInventoryProvider: DeviceInventoryProvider; deviceInventorySource?: never } - | { deviceInventorySource: ProviderDeviceInventorySource; deviceInventoryProvider?: never } - ) & { - /** - * Eager provider ownership metadata used only to build explicit provider registrations for - * a scenario. It is intentionally not forwarded as ambient router policy. - */ - providerRuntimes?: readonly ProviderDeviceRuntime[]; - platformRuntime?: ProviderScenarioPlatformRuntime; - }, + deps: ProviderScenarioDependencies, ): Promise { - const sessionDir = fs.mkdtempSync( - path.join(os.tmpdir(), 'agent-device-provider-scenario-session-'), - ); - const sessionStore = new SessionStore(sessionDir); - const ownedProcessRecords = createOwnedProcessRecordStore({ - stateDir: path.dirname(sessionDir), - sessionsDir: sessionDir, - resolveSessionDir: (sessionId) => sessionStore.resolveSessionDir(sessionId), - }); + const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-provider-scenario-')); + const stateDir = path.join(testRoot, 'state'); + const daemonPaths = resolveDaemonPaths(stateDir); + const env: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_CLAIMS_DIR: path.join(testRoot, 'claims'), + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '3600000', + }; const { deviceInventoryProvider, deviceInventorySource, deviceRuntimeGateway: configuredDeviceRuntimeGateway, - platformRuntime = true, + platformRuntime, providerRuntimes, + leaseTtlMs, requestPlatformProviders: configuredRequestPlatformProviders, androidAdbProvider, appleRunnerProvider, @@ -139,111 +130,213 @@ export async function createProviderScenarioHarness( vegaToolProvider, webProvider, appleSimulatorScreenRecordingTransport, - providerCredentials = readDaemonProviderCredentials({}, path.dirname(sessionDir)), - ...routerDeps + ...dependencyOverrides } = deps; - const platformRuntimeOptions = - typeof platformRuntime === 'object' - ? platformRuntime - : { - // Provider runtime mechanics remain implementation-lazy, but their owner metadata is - // present before the first facts admission. This matches daemon composition and makes - // a missing provider module fail closed instead of falling back to host tooling. - providerRuntimes: providerRuntimes ?? [], - providerModules: createProviderPlatformRuntimeRegistrations(providerRuntimes ?? []), - }; - const deviceRuntimeGateway = - configuredDeviceRuntimeGateway ?? - (platformRuntime - ? createPlatformRuntimeGateway({ - ...platformRuntimeOptions, - sessionsDir: sessionDir, - ownedProcesses: ownedProcessRecords, - resolveSessionArtifacts: (sessionId) => ({ - outputPath: sessionStore.resolveAppLogPath(sessionId), - pidPath: sessionStore.resolveAppLogPidPath(sessionId), - }), + if (leaseTtlMs !== undefined) env.AGENT_DEVICE_LEASE_TTL_MS = String(leaseTtlMs); + + let scopedAppleToolProvider: AppleToolProvider | undefined; + const composeDeps: ComposeDaemonRuntimeDeps = async (input) => { + const rootDeps = await createDaemonRuntimeDeps(input); + const configuredRuntimes = platformRuntime?.providerRuntimes ?? providerRuntimes ?? []; + const providerModules = + platformRuntime?.providerModules ?? + createProviderPlatformRuntimeRegistrations(configuredRuntimes); + const defaultDeviceRuntimeGateway = createPlatformRuntimeGateway({ + assertShutdownAllowed: input.assertShutdownAllowed, + providerRuntimes: configuredRuntimes, + providerModules, + sessionsDir: input.sessionsDir, + ownedProcesses: rootDeps.ownedProcessRecords, + resolveSessionArtifacts: input.resolveSessionArtifacts, + }); + const deviceRuntimeGateway = configuredDeviceRuntimeGateway + ? Object.freeze({ + ...defaultDeviceRuntimeGateway, + ...configuredDeviceRuntimeGateway, + applicationLifecycle: + configuredDeviceRuntimeGateway.applicationLifecycle ?? + defaultDeviceRuntimeGateway.applicationLifecycle, }) - : unavailableDeviceRuntimeGateway); - const requestHandler = createRequestHandler({ - createCommandSurface: testCreateCommandSurface, - logPath: path.join(os.tmpdir(), 'agent-device-provider-scenario-daemon.log'), - token: PROVIDER_SCENARIO_TOKEN, - sessionStore, - leaseRegistry: new LeaseRegistry(), - deviceInventoryGateways: deviceInventorySource + : defaultDeviceRuntimeGateway; + const deviceInventoryGateways = deviceInventorySource ? createTestDeviceInventoryGateways({ provider: deviceInventorySource }) - : createTestDeviceInventoryGatewaysFromProvider(deviceInventoryProvider), - deviceRuntimeGateway, - trackDownloadableArtifact, - // Match daemon composition (src/daemon-composition.ts): doctor's host-scoped - // diagnostics are injected at the root, so the harness composes them the same way. - hostDiagnostics: createHostDiagnostics(), - androidObservation, - platformResourceCleanup, - platformServices: createDaemonPlatformServices(), - claimRecoveryGateway: createClaimRecoveryGateway, - requestPlatformProviders: + : createTestDeviceInventoryGatewaysFromProvider(deviceInventoryProvider); + const trackedAppleToolProvider = appleToolProvider + ? (context: Parameters>[0]) => { + scopedAppleToolProvider = appleToolProvider(context); + return scopedAppleToolProvider; + } + : undefined; + const requestPlatformProviders = configuredRequestPlatformProviders ?? createRequestPlatformProviders({ providers: { androidAdbProvider, appleRunnerProvider, appleRunnerScreenRecordingTransport, - appleToolProvider, + appleToolProvider: trackedAppleToolProvider, linuxToolProvider, vegaToolProvider, webProvider, appleSimulatorScreenRecordingTransport, }, defaultWebProvider: { - stateDir: path.dirname(sessionDir), - openWebSessionNames: () => openWebSessionNames(sessionStore), - ownedProcessRecords, + stateDir: input.baseDir, + openWebSessionNames: input.openWebSessionNames, + ownedProcessRecords: rootDeps.ownedProcessRecords, }, + }); + return Object.freeze({ + ...rootDeps, + ...dependencyOverrides, + providerDeviceRuntimes: configuredRuntimes, + deviceRuntimeGateway, + requestRouter: Object.freeze({ + ...rootDeps.requestRouter, + ...dependencyOverrides, + deviceInventoryGateways, + requestPlatformProviders, }), - providerCredentials, - ...routerDeps, - }); - const handleRequest: typeof requestHandler = async (request) => { - // #1802: a raw `callCommand('replay', [path])` stands in for a client request, and every - // request that reaches a daemon carries the script sources the CLIENT read. - const response = await requestHandler(await withClientReplayScriptSources(request)); - assertNoInternalChromeProvenance(response); - return response; + }) as DaemonRuntimeDeps; }; - const transport: AgentDeviceDaemonTransport = async (req) => - await handleRequest({ - token: PROVIDER_SCENARIO_TOKEN, - session: req.session ?? 'default', - command: req.command, - positionals: req.positionals, - input: req.input, - flags: req.flags, - runtime: req.runtime, - meta: req.meta as DaemonWireRequest['meta'], + let runtime: Awaited>; + try { + runtime = await startDaemonRuntime({ + env, + registerProcessHandlers: false, + stdout: { write: () => undefined }, + stderr: { write: () => undefined }, + exit: () => undefined, + composeDeps, + }); + } catch (error) { + await removeProviderScenarioTempDir(testRoot); + throw error; + } + if (!runtime?.httpPort) { + await removeProviderScenarioTempDir(testRoot); + throw new Error('Provider scenario daemon did not start an HTTP server'); + } + const httpPort = runtime.httpPort; + const baseUrl = `http://127.0.0.1:${httpPort}`; + const transport: AgentDeviceDaemonTransport = async (request, context) => { + const wireRequest: DaemonWireRequest = { + ...request, + token: context?.authToken ?? runtime.token, + session: request.session ?? 'default', + }; + const payload = buildHttpRpcPayload(wireRequest, { includeTokenParam: true }); + const { body } = await postProviderScenarioRpc({ + port: httpPort, + token: wireRequest.token, + payload, + signal: context?.signal, + }); + return await new Promise((resolve, reject) => { + handleDaemonHttpResponseBody(body, { + info: { + transport: 'http', + httpPort, + token: wireRequest.token, + pid: process.pid, + }, + req: wireRequest, + stateDir, + resolve, + reject, + }); }); + }; + const client = createAgentDeviceClient({ daemonAuthToken: runtime.token }, { transport }); + let closed = false; return { - callCommand: async (command, positionals = [], flags = {}, options = {}) => - responseToRpcResult( - await handleRequest(commandRequest(command, positionals, flags, options)), - `direct-${command}-${Date.now()}`, - ), - client: () => createAgentDeviceClient({}, { transport }), - handleRequest, - token: PROVIDER_SCENARIO_TOKEN, - session: (name = 'default') => sessionStore.get(name), - sessionDir: (name = 'default') => sessionStore.resolveSessionDir(name), - publishSession: (name, session) => sessionStore.publish(name, session), + callCommand: async (command, positionals = [], flags = {}, options = {}) => { + const request = await withClientReplayScriptSources( + commandRequest(runtime.token, command, positionals, flags, options), + ); + const payload = buildHttpRpcPayload(request, { includeTokenParam: true }); + const { statusCode, body } = await postProviderScenarioRpc({ + port: httpPort, + token: runtime.token, + payload, + }); + const envelope = JSON.parse(body) as JsonRpcResponseEnvelope; + if (envelope.result) assertNoInternalChromeProvenance(envelope.result); + if (envelope.result) { + return responseToRpcResult(envelope.result, String(payload.id), statusCode); + } + return { + statusCode, + json: { + jsonrpc: '2.0', + id: payload.id, + error: envelope.error ?? { code: statusCode, message: '' }, + }, + }; + }, + client: () => client, + hasSession: async (name = 'default') => + (await client.sessions.list()).some((session) => session.name === name), + baseUrl, + httpPort: runtime.httpPort, + token: runtime.token, + sessionDir: (name = 'default') => resolveSessionDir(daemonPaths.sessionsDir, name), close: async () => { - await deviceRuntimeGateway.shutdown(); - await removeProviderScenarioTempDir(sessionDir); + if (closed) return; + closed = true; + if (scopedAppleToolProvider) { + const { withAppleToolProvider } = + await import('@agent-device/platform-apple/tool-provider'); + await withAppleToolProvider(scopedAppleToolProvider, async () => await runtime?.shutdown()); + } else { + await runtime?.shutdown(); + } + await removeProviderScenarioTempDir(testRoot); }, }; } +async function postProviderScenarioRpc(params: { + port: number; + token: string; + payload: ReturnType; + signal?: AbortSignal; +}): Promise<{ statusCode: number; body: string }> { + const body = JSON.stringify(params.payload); + return await new Promise((resolve, reject) => { + const request = http.request( + { + hostname: '127.0.0.1', + port: params.port, + path: '/rpc', + method: 'POST', + headers: { + authorization: `Bearer ${params.token}`, + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(body), + }, + signal: params.signal, + }, + (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer | string) => { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + response.on('end', () => { + resolve({ + statusCode: response.statusCode ?? 0, + body: Buffer.concat(chunks).toString('utf8'), + }); + }); + }, + ); + request.on('error', reject); + request.end(body); + }); +} + export async function withProviderScenarioResource< TResource extends ClosableProviderScenarioResource, TResult, @@ -326,50 +419,6 @@ function atom(type: string, payload = Buffer.alloc(0)): Buffer { return Buffer.concat([header, payload]); } -function commandRequest( - command: string, - positionals: string[] = [], - flags: DaemonWireRequest['flags'] = {}, - options: { - input?: DaemonWireRequest['input']; - meta?: DaemonWireRequest['meta']; - runtime?: DaemonWireRequest['runtime']; - session?: string; - } = {}, -): DaemonWireRequest { - return { - token: PROVIDER_SCENARIO_TOKEN, - session: options.session ?? 'default', - command, - positionals, - input: options.input, - flags, - runtime: options.runtime, - meta: options.meta, - }; -} - -function responseToRpcResult(response: DaemonResponse, id: string): ProviderScenarioRpcResult { - return { - statusCode: 200, - json: response.ok - ? { - jsonrpc: '2.0', - id, - result: { data: response.data ?? {} }, - } - : { - jsonrpc: '2.0', - id, - error: { - code: -32000, - message: response.error.message, - data: response.error, - }, - }, - }; -} - function assertNoInternalChromeProvenance(response: DaemonResponse): void { assert.equal( JSON.stringify(response).includes('"systemChrome"'), diff --git a/test/integration/provider-scenarios/help-conformance-error-routes.test.ts b/test/integration/provider-scenarios/help-conformance-error-routes.test.ts new file mode 100644 index 0000000000..ee6d42e7bd --- /dev/null +++ b/test/integration/provider-scenarios/help-conformance-error-routes.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test, vi } from 'vitest'; + +const { createProviderScenarioHarness } = vi.hoisted(() => ({ + createProviderScenarioHarness: vi.fn(), +})); + +vi.mock('./harness.ts', () => ({ createProviderScenarioHarness })); + +import { captureHelpConformanceErrorRoutes } from './help-conformance-error-routes.ts'; + +test('failed daemon startup restores help-conformance environment and temp directories', async () => { + const originalTempDir = process.env.TMPDIR; + const originalClaimsDirectory = process.env.AGENT_DEVICE_CLAIMS_DIR; + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-help-route-cleanup-')); + createProviderScenarioHarness.mockRejectedValue(new Error('forced startup failure')); + + try { + process.env.TMPDIR = tempRoot; + process.env.AGENT_DEVICE_CLAIMS_DIR = 'previous-claims-directory'; + await assert.rejects(captureHelpConformanceErrorRoutes(), /forced startup failure/); + + assert.equal(createProviderScenarioHarness.mock.calls.length, 1); + assert.equal(process.env.AGENT_DEVICE_CLAIMS_DIR, 'previous-claims-directory'); + assert.deepEqual(fs.readdirSync(tempRoot), []); + } finally { + if (originalTempDir === undefined) delete process.env.TMPDIR; + else process.env.TMPDIR = originalTempDir; + if (originalClaimsDirectory === undefined) delete process.env.AGENT_DEVICE_CLAIMS_DIR; + else process.env.AGENT_DEVICE_CLAIMS_DIR = originalClaimsDirectory; + fs.rmSync(tempRoot, { recursive: true, force: true }); + createProviderScenarioHarness.mockReset(); + } +}); diff --git a/test/integration/provider-scenarios/help-conformance-error-routes.ts b/test/integration/provider-scenarios/help-conformance-error-routes.ts new file mode 100644 index 0000000000..906350c07e --- /dev/null +++ b/test/integration/provider-scenarios/help-conformance-error-routes.ts @@ -0,0 +1,341 @@ +import assert from 'node:assert/strict'; +import type { AndroidAdbProvider } from '@agent-device/platform-android/mechanics'; +import { deviceIdentity } from '@agent-device/kernel/device'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { WebProvider } from '@agent-device/platform-web'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { deviceClaimFilePath } from '@agent-device/test-kit'; +import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; +import { ANDROID_IME_HELPER_FIXTURE_ARTIFACT } from '../../../src/__tests__/test-utils/android-ime-helper.ts'; +import { + ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + androidSnapshotHelperOutput, +} from '../../../src/__tests__/test-utils/android-snapshot-helper.ts'; +import { createProviderScenarioHarness, type ProviderScenarioRpcResult } from './harness.ts'; +import { PROVIDER_SCENARIO_ANDROID, PROVIDER_SCENARIO_WEB } from './fixtures.ts'; +import { androidSettingsXml } from './android-world.ts'; + +type HelpConformanceErrorResponse = { + ok: false; + error: { + code: string; + message: string; + hint?: string; + retriable?: boolean; + details?: Record; + }; +}; + +const checkoutNodes: RawSnapshotNode[] = [ + { + index: 0, + ref: 'e1', + type: 'Window', + role: 'window', + label: 'Checkout', + depth: 0, + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + role: 'button', + label: 'Place order', + hittable: true, + enabled: true, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 720, width: 350, height: 48 }, + }, +]; + +const ambiguousNodes: RawSnapshotNode[] = [ + { + index: 0, + ref: 'e1', + type: 'Window', + role: 'window', + label: 'Feed', + depth: 0, + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ...[ + { index: 1, ref: 'e2' }, + { index: 4, ref: 'e5' }, + { index: 8, ref: 'e9' }, + ].map(({ index, ref }) => ({ + index, + ref, + type: 'Button', + role: 'button', + label: 'Follow', + hittable: true, + enabled: true, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: index * 40, width: 120, height: 32 }, + })), +]; + +function emptyWebProvider(nodes: () => RawSnapshotNode[]): WebProvider { + return { + async open() {}, + async close() {}, + async snapshot() { + return { nodes: nodes(), truncated: false }; + }, + async screenshot() {}, + async setViewport() {}, + async click() {}, + async fill() {}, + async typeText() {}, + async scroll() {}, + }; +} + +function responseError( + response: ProviderScenarioRpcResult, + expectedCode: string, + expectedStatus: number, + label: string, +): HelpConformanceErrorResponse { + assert.equal(response.statusCode, expectedStatus, `${label}: ${JSON.stringify(response.json)}`); + const error = response.json?.error?.data; + assert.equal(error?.code, expectedCode, `${label}: ${JSON.stringify(response.json)}`); + assert.equal(typeof error?.message, 'string'); + return { + ok: false, + error: { + code: error.code, + message: error.message, + ...(typeof error.hint === 'string' ? { hint: error.hint } : {}), + ...(typeof error.retriable === 'boolean' ? { retriable: error.retriable } : {}), + ...(error.details && typeof error.details === 'object' + ? { details: error.details as Record } + : {}), + }, + }; +} + +function androidAdbProvider(): AndroidAdbProvider { + return { + snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + imeHelperArtifact: ANDROID_IME_HELPER_FIXTURE_ARTIFACT, + exec: async (args) => { + const command = args.join(' '); + if (command === 'shell getprop sys.boot_completed') { + return { stdout: '1\n', stderr: '', exitCode: 0 }; + } + if (command === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{42 u0 com.android.settings/.Settings}\n', + stderr: '', + exitCode: 0, + }; + } + if (command.startsWith('shell am instrument ')) { + return { + stdout: androidSnapshotHelperOutput(androidSettingsXml('')), + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }; +} + +export async function captureHelpConformanceErrorRoutes(): Promise<{ + deviceInUse: HelpConformanceErrorResponse; + deviceClaimInUse: HelpConformanceErrorResponse; + staleRef: HelpConformanceErrorResponse; + ambiguousMatch: HelpConformanceErrorResponse; +}> { + let currentPage: 'checkout' | 'feed' = 'checkout'; + const webProvider = emptyWebProvider(() => + currentPage === 'checkout' ? checkoutNodes : ambiguousNodes, + ); + let claimsDirectory: string | undefined; + let ownerStateDirectory: string | undefined; + const previousClaimsDirectory = process.env.AGENT_DEVICE_CLAIMS_DIR; + let webDaemon: Awaited> | undefined; + let androidDaemon: Awaited> | undefined; + + try { + const activeClaimsDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'agent-device-help-conformance-'), + ); + claimsDirectory = activeClaimsDirectory; + const activeOwnerStateDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'agent-device-help-conformance-owner-'), + ); + ownerStateDirectory = activeOwnerStateDirectory; + process.env.AGENT_DEVICE_CLAIMS_DIR = activeClaimsDirectory; + const activeWebDaemon = await createProviderScenarioHarness({ + deviceInventoryProvider: async (request) => + request.platform === 'web' ? [PROVIDER_SCENARIO_WEB] : [], + webProvider: () => webProvider, + }); + webDaemon = activeWebDaemon; + + const webOpen = async (session: string) => { + currentPage = 'checkout'; + const response = await activeWebDaemon.callCommand( + 'open', + ['https://example.test'], + { platform: 'web' }, + { session }, + ); + assert.equal(response.json?.error, undefined, JSON.stringify(response.json)); + }; + + await webOpen('checkout'); + const deviceInUse = responseError( + await activeWebDaemon.callCommand( + 'open', + ['https://example.test'], + { platform: 'web' }, + { session: 'default' }, + ), + 'DEVICE_IN_USE', + 500, + 'session owns the selected web device', + ); + + const firstSnapshot = await activeWebDaemon.callCommand( + 'snapshot', + [], + { platform: 'web' }, + { session: 'checkout' }, + ); + const firstGeneration = firstSnapshot.json?.result?.data?.refsGeneration; + assert.equal(typeof firstGeneration, 'number'); + const currentPress = await activeWebDaemon.callCommand( + 'press', + [`@e2~s${firstGeneration}`], + { platform: 'web' }, + { session: 'checkout' }, + ); + assert.equal(currentPress.json?.error, undefined, JSON.stringify(currentPress.json)); + const refreshedSnapshot = await activeWebDaemon.callCommand( + 'snapshot', + [], + { platform: 'web' }, + { session: 'checkout' }, + ); + const currentGeneration = refreshedSnapshot.json?.result?.data?.refsGeneration; + assert.equal(typeof currentGeneration, 'number'); + assert.notEqual(currentGeneration, firstGeneration); + const staleRefRaw = responseError( + await activeWebDaemon.callCommand( + 'press', + [`@e2~s${firstGeneration}`], + { platform: 'web' }, + { session: 'checkout' }, + ), + 'COMMAND_FAILED', + 500, + 'superseded pinned ref', + ); + assert.equal(staleRefRaw.error.details?.reason, 'ref_generation_mismatch'); + assert.equal(staleRefRaw.error.details?.mintedGeneration, firstGeneration); + assert.equal(staleRefRaw.error.details?.currentGeneration, currentGeneration); + const hint = staleRefRaw.error.hint; + if (!hint) throw new Error('stale-ref route omitted its generation recovery hint'); + assert.ok(hint.includes(`s${firstGeneration}`)); + assert.ok(hint.includes(`s${currentGeneration}`)); + const staleRef = { + ...staleRefRaw, + error: { + ...staleRefRaw.error, + hint: hint.replace(`s${firstGeneration}`, 's5').replace(`s${currentGeneration}`, 's7'), + }, + }; + + currentPage = 'feed'; + const ambiguousSnapshot = await activeWebDaemon.callCommand( + 'snapshot', + [], + { platform: 'web' }, + { session: 'checkout' }, + ); + assert.equal(ambiguousSnapshot.json?.error, undefined, JSON.stringify(ambiguousSnapshot.json)); + const ambiguousMatch = responseError( + await activeWebDaemon.callCommand( + 'find', + ['text', 'Follow'], + { platform: 'web' }, + { session: 'checkout' }, + ), + 'AMBIGUOUS_MATCH', + 500, + 'ambiguous Follow match', + ); + + const androidDevice = { ...PROVIDER_SCENARIO_ANDROID, name: 'Pixel' }; + const identity = deviceIdentity(androidDevice); + const owner = readCurrentOwnerIdentity(); + fs.writeFileSync( + deviceClaimFilePath('local:android:none:emulator-5554'), + JSON.stringify({ + schemaVersion: 2, + deviceKey: 'local:android:none:emulator-5554', + device: { ...identity, name: androidDevice.name }, + session: 'checkout', + workspace: '/worktrees/checkout', + stateDir: activeOwnerStateDirectory, + ownerPid: owner.pid, + ownerStartTime: owner.startTime, + ownerToken: 'help-conformance-owner', + createdAtMs: 1, + updatedAtMs: 1, + }), + 'utf8', + ); + androidDaemon = await createProviderScenarioHarness({ + androidAdbProvider, + deviceInventoryProvider: async (request) => + request.platform === 'android' ? [androidDevice] : [], + }); + const deviceClaimInUse = responseError( + await androidDaemon.callCommand( + 'open', + ['settings'], + { platform: 'android', serial: androidDevice.id }, + { meta: { cwd: '/worktrees/other' } }, + ), + 'DEVICE_IN_USE', + 500, + 'cross-daemon device claim', + ); + assert.match(deviceClaimInUse.error.message, /session "checkout"/); + assert.match(deviceClaimInUse.error.message, /workspace "\/worktrees\/checkout"/); + assert.equal(deviceClaimInUse.error.details?.reason, 'DEVICE_CLAIM_LIVE_OWNER'); + + return { deviceInUse, deviceClaimInUse, staleRef, ambiguousMatch }; + } finally { + try { + await androidDaemon?.close(); + } finally { + try { + await webDaemon?.close(); + } finally { + try { + if (claimsDirectory) fs.rmSync(claimsDirectory, { recursive: true, force: true }); + } finally { + try { + if (ownerStateDirectory) { + fs.rmSync(ownerStateDirectory, { recursive: true, force: true }); + } + } finally { + if (previousClaimsDirectory === undefined) delete process.env.AGENT_DEVICE_CLAIMS_DIR; + else process.env.AGENT_DEVICE_CLAIMS_DIR = previousClaimsDirectory; + } + } + } + } + } +} diff --git a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts index 7b158c2696..1a27db946e 100644 --- a/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts +++ b/test/integration/provider-scenarios/interaction-direct-selector-fallback.test.ts @@ -171,7 +171,7 @@ test('Provider-backed integration maestro replay dispatch keeps runner AMBIGUOUS const click = await daemon.callCommand('click', ['label="Continue"'], { maestro: { allowNonHittableCoordinateFallback: true }, }); - assertRpcError(click, 'AMBIGUOUS_MATCH', /matched multiple/); + assertRpcError(click, 500, 'AMBIGUOUS_MATCH', /matched multiple/); }); }); @@ -201,7 +201,7 @@ test( const click = await daemon.callCommand('click', ['label="Continue"'], { maestro: { allowNonHittableCoordinateFallback: true }, }); - assertRpcError(click, 'ELEMENT_OFFSCREEN', /resolved off-screen/); + assertRpcError(click, 500, 'ELEMENT_OFFSCREEN', /resolved off-screen/); }); }, PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, diff --git a/test/integration/provider-scenarios/ios-fold.test.ts b/test/integration/provider-scenarios/ios-fold.test.ts index 9cc2c082d8..ea3ed76d20 100644 --- a/test/integration/provider-scenarios/ios-fold.test.ts +++ b/test/integration/provider-scenarios/ios-fold.test.ts @@ -1,12 +1,9 @@ -import { formatPortableActionLine, parseReplayScriptDetailed } from '@agent-device/ad-script'; import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry'; import { resolveCommandRequestTimeoutMs } from '@agent-device/command-registry/timeout-policy'; import { MAX_FOLD_DURATION_MS } from '@agent-device/contracts/device'; import type { AppleToolProvider } from '@agent-device/platform-apple/tool-provider'; import type { ExecResult } from '@agent-device/host-kit/command'; -import { recordActionEntry } from '@agent-device/daemon/testing/session-action-recorder'; import { assertRpcError, assertRpcOk } from './assertions.ts'; -import { makeIosAppSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import os from 'node:os'; @@ -75,6 +72,18 @@ test('timed fold keyframes reach simulator HID through the public client and dae let angle = 0; const tool = createRecordingAppleToolProvider({ simctl: async (args, options) => { + if (args[0] === 'list') { + return { + stdout: JSON.stringify({ + devices: { + 'iOS 18.5': [{ udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, state: 'Booted' }], + }, + }), + stderr: '', + exitCode: 0, + }; + } + if (args[0] === 'launch') return ok; assert.equal(args[0], 'spawn'); assert.equal(args[1], PROVIDER_SCENARIO_IOS_SIMULATOR.id); assert.deepEqual(JSON.parse(args[3]!), trajectory); @@ -92,6 +101,10 @@ test('timed fold keyframes reach simulator HID through the public client and dae let builds = 0; const daemon = await createProviderScenarioHarness({ deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + appleRunnerProvider: () => ({ + hasLiveSession: () => true, + runCommand: async () => ({}), + }), appleToolProvider: () => ({ ...tool.provider, runCommand: async (command, args) => { @@ -105,11 +118,13 @@ test('timed fold keyframes reach simulator HID through the public client and dae }, }), }); - daemon.publishSession( - 'default', - makeIosAppSession('default', { device: PROVIDER_SCENARIO_IOS_SIMULATOR }), - ); try { + await daemon.client().apps.open({ + app: 'com.example.demo', + platform: 'ios', + udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + }); + tool.calls.splice(0); const result = await daemon.client().command.fold({ platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, @@ -117,20 +132,11 @@ test('timed fold keyframes reach simulator HID through the public client and dae }); assert.equal(result.pose, 'half-open'); assert.equal(result.hingeAngleDegrees, 100); - const recorded = recordActionEntry(daemon.session()!, { - command: 'fold', - positionals: [], - flags: { keyframes: JSON.stringify(trajectory) }, - result, + const replayed = await daemon.client().command.fold({ + platform: 'ios', + udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + keyframes: trajectory, }); - assert.ok(recorded); - const line = formatPortableActionLine(recorded); - assert.match(line, /--keyframes/); - const [parsed] = parseReplayScriptDetailed(line).actions; - assert.ok(parsed); - const replayed = assertRpcOk( - await daemon.callCommand(parsed.command, parsed.positionals ?? [], parsed.flags), - ); assert.equal(replayed.hingeAngleDegrees, 100); // The second fold call reuses the cached fold helper binary instead of rebuilding (#2796). assert.equal(builds, 1); @@ -163,6 +169,7 @@ function alternatingHingeAngle(readIndex: number): number { function createFoldLedgerAppleToolProvider(params: { hingeAngleAt: (readIndex: number) => number; onCall: (call: FoldLedgerCall) => void; + isTracking: () => boolean; }): { provider: AppleToolProvider; ledger: FoldLedgerCall[]; hingeReadCount: () => number } { const ledger: FoldLedgerCall[] = []; let hingeReads = 0; @@ -193,6 +200,19 @@ function createFoldLedgerAppleToolProvider(params: { // macosHost, plist, or an unexpected runCommand) throws instead of being silently answered. const recording = createRecordingAppleToolProvider({ simctl: async (args, options) => { + if (args[0] === 'list') { + if (params.isTracking()) record('simctl', args, options); + return { + stdout: JSON.stringify({ + devices: { + 'iOS 18.5': [{ udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, state: 'Booted' }], + }, + }), + stderr: '', + exitCode: 0, + }; + } + if (args[0] === 'launch') return ok; record('simctl', args, options); return ok; }, @@ -264,25 +284,36 @@ async function runFoldLedgerScenario(params: { { atMs: MAX_FOLD_DURATION_MS, angle: 100 }, ]; let virtualElapsedMs = 0; - const originNowMs = Date.now(); - const dateSpy = params.withVirtualClock - ? vi.spyOn(Date, 'now').mockImplementation(() => originNowMs + virtualElapsedMs) - : undefined; + const dateSpy = params.withVirtualClock ? vi.spyOn(Date, 'now') : undefined; + let trackFoldCalls = false; const { provider, ledger, hingeReadCount } = createFoldLedgerAppleToolProvider({ hingeAngleAt: params.hingeAngleAt, + isTracking: () => trackFoldCalls, onCall: (call) => { virtualElapsedMs += Math.max(0, call.timeoutMs - 1) + (call.graceMs ?? 0); }, }); const daemon = await createProviderScenarioHarness({ deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + appleRunnerProvider: () => ({ + hasLiveSession: () => true, + runCommand: async () => ({}), + }), appleToolProvider: () => provider, }); - daemon.publishSession( - 'default', - makeIosAppSession('default', { device: PROVIDER_SCENARIO_IOS_SIMULATOR }), - ); try { + await daemon.client().apps.open({ + app: 'com.example.demo', + platform: 'ios', + udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + }); + ledger.splice(0); + virtualElapsedMs = 0; + trackFoldCalls = true; + if (dateSpy) { + const originNowMs = Date.now(); + dateSpy.mockImplementation(() => originNowMs + virtualElapsedMs); + } const response = await daemon.callCommand('fold', [], { platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, @@ -328,6 +359,7 @@ test('fold worst-case ledger covers the client envelope with the daemon-result m }); const calibrationError = assertRpcError( calibration.response, + 500, 'COMMAND_FAILED', /did not settle/, ) as { details?: { reason?: unknown } }; diff --git a/test/integration/provider-scenarios/ios-record-trace.test.ts b/test/integration/provider-scenarios/ios-record-trace.test.ts index 073cccfcab..2c60281944 100644 --- a/test/integration/provider-scenarios/ios-record-trace.test.ts +++ b/test/integration/provider-scenarios/ios-record-trace.test.ts @@ -39,7 +39,6 @@ test('generic scoped iOS physical runner recording fails closed without local fa }, }); const daemon = await createProviderScenarioHarness({ - platformRuntime: true, appleRunnerProvider: () => createAppleRunnerProviderFromTranscript(runnerTranscript, 'ios.runner'), appleToolProvider: () => appleTool.provider, @@ -106,7 +105,6 @@ test('iOS simulator recording reports host contention and recovers through the f }, }; const daemon = await createProviderScenarioHarness({ - platformRuntime: true, appleRunnerProvider: () => appleRunnerProvider, appleToolProvider: () => appleTool.provider, appleSimulatorScreenRecordingTransport: () => recordingTransport, @@ -137,7 +135,10 @@ test('iOS simulator recording reports host contention and recovers through the f assert.equal(busyStart.json?.error?.data?.details?.exitCode, 16); assert.match(busyStart.json?.error?.data?.hint ?? '', /record stop/); assert.match(busyStart.json?.error?.data?.hint ?? '', /CoreSimulator/); - assert.equal(daemon.session()?.screenRecording, undefined); + assert.equal( + fs.existsSync(path.join(daemon.sessionDir(), 'screen-recording.resource.json')), + false, + ); assert.equal(fs.existsSync(recordingPath), false); const recordStart = await daemon.callCommand( diff --git a/test/integration/provider-scenarios/ios-wait-selector.test.ts b/test/integration/provider-scenarios/ios-wait-selector.test.ts index 12c0452eb5..92cfe8a903 100644 --- a/test/integration/provider-scenarios/ios-wait-selector.test.ts +++ b/test/integration/provider-scenarios/ios-wait-selector.test.ts @@ -3,7 +3,6 @@ import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { makeIosAppSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { assertRpcOk } from './assertions.ts'; @@ -17,7 +16,23 @@ test.each([ ['replay', 'text'], ] as const)('Apple %s %s captures like is when native queries fail', async (command, target) => { const commands: string[] = []; - const appleTool = createRecordingAppleToolProvider(); + const appleTool = createRecordingAppleToolProvider({ + simctl: async (args) => { + if (args[0] === 'list') { + return { + stdout: JSON.stringify({ + devices: { + 'iOS 18.5': [{ udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, state: 'Booted' }], + }, + }), + stderr: '', + exitCode: 0, + }; + } + if (args[0] === 'launch') return { stdout: '', stderr: '', exitCode: 0 }; + throw new Error(`Unexpected Apple simulator call: ${args.join(' ')}`); + }, + }); const daemon = await createProviderScenarioHarness({ deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], appleToolProvider: () => appleTool.provider, @@ -54,11 +69,13 @@ test.each([ }, }), }); - daemon.publishSession('default', { - ...makeIosAppSession('default'), - device: PROVIDER_SCENARIO_IOS_SIMULATOR, - }); try { + await daemon.client().apps.open({ + app: 'com.example.demo', + platform: 'ios', + udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + }); + commands.length = 0; const flags = { platform: 'ios' as const }; assert.equal( assertRpcOk(await daemon.callCommand('is', ['visible', 'id="LANDMARK_A"'], flags)).pass, @@ -84,7 +101,9 @@ test.each([ ); assert.deepEqual( commands, - target === 'selector' ? ['snapshot', 'snapshot'] : ['snapshot', 'findText', 'snapshot'], + target === 'selector' + ? ['snapshot', 'snapshot', 'snapshot'] + : ['snapshot', 'snapshot', 'findText', 'snapshot'], ); } finally { await daemon.close(); diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 345a668cf7..64b92c0ddd 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -236,7 +236,6 @@ export async function createIosSettingsWorld(): Promise { }); const daemon = await createProviderScenarioHarness({ - platformRuntime: true, appleRunnerProvider: () => appleRunnerProvider, appleToolProvider: () => appleTool.provider, deviceInventoryProvider: async (request) => { @@ -406,7 +405,6 @@ export async function createIosPhysicalReinstallWorld(): Promise appleTool.provider, deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_REINSTALL_DEVICE], - platformRuntime: true, }); const { tempRoot, appPath } = createDemoIosApp( 'agent-device-provider-scenario-ios-physical-deploy-', diff --git a/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts b/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts index fc27eed829..fa879ce941 100644 --- a/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts +++ b/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts @@ -58,16 +58,6 @@ vi.mock('@limrun/api/ios-client', () => ({ })), })); -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { ...actual, runAppleRunnerCommand: vi.fn(async () => ({})) }; -}); - -vi.mock('@agent-device/daemon/testing/ios-app-session-hint', () => ({ - buildIosOpenCommandHint: vi.fn(async () => undefined), -})); - beforeEach(() => { limrunApiState.createInstance.mockClear(); limrunApiState.deleteInstance.mockClear(); @@ -94,6 +84,7 @@ test('Limrun iOS acquisition reaches one host presenter and the public snapshot try { const lease = await allocateLease(daemon); + assert.equal(limrunApiState.createInstance.mock.calls.length, 1); const flags = { platform: 'ios' as const, tenant: 'team-a', diff --git a/test/integration/provider-scenarios/linux-desktop.test.ts b/test/integration/provider-scenarios/linux-desktop.test.ts index 27305ad166..8328958abf 100644 --- a/test/integration/provider-scenarios/linux-desktop.test.ts +++ b/test/integration/provider-scenarios/linux-desktop.test.ts @@ -206,26 +206,6 @@ test('Provider-backed integration Linux desktop flow uses semantic desktop and i { name: 'show desktop', command: 'home' }, ]); - const actions = daemon.session()?.actions ?? []; - assert.ok( - actions.some( - (action) => - action.command === 'fill' && - action.positionals.join(' ') === '@e2 Seven' && - action.flags.delayMs === 1, - ), - 'Expected ref fill action to be recorded on the session', - ); - assert.ok( - actions.some( - (action) => - action.command === 'fill' && - action.positionals.join(' ') === '42 84 Eight' && - action.flags.delayMs === 1, - ), - 'Expected coordinate fill action to be recorded on the session', - ); - const close = await daemon.callCommand('close', ['gnome-calculator']); assert.equal(close.statusCode, 200, JSON.stringify(close.json)); assert.deepEqual(desktopCalls, [ diff --git a/test/integration/provider-scenarios/macos-desktop.test.ts b/test/integration/provider-scenarios/macos-desktop.test.ts index e982520fa5..c49e4d457c 100644 --- a/test/integration/provider-scenarios/macos-desktop.test.ts +++ b/test/integration/provider-scenarios/macos-desktop.test.ts @@ -264,8 +264,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help (node: { label?: string }) => node.label === 'General', ); assert.equal(general?.ref, 'e2', JSON.stringify(snapshot.json)); - assert.equal(daemon.session()?.snapshot?.backend, 'macos-helper'); - assert.equal(daemon.session()?.snapshot?.nodes[0]?.surface, 'frontmost-app'); }, }, { @@ -309,6 +307,7 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help }); const frontmostRefusalErrorData = assertRpcError( frontmostFullscreenRefusal, + 400, 'INVALID_ARGS', /--fullscreen is not accepted on the macOS frontmost-app surface/, ); @@ -351,6 +350,7 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help }); const refusalErrorData = assertRpcError( fullscreenRefusal, + 400, 'INVALID_ARGS', /--fullscreen is not accepted on the macOS desktop surface/, ); @@ -410,8 +410,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help snapshot.json?.result?.data?.nodes?.map((node: { label?: string }) => node.label), ['Desktop', 'Notes', 'Notes', 'Pinned'], ); - assert.equal(daemon.session()?.snapshot?.backend, 'macos-helper'); - assert.equal(daemon.session()?.snapshot?.nodes[0]?.surface, 'desktop'); }, }, { @@ -430,7 +428,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help assert.equal(nodes[0]?.label, 'Notes', JSON.stringify(snapshot.json)); assert.equal(nodes[0]?.depth, 0, JSON.stringify(snapshot.json)); assert.equal(nodes[0]?.parentIndex, undefined, JSON.stringify(snapshot.json)); - assert.equal(daemon.session()?.snapshot?.backend, 'macos-helper'); }, }, { @@ -453,8 +450,6 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help snapshot.json?.result?.data?.nodes?.map((node: { label?: string }) => node.label), ['Menu Bar', 'File'], ); - assert.equal(daemon.session()?.snapshot?.backend, 'macos-helper'); - assert.equal(daemon.session()?.snapshot?.nodes[0]?.surface, 'menubar'); }, }, { @@ -485,8 +480,7 @@ test('Provider-backed integration macOS desktop flow uses semantic host and help snapshot.json?.result?.data?.nodes?.map((node: { label?: string }) => node.label), ['Menu Bar', 'Demo'], ); - assert.equal(daemon.session()?.snapshot?.backend, 'macos-helper'); - assert.equal(daemon.session()?.snapshot?.nodes[1]?.bundleId, 'com.example.demo'); + assert.equal(snapshot.json?.result?.data?.nodes?.[1]?.bundleId, 'com.example.demo'); }, }, ]); diff --git a/test/integration/provider-scenarios/macos-recording.test.ts b/test/integration/provider-scenarios/macos-recording.test.ts index db8a2144a0..0cc74eb3c5 100644 --- a/test/integration/provider-scenarios/macos-recording.test.ts +++ b/test/integration/provider-scenarios/macos-recording.test.ts @@ -83,11 +83,10 @@ test('Provider-backed integration macOS recording keeps a runner-stop failure’ assertRecordingStarted(recordStart, { outPath: recordingPath, showTouches: false }); const recordStop = await daemon.callCommand('record', ['stop']); - assertRpcError(recordStop, 'COMMAND_FAILED', /recordStop rejected/); + assertRpcError(recordStop, 500, 'COMMAND_FAILED', /recordStop rejected/); assert.equal(recordStop.json?.result, undefined); - const session = daemon.session(); - assert.ok(session?.screenRecording, 'failed stop must retain the live session resource'); + assert.equal(await daemon.hasSession(), true); const resource = screenRecordingResourceStore.read( screenRecordingResourceStore.resolvePath(daemon.sessionDir()), ); @@ -127,12 +126,9 @@ test('Provider-backed integration macOS recording keeps an unplayable recording assertRecordingStarted(recordStart, { outPath: recordingPath, showTouches: false }); const recordStop = await daemon.callCommand('record', ['stop']); - assertRpcError(recordStop, 'COMMAND_FAILED', /was not finalized into a playable video/); + assertRpcError(recordStop, 500, 'COMMAND_FAILED', /was not finalized into a playable video/); assert.equal(recordStop.json?.result, undefined); - assert.ok( - daemon.session()?.screenRecording, - 'an unplayable recording must stay attached so the next record stop can retry it', - ); + assert.equal(await daemon.hasSession(), true); const resource = screenRecordingResourceStore.read( screenRecordingResourceStore.resolvePath(daemon.sessionDir()), @@ -145,7 +141,7 @@ test('Provider-backed integration macOS recording keeps an unplayable recording runnerTranscript.assertComplete(); }, ); -}); +}, 15_000); function createMacOsRecordingScenario( recordingPath: string, diff --git a/test/integration/provider-scenarios/macos-world.ts b/test/integration/provider-scenarios/macos-world.ts index 3fbb9897cc..32d4fd9603 100644 --- a/test/integration/provider-scenarios/macos-world.ts +++ b/test/integration/provider-scenarios/macos-world.ts @@ -49,7 +49,6 @@ export async function createMacOsDesktopWorld( }, }); const daemon = await createProviderScenarioHarness({ - platformRuntime: true, appleRunnerProvider: options.appleRunnerProvider ? () => options.appleRunnerProvider : undefined, diff --git a/test/integration/provider-scenarios/network-runtime-provider.test.ts b/test/integration/provider-scenarios/network-runtime-provider.test.ts index 84f61db51c..4a8d5e95eb 100644 --- a/test/integration/provider-scenarios/network-runtime-provider.test.ts +++ b/test/integration/provider-scenarios/network-runtime-provider.test.ts @@ -2,11 +2,23 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness } from './harness.ts'; -import { createRecordingAppleToolProvider } from './providers.ts'; +import { createRecordingAppleToolProvider, simctlListDevicesResult } from './providers.ts'; test('iOS simulator network recovery stays on the request-scoped Apple tool provider', async () => { const appleTool = createRecordingAppleToolProvider({ simctl: async (args) => { + const devices = simctlListDevicesResult(args, 'com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: 'iPhone 15', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id }, + ]); + if (devices) return devices; + if (args.join(' ') === 'listapps sim-1') { + return { + stdout: '{"com.example.demo":{"CFBundleDisplayName":"Demo"}}', + stderr: '', + exitCode: 0, + }; + } + if (args[0] === 'launch') return { stdout: '', stderr: '', exitCode: 0 }; assert.deepEqual(args.slice(0, 4), ['spawn', 'sim-1', 'log', 'show']); return { stdout: '2026-08-10T10:00:00Z GET https://provider-recovery.example.test status=200\n', @@ -16,19 +28,15 @@ test('iOS simulator network recovery stays on the request-scoped Apple tool prov }, }); const daemon = await createProviderScenarioHarness({ - platformRuntime: true, appleToolProvider: () => appleTool.provider, deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], }); - daemon.publishSession('default', { - name: 'default', - device: PROVIDER_SCENARIO_IOS_SIMULATOR, - appBundleId: 'com.example.app', - createdAt: Date.now(), - actions: [], - }); - try { + await daemon.client().apps.open({ + app: 'com.example.demo', + platform: 'ios', + udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + }); const result = await daemon.client().observability.network({ action: 'dump', limit: 5, diff --git a/test/integration/provider-scenarios/press-selector-resolution.test.ts b/test/integration/provider-scenarios/press-selector-resolution.test.ts index 8928c630da..0dad270e79 100644 --- a/test/integration/provider-scenarios/press-selector-resolution.test.ts +++ b/test/integration/provider-scenarios/press-selector-resolution.test.ts @@ -119,7 +119,7 @@ test('press refuses a selector miss after one interactive and full capture attem async (daemon, transcript) => { const callsBeforePress = transcript.calls.length; const press = await daemon.callCommand('press', ['label=Continue']); - const error = assertRpcError(press, 'COMMAND_FAILED', /Selector did not match/); + const error = assertRpcError(press, 500, 'COMMAND_FAILED', /Selector did not match/); const details = error.details as { reason: unknown; dispatched?: unknown }; assert.equal(details.reason, 'selector_not_found'); assert.equal(details.dispatched, 'no'); diff --git a/test/integration/provider-scenarios/provider-device-runtime.test.ts b/test/integration/provider-scenarios/provider-device-runtime.test.ts index b2ef662295..75cf201bd1 100644 --- a/test/integration/provider-scenarios/provider-device-runtime.test.ts +++ b/test/integration/provider-scenarios/provider-device-runtime.test.ts @@ -29,6 +29,34 @@ test('Provider-backed scenario composes lease, inventory, dispatch, and port rev }); }, 15_000); +test('provider session retains the allocated device and lease for implicit commands', async () => { + await withProviderScenarioResource(createFakeProviderWorld, async ({ daemon, runtime }) => { + const lease = await allocateFakeProviderLease(daemon); + assertRpcOk( + await daemon.callCommand('open', ['com.example.demo'], leaseFlags(lease.leaseId), { + meta: leaseMeta(lease.leaseId), + }), + ); + + const sessions = await daemon.client().sessions.list(); + assert.equal(sessions.length, 1); + assert.equal(sessions[0]?.device.id, runtime.deviceIdForLease(lease.leaseId)); + + const click = await daemon.callCommand('click', ['10', '20']); + assertRpcOk(click); + assert.deepEqual( + runtime.calls.filter((call) => call.type === 'tap').map((call) => call.deviceId), + [runtime.deviceIdForLease(lease.leaseId)], + ); + + assertRpcOk(await daemon.callCommand('close')); + assert.deepEqual( + runtime.calls.filter((call) => call.type === 'lease.release').map((call) => call.leaseId), + [lease.leaseId], + ); + }); +}, 15_000); + test('provider-owned iOS simulators relaunch without local CoreSimulator refresh', async () => { await withProviderScenarioResource( async () => await createFakeProviderWorld('ios'), @@ -88,6 +116,7 @@ test('provider lease allocation fails when the daemon lacks the requested runtim ); const error = assertRpcError( response, + 500, 'UNSUPPORTED_OPERATION', /Provider "limrun" is not available in this daemon runtime/, ); @@ -134,7 +163,7 @@ async function runFakeProviderScenario( flags: leaseFlags(lease.leaseId), meta: leaseMeta(lease.leaseId), }); - assertFakeProviderScenarioResult(daemon, runtime, lease, appPath); + assertFakeProviderScenarioResult(runtime, lease, appPath); }); } @@ -221,15 +250,11 @@ function expectedPortReverseData(leaseId: string): Record { } function assertFakeProviderScenarioResult( - daemon: Awaited>, runtime: FakeProviderDeviceRuntime, lease: DeviceLease, appPath: string, ): void { const deviceId = runtime.deviceIdForLease(lease.leaseId); - const session = daemon.session(); - assert.equal(session?.device.id, deviceId); - assert.equal(session?.lease?.leaseId, lease.leaseId); assert.deepEqual( runtime.calls.find((call) => call.type === 'install'), { diff --git a/test/integration/provider-scenarios/provider-failures.test.ts b/test/integration/provider-scenarios/provider-failures.test.ts index 2fdaeccea9..49253f898d 100644 --- a/test/integration/provider-scenarios/provider-failures.test.ts +++ b/test/integration/provider-scenarios/provider-failures.test.ts @@ -31,7 +31,7 @@ test('Provider-backed integration normalizes provider failures through the reque snapshotInteractiveOnly: true, }); - assert.equal(response.statusCode, 200); + assert.equal(response.statusCode, 500); assert.equal(response.json?.error?.data?.code, 'COMMAND_FAILED'); assert.match( response.json?.error?.message ?? '', diff --git a/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts b/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts index dc438b5648..6895cb665b 100644 --- a/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts +++ b/test/integration/provider-scenarios/provider-ios-runner-transport.test.ts @@ -144,6 +144,7 @@ test('provider transport refuses a fold before any runner traffic', async () => calls.runner.length = 0; assertRpcError( await daemon.callCommand('fold', ['open'], request.flags, request), + 500, 'UNSUPPORTED_OPERATION', /fold/, ); diff --git a/test/integration/provider-scenarios/remote-proxy-artifact-tenant.test.ts b/test/integration/provider-scenarios/remote-proxy-artifact-tenant.test.ts index 7f2050a967..6cdb73743d 100644 --- a/test/integration/provider-scenarios/remote-proxy-artifact-tenant.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-artifact-tenant.test.ts @@ -1,171 +1,299 @@ +import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; -import { createAgentDeviceClient } from '../../../src/agent-device-client.ts'; -import { - cleanupDownloadableArtifact, - trackDownloadableArtifact, -} from '@agent-device/daemon/testing/artifact-tracking'; -import { finalizeDaemonResponse } from '@agent-device/daemon/testing/request-finalization'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { normalizeAgentDeviceError } from '@agent-device/kernel/errors'; -import { downloadRemoteArtifact } from '@agent-device/remote/daemon-artifacts'; -import { createDaemonProxyServer } from '@agent-device/proxy'; import { - closeLoopbackServer, - listenOnLoopback, - skipWhenLoopbackUnavailable, -} from '@agent-device/test-kit'; + downloadRemoteArtifact, + materializeRemoteArtifacts, +} from '@agent-device/remote/daemon-artifacts'; +import { skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; +import { withProviderScenarioResource, withProviderScenarioTempDir } from './harness.ts'; +import { createProviderScenarioProxy } from './harness-proxy.ts'; +import { FAKE_PROVIDER, createFakeProviderWorld } from './provider-device-runtime.fixtures.ts'; const TENANT = 'local-proxy-tenant'; const OTHER_TENANT = 'other-tenant'; +const TENANT_TOKEN = 'tenant-a-key'; +const OTHER_TENANT_TOKEN = 'tenant-b-key'; -test('Provider-backed integration local proxy materializes tenant-scoped screenshots', async (t) => { +function tenantAuthentication(request: Request) { + const token = request.headers.get('authorization'); + if (token === `Bearer ${TENANT_TOKEN}`) return { tenantId: TENANT }; + if (token === `Bearer ${OTHER_TENANT_TOKEN}`) return { tenantId: OTHER_TENANT }; + return null; +} + +test('Provider-backed integration proxy materializes tenant-owned test artifacts from the daemon route', async (t) => { if (await skipWhenLoopbackUnavailable(t, 'local proxy artifact tenant coverage')) return; - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-proxy-artifact-tenant-')); - const remoteScreenshotPath = path.join(tempDir, 'remote-shot.png'); - const localScreenshotPath = path.join(tempDir, 'local-shot.png'); - const rejectedScreenshotPath = path.join(tempDir, 'rejected-shot.png'); - fs.writeFileSync(remoteScreenshotPath, 'tenant-scoped-png'); - const artifactIds: string[] = []; - const upstream = await createDaemonHttpServer({ - token: 'upstream-token', - handleRequest: async (req) => { - assert.equal(req.command, 'screenshot'); - assert.equal(req.meta?.tenantId, TENANT); - assert.equal(req.meta?.runId, 'local-proxy-run'); - assert.equal(req.meta?.sessionIsolation, 'tenant'); - return finalizeDaemonResponse( - req, - { ok: true, data: { path: remoteScreenshotPath } }, - (artifact) => { - const artifactId = trackDownloadableArtifact(artifact); - artifactIds.push(artifactId); - return artifactId; - }, - ); - }, - }); - const protectedArtifactId = trackDownloadableArtifact({ - artifactPath: remoteScreenshotPath, - tenantId: TENANT, - artifactType: 'screenshot', - fileName: 'remote-shot.png', - }); - artifactIds.push(protectedArtifactId); - const proxy = createDaemonProxyServer({ - upstreamBaseUrl: `http://127.0.0.1:${await listenOnLoopback(upstream)}`, - upstreamToken: 'upstream-token', - clientToken: 'proxy-token', - }); + await withProviderScenarioResource( + async () => await createFakeProviderWorld(), + async ({ daemon }) => { + await withProviderScenarioTempDir('agent-device-proxy-artifact-tenant-', async (tempDir) => { + const sourceDir = path.join(tempDir, 'source'); + const daemonArtifactsRoot = path.join(tempDir, 'daemon-side'); + const clientRoot = path.join(tempDir, 'caller-side', 'artifacts'); + const scriptPath = path.join(sourceDir, 'flow.ad'); + const rejectedArchivePath = path.join(tempDir, 'rejected.tar.gz'); + fs.mkdirSync(sourceDir, { recursive: true }); + fs.writeFileSync(scriptPath, ''); + const proxy = await createProviderScenarioProxy(daemon, { + authenticate: tenantAuthentication, + }); + try { + const allocation = await proxy.requestRpc(TENANT_TOKEN, { + jsonrpc: '2.0', + id: 'tenant-artifact-lease', + method: 'agent_device.lease.allocate', + params: { + tenantId: TENANT, + runId: 'artifact-run', + backend: 'android-instance', + leaseProvider: FAKE_PROVIDER, + platform: 'android', + deviceKey: 'android-a', + clientId: 'tenant-artifact-client', + }, + }); + assert.equal(allocation.statusCode, 200, JSON.stringify(allocation.json)); + const leaseId = allocation.json.result?.data?.lease?.leaseId; + assert.equal(typeof leaseId, 'string', JSON.stringify(allocation.json)); + const leaseScope = { + leaseId, + leaseBackend: 'android-instance', + leaseProvider: FAKE_PROVIDER, + deviceKey: 'android-a', + clientId: 'tenant-artifact-client', + } as const; + const request: DaemonWireRequest = { + token: TENANT_TOKEN, + session: 'default', + command: 'test', + positionals: [scriptPath], + flags: { artifactsDir: daemonArtifactsRoot }, + meta: { + cwd: sourceDir, + tenantId: TENANT, + runId: 'artifact-run', + requestId: 'tenant-artifact-run', + ...leaseScope, + clientArtifactPaths: { artifactsDir: clientRoot }, + }, + }; - try { - const proxyPort = await listenOnLoopback(proxy); - const daemonBaseUrl = `http://127.0.0.1:${proxyPort}/agent-device`; - - await assert.rejects( - async () => - await downloadRemoteArtifact({ - baseUrl: daemonBaseUrl, - token: 'proxy-token', - artifactId: protectedArtifactId, - destinationPath: rejectedScreenshotPath, - requestScope: { tenantId: OTHER_TENANT }, - }), - (error: unknown) => { - const normalized = normalizeAgentDeviceError(error); - assert.equal(normalized.details?.statusCode, 401); - assert.match(String(normalized.details?.body), /different tenant/i); - return true; - }, - ); - assert.equal(fs.existsSync(rejectedScreenshotPath), false); - - const client = createAgentDeviceClient({ - daemonBaseUrl, - daemonAuthToken: 'proxy-token', - tenant: TENANT, - runId: 'local-proxy-run', - sessionIsolation: 'tenant', - stateDir: tempDir, - }); - const screenshot = await client.capture.screenshot({ path: localScreenshotPath }); - - assert.equal(screenshot.path, localScreenshotPath); - assert.equal(fs.readFileSync(localScreenshotPath, 'utf8'), 'tenant-scoped-png'); - } finally { - for (const artifactId of artifactIds) cleanupDownloadableArtifact(artifactId); - await closeLoopbackServer(proxy); - await closeLoopbackServer(upstream); - fs.rmSync(tempDir, { recursive: true, force: true }); - } + const rpc = await proxy.callCommand( + TENANT_TOKEN, + request.command, + request.positionals, + request.flags, + { meta: request.meta, session: request.session }, + ); + assert.equal(rpc.statusCode, 200, JSON.stringify(rpc.json)); + const data = rpc.json.result?.data as + | { + total?: number; + passed?: number; + failed?: number; + artifacts?: Array<{ artifactId?: string; artifactType?: string }>; + } + | undefined; + assert.equal(data?.total, 1, JSON.stringify(rpc.json)); + assert.equal(data?.passed, 1, JSON.stringify(rpc.json)); + assert.equal(data?.failed, 0, JSON.stringify(rpc.json)); + const artifact = data?.artifacts?.[0]; + assert.equal(artifact?.artifactType, 'test-artifacts'); + assert.equal(typeof artifact?.artifactId, 'string'); + const artifactId = artifact?.artifactId as string; + + const testRequest = proxy.forwarded.find((entry) => { + if (entry.method !== 'POST' || entry.route !== '/rpc') return false; + const payload = JSON.parse(entry.requestBody) as { + params?: { command?: string }; + }; + return payload.params?.command === 'test'; + }); + assert.ok(testRequest, 'the test command did not reach the daemon'); + const forwardedParams = JSON.parse(testRequest.requestBody) as { + params: { command?: string; tenantId?: string; meta?: Record }; + }; + assert.equal(forwardedParams.params.command, 'test'); + assert.equal(forwardedParams.params.tenantId, TENANT); + assert.equal(forwardedParams.params.meta?.tenantId, TENANT); + assert.equal(testRequest.authorization, `Bearer ${daemon.token}`); + assert.equal(testRequest.tenantId, TENANT); + + await assert.rejects( + async () => + await downloadRemoteArtifact({ + baseUrl: proxy.baseUrl, + token: OTHER_TENANT_TOKEN, + artifactId, + destinationPath: rejectedArchivePath, + requestScope: { tenantId: OTHER_TENANT }, + isDirectory: true, + }), + (error: unknown) => { + const normalized = normalizeAgentDeviceError(error); + assert.equal(normalized.details?.statusCode, 401); + return true; + }, + ); + assert.deepEqual(fs.readdirSync(rejectedArchivePath), []); + + const response: Extract = { + ok: true, + data: data as NonNullable['data']>, + }; + const materialized = await materializeRemoteArtifacts( + { baseUrl: proxy.baseUrl, token: TENANT_TOKEN }, + request, + response, + ); + assert.equal(materialized.ok, true); + if (!materialized.ok) return; + const materializedSuiteDir = materialized.data?.artifactsDir; + if (typeof materializedSuiteDir !== 'string') { + throw new Error('the artifact route did not materialize a suite directory'); + } + assert.equal(path.dirname(materializedSuiteDir), clientRoot); + assert.ok(fs.existsSync(materializedSuiteDir)); + const attemptDir = path.join(materializedSuiteDir, 'flow.ad', 'attempt-1'); + assert.equal(fs.readFileSync(path.join(attemptDir, 'replay.ad'), 'utf8'), ''); + assert.match( + fs.readFileSync(path.join(attemptDir, 'result.txt'), 'utf8'), + /status: passed/, + ); + + const downloads = proxy.forwarded.filter( + (entry) => entry.method === 'GET' && entry.route === `/artifacts/${artifactId}`, + ); + assert.deepEqual( + downloads.map((entry) => [entry.authorization, entry.tenantId, entry.statusCode]), + [ + [`Bearer ${daemon.token}`, OTHER_TENANT, 401], + [`Bearer ${daemon.token}`, TENANT, 200], + ], + ); + } finally { + await proxy.close(); + } + }); + }, + ); }); -test('Provider-backed integration authenticated proxy sends every admitted RPC method as the caller tenant', async (t) => { +test('Provider-backed integration authenticated proxy stamps every admitted RPC method before the daemon handles it', async (t) => { if (await skipWhenLoopbackUnavailable(t, 'authenticated proxy rpc tenant coverage')) return; - const observed: DaemonWireRequest[] = []; - const upstream = await createDaemonHttpServer({ - token: 'upstream-token', - handleRequest: async (req): Promise => { - observed.push(req); - return { ok: true, data: {} }; - }, - }); - const proxy = createDaemonProxyServer({ - upstreamBaseUrl: `http://127.0.0.1:${await listenOnLoopback(upstream)}`, - upstreamToken: 'upstream-token', - authenticate: (request) => - request.headers.get('authorization') === 'Bearer alice-key' ? { tenantId: 'alice' } : null, - }); - const otherTenantParams = { session: 'bob:default', tenantId: 'bob', runId: 'run-1' }; - const admitted: Array<[string, Record]> = [ - ['agent_device.command', { command: 'devices', meta: { tenantId: 'bob' } }], - ['agent_device.lease.allocate', { backend: 'ios-simulator' }], - ['agent_device.lease.heartbeat', { leaseId: 'lease-1' }], - ['agent_device.lease.release', { leaseId: 'lease-1' }], - ]; - const refused: Array<[string, Record]> = [ - [ - 'agent_device.install_from_source', - { platform: 'ios', source: { kind: 'url', url: 'https://example.test/app.ipa' } }, - ], - ['agent_device.release_materialized_paths', { materializationId: 'materialization-1' }], - ]; - - try { - const rpcUrl = `http://127.0.0.1:${await listenOnLoopback(proxy)}/agent-device/rpc`; - const call = async (method: string, params: Record) => - await fetch(rpcUrl, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: 'Bearer alice-key' }, - body: JSON.stringify({ - jsonrpc: '2.0', - id: method, - method, - params: { ...otherTenantParams, ...params }, - }), + await withProviderScenarioResource(createFakeProviderWorld, async ({ daemon }) => { + const proxy = await createProviderScenarioProxy(daemon, { + authenticate: (request) => + request.headers.get('authorization') === 'Bearer alice-key' ? { tenantId: 'alice' } : null, + }); + const nextTenantParams = { session: 'default', tenantId: 'bob', runId: 'run-1' }; + const leaseOwnerScope = { + leaseBackend: 'android-instance', + leaseProvider: FAKE_PROVIDER, + deviceKey: 'android-a', + clientId: 'client-a', + }; + const rpc = async (id: string, method: string, params: Record) => + await proxy.requestRpc('alice-key', { + jsonrpc: '2.0', + id, + method, + params: { ...nextTenantParams, ...params }, }); - for (const [method, params] of admitted) { - const response = await call(method, params); - assert.equal(response.status, 200, method); - assert.equal(observed.at(-1)?.meta?.tenantId, 'alice', method); - } - assert.equal(observed[0]?.meta?.sessionIsolation, 'tenant'); + try { + const commandId = 'tenant-command'; + const command = await rpc(commandId, 'agent_device.command', { + command: 'devices', + positionals: [], + flags: { tenant: 'bob', sessionIsolation: 'none' }, + meta: { tenantId: 'bob', sessionIsolation: 'none', requestId: commandId }, + }); + assert.equal(command.statusCode, 200, JSON.stringify(command.json)); + assert.equal(command.json.result?.ok, true, JSON.stringify(command.json)); + + const allocationId = 'tenant-lease-allocate'; + const allocation = await rpc(allocationId, 'agent_device.lease.allocate', { + backend: 'android-instance', + leaseProvider: FAKE_PROVIDER, + platform: 'android', + deviceKey: 'android-a', + clientId: 'client-a', + }); + assert.equal(allocation.statusCode, 200, JSON.stringify(allocation.json)); + assert.equal(allocation.json.result?.ok, true, JSON.stringify(allocation.json)); + const leaseId = allocation.json.result?.data?.lease?.leaseId; + assert.equal(typeof leaseId, 'string', JSON.stringify(allocation.json)); - for (const [method, params] of refused) { - const response = await call(method, params); - assert.equal(response.status, 401, method); + const heartbeatId = 'tenant-lease-heartbeat'; + const heartbeat = await rpc(heartbeatId, 'agent_device.lease.heartbeat', { + leaseId, + ...leaseOwnerScope, + }); + assert.equal(heartbeat.statusCode, 200, JSON.stringify(heartbeat.json)); + assert.equal(heartbeat.json.result?.ok, true, JSON.stringify(heartbeat.json)); + + const releaseId = 'tenant-lease-release'; + const release = await rpc(releaseId, 'agent_device.lease.release', { + leaseId, + ...leaseOwnerScope, + }); + assert.equal(release.statusCode, 200, JSON.stringify(release.json)); + assert.equal(release.json.result?.ok, true, JSON.stringify(release.json)); + + for (const [id, method] of [ + [commandId, 'agent_device.command'], + [allocationId, 'agent_device.lease.allocate'], + [heartbeatId, 'agent_device.lease.heartbeat'], + [releaseId, 'agent_device.lease.release'], + ]) { + const forwarded = proxy.forwarded.find((entry) => { + if (entry.method !== 'POST' || entry.route !== '/rpc') return false; + return (JSON.parse(entry.requestBody) as { id?: unknown }).id === id; + }); + assert.ok(forwarded, `${method} was not forwarded to the daemon`); + const payload = JSON.parse(forwarded.requestBody) as { + method?: string; + params?: Record; + }; + assert.equal(payload.method, method); + assert.equal(payload.params?.tenantId, 'alice', method); + assert.equal(payload.params?.token, daemon.token, method); + assert.equal(forwarded.authorization, `Bearer ${daemon.token}`, method); + assert.equal(forwarded.tenantId, 'alice', method); + if (method === 'agent_device.command') { + assert.equal((payload.params?.meta as Record)?.tenantId, 'alice'); + assert.equal( + (payload.params?.meta as Record)?.sessionIsolation, + 'tenant', + ); + } + } + + const forwardedCount = proxy.forwarded.length; + const refused = await Promise.all([ + rpc('tenant-install-refusal', 'agent_device.install_from_source', { + platform: 'ios', + source: { kind: 'url', url: 'https://example.test/app.ipa' }, + }), + rpc('tenant-release-materialized-refusal', 'agent_device.release_materialized_paths', { + materializationId: 'materialization-1', + }), + ]); + assert.deepEqual( + refused.map((response) => response.statusCode), + [401, 401], + ); + assert.equal(proxy.forwarded.length, forwardedCount); + } finally { + await proxy.close(); } - assert.equal(observed.length, admitted.length); - } finally { - await closeLoopbackServer(proxy); - await closeLoopbackServer(upstream); - } + }); }); diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index 52db7d4b30..30304ca34c 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -1,24 +1,19 @@ -import { resolveSessionRequestLogPath } from '@agent-device/daemon-contracts/session-artifact-paths'; import type { DaemonWireRequest } from '@agent-device/contracts/command'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; -import http from 'node:http'; import os from 'node:os'; import path from 'node:path'; -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import { sendToDaemon } from '../../../src/daemon-client/daemon-client.ts'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { createDaemonProxyServer } from '@agent-device/proxy'; import { AppError, type DaemonError } from '@agent-device/kernel/errors'; -import { - closeLoopbackServer, - listenOnLoopback, - skipWhenLoopbackUnavailable, -} from '@agent-device/test-kit'; +import { skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; import { createProviderScenarioHarness, type ProviderScenarioHarness } from './harness.ts'; +import { + createProviderScenarioProxy, + type ProviderScenarioForwardedRequest, +} from './harness-proxy.ts'; import { createAppleRunnerProviderFromTranscript, createRecordingAppleToolProvider, @@ -44,6 +39,7 @@ const VOLATILE_KEYS = new Set([ 'requestId', 'diagnosticId', 'logPath', + 'logPathUnavailable', 'timing', 'durationMs', 'elapsedMs', @@ -68,6 +64,10 @@ type ParityWorld = { close: () => Promise; }; +function loopbackProxyBaseUrl(port: number): string { + return `http://127.0.0.1:${port}/agent-device`; +} + function scriptedTree() { return { nodes: [ @@ -94,7 +94,7 @@ function scriptedTree() { }; } -type ParityWorldOptions = { leaseRegistry?: LeaseRegistry }; +type ParityWorldOptions = { leaseTtlMs?: number }; async function createParityWorld(options: ParityWorldOptions = {}): Promise { const runnerTranscript = createProviderTranscript([ @@ -112,8 +112,7 @@ async function createParityWorld(options: ParityWorldOptions = {}): Promise createAppleRunnerProviderFromTranscript(runnerTranscript, 'ios.runner'), appleToolProvider: () => appleTool.provider, @@ -228,81 +227,84 @@ async function runLeg(leg: ScenarioLeg, legName: string): Promise { + const response = await daemon.callCommand(request.command, request.positionals, request.flags, { + input: request.input, + meta: request.meta, + runtime: request.runtime, + session: request.session, + }); + const result = response.json?.result; + if (result) return { ok: true, data: result.data }; + const error = response.json?.error?.data; + if (error) return { ok: false, error: error as DaemonError }; + throw new Error(`daemon returned no response for ${request.command}`); +} + async function withProxiedWorld( - run: (context: { world: ParityWorld; proxied: ScenarioLeg; upstream: http.Server }) => Promise, + run: (context: { + world: ParityWorld; + proxied: ScenarioLeg; + forwarded: ProviderScenarioForwardedRequest[]; + }) => Promise, options: ParityWorldOptions = {}, + dependencies: { + createWorld?: typeof createParityWorld; + createProxy?: typeof createProviderScenarioProxy; + } = {}, ): Promise { - const world = await createParityWorld(options); - const upstream = await createDaemonHttpServer({ - token: world.daemon.token, - handleRequest: world.daemon.handleRequest, - // Same composition as the daemon runtime: remote clients localize failure records. - resolveRequestDiagnosticsPath: (ref) => - resolveSessionRequestLogPath(world.daemon.sessionDir(ref.session), ref.requestId), - }); - const proxy = createDaemonProxyServer({ - upstreamBaseUrl: `http://127.0.0.1:${await listenOnLoopback(upstream)}`, - upstreamToken: world.daemon.token, - clientToken: PROXY_TOKEN, - }); - // The remote client's own state dir: localized failure records land here, not in the - // developer's real state dir. - const clientStateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-proxy-parity-')); + let world: ParityWorld | undefined; + let proxy: Awaited> | undefined; + let clientStateDir: string | undefined; try { - const daemonBaseUrl = `http://127.0.0.1:${await listenOnLoopback(proxy)}/agent-device`; + const activeWorld = await (dependencies.createWorld ?? createParityWorld)(options); + world = activeWorld; + const activeProxy = await (dependencies.createProxy ?? createProviderScenarioProxy)( + activeWorld.daemon, + { + clientToken: PROXY_TOKEN, + }, + ); + proxy = activeProxy; + const proxyBaseUrl = loopbackProxyBaseUrl(activeProxy.port); + const activeClientStateDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'agent-device-proxy-parity-'), + ); + clientStateDir = activeClientStateDir; const proxied: ScenarioLeg = async (request) => { try { return await sendToDaemon( - { ...request, flags: { ...request.flags, daemonBaseUrl, stateDir: clientStateDir } }, + { + ...request, + flags: { + ...request.flags, + daemonBaseUrl: proxyBaseUrl, + stateDir: activeClientStateDir, + }, + }, { authToken: PROXY_TOKEN }, ); } catch (error) { return responseFromClientError(error); } }; - return await run({ world, proxied, upstream }); + return await run({ world: activeWorld, proxied, forwarded: activeProxy.forwarded }); } finally { - await closeLoopbackServer(proxy); - await closeLoopbackServer(upstream); - await world.close(); - fs.rmSync(clientStateDir, { recursive: true, force: true }); + try { + await proxy?.close(); + } finally { + try { + await world?.close(); + } finally { + if (clientStateDir) fs.rmSync(clientStateDir, { recursive: true, force: true }); + } + } } } -/** - * Every request the proxy forwards upstream, in order. Registering a second `request` - * listener records without displacing the server's own handler, so the daemon behaves - * exactly as it does untraced. - */ -function recordUpstreamRequests(upstream: http.Server): { forwarded: ForwardedRequest[] } { - const forwarded: ForwardedRequest[] = []; - upstream.on('request', (req, res) => { - const route = (req.url ?? '').split('?', 1)[0] ?? ''; - const entry: ForwardedRequest = { method: req.method ?? '', route, body: '' }; - const end = res.end.bind(res); - const write = res.write.bind(res); - res.write = ((chunk: unknown, ...rest: unknown[]) => { - entry.body += textOf(chunk); - return (write as (...args: unknown[]) => boolean)(chunk, ...rest); - }) as typeof res.write; - res.end = ((chunk?: unknown, ...rest: unknown[]) => { - entry.body += textOf(chunk); - return (end as (...args: unknown[]) => http.ServerResponse)(chunk, ...rest); - }) as typeof res.end; - forwarded.push(entry); - }); - return { forwarded }; -} - -/** One request the proxy forwarded, with the exact bytes the daemon answered it with. */ -type ForwardedRequest = { method: string; route: string; body: string }; - -function textOf(chunk: unknown): string { - if (typeof chunk === 'string') return chunk; - if (chunk instanceof Uint8Array) return Buffer.from(chunk).toString('utf8'); - return ''; -} - /** * What a published `snapshot --interactive-only` response is made of. #2198 forbids the * internal acquired tree, lineage, target-generation facts and runner quality payload from @@ -341,13 +343,18 @@ function assertPublishedParity(direct: LegRun, proxied: LegRun): void { }); } -/** - * The remote client may add to the envelope (it materializes the failure record locally, - * which yields a logPath the daemon never published); it may not lose anything. - */ +/** The proxy may report an unavailable remote log path where direct execution has a local path. */ function assertErrorEnvelopeKept(direct: DaemonError, proxied: DaemonError, step: string): void { for (const key of PRESERVED_ERROR_KEYS) { if (direct[key] === undefined) continue; + if (key === 'logPath' && proxied.logPathUnavailable !== undefined) { + assert.equal( + typeof proxied.logPathUnavailable, + 'string', + `missing remote path reason at ${step}`, + ); + continue; + } assert.equal( typeof proxied[key], typeof direct[key], @@ -386,8 +393,7 @@ test( try { direct = { responses: await runLeg( - async (request) => - await directWorld.daemon.handleRequest({ ...request, token: directWorld.daemon.token }), + async (request) => await callDaemonCommand(directWorld.daemon, request), 'direct', ), sessionRoot: sessionRootOf(directWorld.daemon), @@ -396,10 +402,21 @@ test( await directWorld.close(); } - const proxied = await withProxiedWorld(async ({ world, proxied: leg }) => ({ - responses: await runLeg(leg, 'proxied'), - sessionRoot: sessionRootOf(world.daemon), - })); + const proxied = await withProxiedWorld(async ({ world, proxied: leg, forwarded }) => { + const responses = await runLeg(leg, 'proxied'); + const missingRef = responses[4]; + assert.equal(missingRef?.ok, false); + if (!missingRef || missingRef.ok) throw new Error('expected the missing-ref route to fail'); + assert.equal(typeof missingRef.error.logPath, 'string'); + assert.equal(missingRef.error.logPathUnavailable, undefined); + assert.equal( + forwarded.filter( + (entry) => entry.method === 'GET' && /\/requests\/[^/]+\/diagnostics$/.test(entry.route), + ).length, + 1, + ); + return { responses, sessionRoot: sessionRootOf(world.daemon) }; + }); assertPublishedParity(direct, proxied); assertScriptOutcomes(direct.responses); @@ -407,6 +424,80 @@ test( PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, ); +test( + 'closes the daemon when proxy startup fails', + async (t) => { + if (await skipWhenLoopbackUnavailable(t, 'daemon proxy startup cleanup coverage')) return; + + let closeCalls = 0; + const startupError = new Error('simulated proxy listen failure'); + await assert.rejects( + withProxiedWorld( + async () => { + throw new Error('the scenario callback must not run after proxy startup fails'); + }, + {}, + { + createWorld: async (options) => { + const world = await createParityWorld(options); + return { + ...world, + close: async () => { + closeCalls += 1; + await world.close(); + }, + }; + }, + createProxy: async () => { + throw startupError; + }, + }, + ), + (error) => error === startupError, + ); + assert.equal(closeCalls, 1); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + 'recording proxy retains upstream errors without exposing them in the response', + async (t) => { + if (await skipWhenLoopbackUnavailable(t, 'daemon proxy error handling coverage')) return; + + let world: ParityWorld | undefined; + let proxy: Awaited> | undefined; + try { + const activeWorld = await createParityWorld(); + world = activeWorld; + const activeProxy = await createProviderScenarioProxy(activeWorld.daemon, { + clientToken: PROXY_TOKEN, + }); + proxy = activeProxy; + await activeWorld.close(); + const response = await fetch(`${loopbackProxyBaseUrl(activeProxy.port)}/rpc`, { + method: 'POST', + headers: { + authorization: `Bearer ${PROXY_TOKEN}`, + 'content-type': 'application/json', + }, + body: '{}', + }); + assert.equal(response.status, 502); + assert.equal(await response.text(), 'upstream request failed'); + + assert.equal(activeProxy.forwarded[0]?.upstreamErrorCode, 'ECONNREFUSED'); + } finally { + try { + await proxy?.close(); + } finally { + await world?.close(); + } + } + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + test( 'Provider-backed integration proxy clients contending for one device fail at lease admission', async (t) => { @@ -475,12 +566,12 @@ test( }, async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; - await withProxiedWorld(async ({ world, proxied, upstream }) => { + await withProxiedWorld(async ({ world, proxied, forwarded }) => { const flags = { platform: 'ios', udid: SIM.id } as const; await proxied({ session: 'default', command: 'open', positionals: [APP], flags }); // Record only the snapshot: the open above is setup and makes its own calls. - const { forwarded } = recordUpstreamRequests(upstream); + const forwardedBeforeSnapshot = forwarded.length; const response = await proxied({ session: 'default', command: 'snapshot', @@ -491,8 +582,9 @@ test( // The open above established the daemon identity. A persistent client should now // forward only the snapshot RPC, with no additional health or helper request. + const snapshotRequests = forwarded.slice(forwardedBeforeSnapshot); assert.deepEqual( - forwarded.map((entry) => `${entry.method} ${entry.route}`), + snapshotRequests.map((entry) => `${entry.method} ${entry.route}`), ['POST /rpc'], 'a proxied snapshot must cross the wire as one RPC', ); @@ -501,7 +593,7 @@ test( // wire's `result` against the client's response would prove nothing — the client // publishes whatever `result` holds, so both sides move together — so each level is // pinned against its declared key set instead. - const rpcBody = JSON.parse(forwarded.at(-1)?.body ?? '{}') as { + const rpcBody = JSON.parse(snapshotRequests.at(-1)?.responseBody ?? '{}') as { result?: { data?: Record }; }; assert.deepEqual( @@ -543,84 +635,90 @@ test( if (await skipWhenLoopbackUnavailable(t, 'daemon proxy parity coverage')) return; let now = 1_000_000; - const leaseRegistry = new LeaseRegistry({ now: () => now }); - await withProxiedWorld( - async ({ world, proxied }) => { - const session = 'leased'; - const flags = { platform: 'ios', udid: SIM.id } as const; - const allocate = async (): Promise<{ leaseId: string; expiresAt: number }> => { - const response = await proxied({ - session, - command: 'lease_allocate', - positionals: [], - flags: {}, - meta: LEASE_SCOPE, - }); - assert.equal(response.ok, true, JSON.stringify(response)); - const lease = (response.ok ? response.data : {})?.lease as { - leaseId?: string; - expiresAt?: number; + const dateSpy = vi.spyOn(Date, 'now').mockImplementation(() => now); + try { + await withProxiedWorld( + async ({ proxied }) => { + const session = 'leased'; + const flags = { platform: 'ios', udid: SIM.id } as const; + const allocate = async (): Promise<{ leaseId: string; expiresAt: number }> => { + const response = await proxied({ + session, + command: 'lease_allocate', + positionals: [], + flags: {}, + meta: LEASE_SCOPE, + }); + assert.equal(response.ok, true, JSON.stringify(response)); + const lease = (response.ok ? response.data : {})?.lease as { + leaseId?: string; + expiresAt?: number; + }; + assert.equal(typeof lease?.leaseId, 'string'); + assert.equal(typeof lease?.expiresAt, 'number'); + return { leaseId: lease.leaseId!, expiresAt: lease.expiresAt! }; }; - assert.equal(typeof lease?.leaseId, 'string'); - assert.equal(typeof lease?.expiresAt, 'number'); - return { leaseId: lease.leaseId!, expiresAt: lease.expiresAt! }; - }; - const run = async ( - command: string, - positionals: string[], - leaseId: string, - stepFlags: DaemonWireRequest['flags'] = flags, - ) => - await proxied({ - session, - command, - positionals, - flags: stepFlags, - meta: { ...LEASE_SCOPE, leaseId }, + const run = async ( + command: string, + positionals: string[], + leaseId: string, + stepFlags: DaemonWireRequest['flags'] = flags, + ) => + await proxied({ + session, + command, + positionals, + flags: stepFlags, + meta: { ...LEASE_SCOPE, leaseId }, + }); + + const firstLease = await allocate(); + assert.equal((await run('open', [APP], firstLease.leaseId)).ok, true); + assert.equal( + (await run('snapshot', [], firstLease.leaseId, { snapshotInteractiveOnly: true })).ok, + true, + ); + assert.equal( + baselineInitialized( + await run('diff', ['snapshot'], firstLease.leaseId, { + snapshotInteractiveOnly: true, + }), + ), + false, + 'the leased session holds comparison state before it expires', + ); + + // The lease lapses without a heartbeat; the next request through the proxy finds it expired. + // The jump is the window the lease itself was allocated with — a client that names no ttl + // gets the registry default, which is the only window the daemon promises it. + now = firstLease.expiresAt + 1; + const expired = await run('diff', ['snapshot'], firstLease.leaseId, { + snapshotInteractiveOnly: true, }); - - const firstLease = await allocate(); - assert.equal((await run('open', [APP], firstLease.leaseId)).ok, true); - assert.equal( - (await run('snapshot', [], firstLease.leaseId, { snapshotInteractiveOnly: true })).ok, - true, - ); - assert.equal( - baselineInitialized( - await run('diff', ['snapshot'], firstLease.leaseId, { snapshotInteractiveOnly: true }), - ), - false, - 'the leased session holds comparison state before it expires', - ); - - // The lease lapses without a heartbeat; the next request through the proxy finds it expired. - // The jump is the window the lease itself was allocated with — a client that names no ttl - // gets the registry default, which is the only window the daemon promises it. - now = firstLease.expiresAt + 1; - const expired = await run('diff', ['snapshot'], firstLease.leaseId, { - snapshotInteractiveOnly: true, - }); - assert.equal(expired.ok, false); - if (expired.ok) return; - assert.equal(expired.error.code, 'UNAUTHORIZED'); - assert.equal(expired.error.details?.reason, 'LEASE_NOT_FOUND'); - assert.equal(typeof expired.error.hint, 'string'); - assert.equal(world.daemon.session(session), undefined, 'expiry tears the session down'); - - const secondLease = await allocate(); - assert.notEqual(secondLease.leaseId, firstLease.leaseId); - assert.equal((await run('open', [APP], secondLease.leaseId)).ok, true); - assert.equal( - baselineInitialized( - await run('diff', ['snapshot'], secondLease.leaseId, { snapshotInteractiveOnly: true }), - ), - true, - 'a reacquired lease must not compare against the expired session tree', - ); - assert.equal((await run('close', [], secondLease.leaseId, {})).ok, true); - }, - { leaseRegistry }, - ); + assert.equal(expired.ok, false); + if (expired.ok) return; + assert.equal(expired.error.code, 'UNAUTHORIZED'); + assert.equal(expired.error.details?.reason, 'LEASE_NOT_FOUND'); + assert.equal(typeof expired.error.hint, 'string'); + const secondLease = await allocate(); + assert.notEqual(secondLease.leaseId, firstLease.leaseId); + assert.equal((await run('open', [APP], secondLease.leaseId)).ok, true); + assert.equal( + baselineInitialized( + await run('diff', ['snapshot'], secondLease.leaseId, { + snapshotInteractiveOnly: true, + }), + ), + true, + 'a reacquired lease must not compare against the expired session tree', + ); + assert.equal((await run('close', [], secondLease.leaseId, {})).ok, true); + }, + { leaseTtlMs: 5_000 }, + ); + } finally { + dateSpy.mockRestore(); + } }, PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, ); @@ -631,52 +729,55 @@ test( if (await skipWhenLoopbackUnavailable(t, 'daemon proxy parity coverage')) return; let now = 2_000_000; - const leaseRegistry = new LeaseRegistry({ now: () => now }); - await withProxiedWorld( - async ({ world, proxied }) => { - const session = 'renewed'; - const flags = { platform: 'ios', udid: SIM.id } as const; - const allocated = await proxied({ - session, - command: 'lease_allocate', - positionals: [], - flags: {}, - meta: LEASE_SCOPE, - }); - assert.equal(allocated.ok, true, JSON.stringify(allocated)); - const lease = (allocated.ok ? allocated.data : {})?.lease as { leaseId: string }; - const meta = { ...LEASE_SCOPE, leaseId: lease.leaseId }; - const run = async ( - command: string, - positionals: string[], - stepFlags: DaemonWireRequest['flags'], - ) => await proxied({ session, command, positionals, flags: stepFlags, meta }); - - assert.equal((await run('open', [APP], flags)).ok, true); - assert.equal((await run('snapshot', [], { snapshotInteractiveOnly: true })).ok, true); - const heartbeatExpiry = async (): Promise => { - const heartbeat = await run('lease_heartbeat', [], {}); - assert.equal(heartbeat.ok, true, JSON.stringify(heartbeat)); - const renewed = (heartbeat.ok ? heartbeat.data : {})?.lease as { expiresAt: number }; - return renewed.expiresAt; - }; - - // One explicit heartbeat through the proxy just before the lease would lapse moves the - // expiry forward; a request past the old expiry but inside the new window still finds - // the session and its diff baseline. - const firstExpiry = await heartbeatExpiry(); - now = firstExpiry - 1_000; - const renewedExpiry = await heartbeatExpiry(); - assert.ok(renewedExpiry > firstExpiry, 'the heartbeat moved the expiry forward'); - now = firstExpiry + 1_000; - const diff = await run('diff', ['snapshot'], { snapshotInteractiveOnly: true }); - assert.equal(diff.ok, true, JSON.stringify(diff)); - assert.equal(baselineInitialized(diff), false, 'the renewed session kept its baseline'); - assert.notEqual(world.daemon.session(session), undefined); - assert.equal((await run('close', [], {})).ok, true); - }, - { leaseRegistry }, - ); + const dateSpy = vi.spyOn(Date, 'now').mockImplementation(() => now); + try { + await withProxiedWorld( + async ({ proxied }) => { + const session = 'renewed'; + const flags = { platform: 'ios', udid: SIM.id } as const; + const allocated = await proxied({ + session, + command: 'lease_allocate', + positionals: [], + flags: {}, + meta: LEASE_SCOPE, + }); + assert.equal(allocated.ok, true, JSON.stringify(allocated)); + const lease = (allocated.ok ? allocated.data : {})?.lease as { leaseId: string }; + const meta = { ...LEASE_SCOPE, leaseId: lease.leaseId }; + const run = async ( + command: string, + positionals: string[], + stepFlags: DaemonWireRequest['flags'], + ) => await proxied({ session, command, positionals, flags: stepFlags, meta }); + + assert.equal((await run('open', [APP], flags)).ok, true); + assert.equal((await run('snapshot', [], { snapshotInteractiveOnly: true })).ok, true); + const heartbeatExpiry = async (): Promise => { + const heartbeat = await run('lease_heartbeat', [], {}); + assert.equal(heartbeat.ok, true, JSON.stringify(heartbeat)); + const renewed = (heartbeat.ok ? heartbeat.data : {})?.lease as { expiresAt: number }; + return renewed.expiresAt; + }; + + // One explicit heartbeat through the proxy just before the lease would lapse moves the + // expiry forward; a request past the old expiry but inside the new window still finds + // the session and its diff baseline. + const firstExpiry = await heartbeatExpiry(); + now = firstExpiry - 1_000; + const renewedExpiry = await heartbeatExpiry(); + assert.ok(renewedExpiry > firstExpiry, 'the heartbeat moved the expiry forward'); + now = firstExpiry + 1_000; + const diff = await run('diff', ['snapshot'], { snapshotInteractiveOnly: true }); + assert.equal(diff.ok, true, JSON.stringify(diff)); + assert.equal(baselineInitialized(diff), false, 'the renewed session kept its baseline'); + assert.equal((await run('close', [], {})).ok, true); + }, + { leaseTtlMs: 5_000 }, + ); + } finally { + dateSpy.mockRestore(); + } }, PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, ); diff --git a/test/integration/provider-scenarios/remote-proxy-request-diagnostics.test.ts b/test/integration/provider-scenarios/remote-proxy-request-diagnostics.test.ts index 04d6bc165f..51eb05e329 100644 --- a/test/integration/provider-scenarios/remote-proxy-request-diagnostics.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-request-diagnostics.test.ts @@ -1,100 +1,136 @@ -/** - * A client behind `agent-device proxy` must get the SAME failure envelope a - * client on the daemon host gets (#1801), including for the plain sessions - * `scopeRequestSession` leaves unscoped. - * - * The proxy forwards `x-agent-device-tenant` verbatim, so a caller that carries - * a tenant identity reaches the diagnostics route with that identity even - * though its command ran in a plain session such as `cwd::default`. The - * record read must not be stricter than the write that produced it: on a daemon - * with no auth hook that same caller can already run any command in any session - * over `/rpc`. - */ - -import { resolveSessionRequestLogPath } from '@agent-device/daemon-contracts/session-artifact-paths'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; +import http from 'node:http'; import path from 'node:path'; -import { test } from 'vitest'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { createDaemonProxyServer } from '@agent-device/proxy'; +import { test, vi } from 'vitest'; import { localizeRemoteDaemonError } from '@agent-device/remote/remote-request-diagnostics'; -import { safeSessionName } from '@agent-device/host-kit/session-paths'; import type { DaemonError } from '@agent-device/kernel/errors'; -import { - closeLoopbackServer, - listenOnLoopback, - mkdtempForTestSync, - skipWhenLoopbackUnavailable, -} from '@agent-device/test-kit'; +import { mkdtempForTestSync, skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; +import { createProviderScenarioHarness } from './harness.ts'; +import { createProviderScenarioProxy } from './harness-proxy.ts'; -const UPSTREAM_TOKEN = 'upstream-token'; const PROXY_TOKEN = 'proxy-token'; -const TENANT = 'local-proxy-tenant'; -/** What `resolveEffectiveSessionName` names an implicit session outside tenant isolation. */ +/** An explicit plain session keeps colons in the diagnostics route segment. */ const PLAIN_SESSION = 'cwd:9f1:default'; const REQUEST_ID = 'abc123'; -const RECORD = '{"phase":"request_start"}\n{"phase":"request_failed"}\n'; - -function remoteError(): DaemonError { - return { - code: 'COMMAND_FAILED', - message: 'wait timed out', - logPath: '/Users/daemon-host/.agent-device/sessions/cwd_9f1_default/requests/abc123.ndjson', - diagnosticsRecord: { session: PLAIN_SESSION, requestId: REQUEST_ID }, - }; -} -test('Provider-backed integration local proxy serves a plain-session diagnostics record', async (t) => { +test('Provider-backed integration local proxy serves the daemon diagnostics record for the encoded plain session', async (t) => { if (await skipWhenLoopbackUnavailable(t, 'local proxy request diagnostics coverage')) return; + const daemon = await createProviderScenarioHarness({ deviceInventoryProvider: async () => [] }); + const proxy = await createProviderScenarioProxy(daemon, { clientToken: PROXY_TOKEN }); const stateDir = mkdtempForTestSync('agent-device-proxy-request-diagnostics-'); - const sessionsDir = path.join(stateDir, 'sessions'); - const recordPath = resolveSessionRequestLogPath( - path.join(sessionsDir, safeSessionName(PLAIN_SESSION)), - REQUEST_ID, - ); - fs.mkdirSync(path.dirname(recordPath), { recursive: true }); - fs.writeFileSync(recordPath, RECORD); - - const upstream = await createDaemonHttpServer({ - token: UPSTREAM_TOKEN, - handleRequest: async (): Promise => ({ ok: true, data: {} }), - resolveRequestDiagnosticsPath: (ref) => - resolveSessionRequestLogPath( - path.join(sessionsDir, safeSessionName(ref.session)), - ref.requestId, - ), - }); - const upstreamBaseUrl = `http://127.0.0.1:${await listenOnLoopback(upstream)}`; - const proxy = createDaemonProxyServer({ - upstreamBaseUrl, - upstreamToken: UPSTREAM_TOKEN, - clientToken: PROXY_TOKEN, - }); try { - const proxyBaseUrl = `http://127.0.0.1:${await listenOnLoopback(proxy)}/agent-device`; + const failed = await proxy.callCommand( + PROXY_TOKEN, + 'trace', + ['stop'], + {}, + { + session: PLAIN_SESSION, + meta: { requestId: REQUEST_ID, sessionExplicit: true }, + }, + ); + assert.equal(failed.statusCode, 404, JSON.stringify(failed.json)); + const error = failed.json.error?.data as DaemonError | undefined; + assert.equal(error?.code, 'SESSION_NOT_FOUND', JSON.stringify(failed.json)); + assert.deepEqual(error?.diagnosticsRecord, { + session: PLAIN_SESSION, + requestId: REQUEST_ID, + }); - const throughProxy = await localizeRemoteDaemonError(remoteError(), { - endpoint: { baseUrl: proxyBaseUrl, token: PROXY_TOKEN, tenantId: TENANT }, + const throughProxy = await localizeRemoteDaemonError(error!, { + endpoint: { baseUrl: proxy.baseUrl, token: PROXY_TOKEN }, stateDir: path.join(stateDir, 'proxy-client'), }); assert.equal(throughProxy.logPathUnavailable, undefined, 'the proxy must serve the record'); assert.notEqual(throughProxy.logPath, undefined); - assert.equal(fs.readFileSync(throughProxy.logPath!, 'utf8'), RECORD); + const proxyRecord = fs.readFileSync(throughProxy.logPath!, 'utf8'); + assert.match(proxyRecord, /"phase":"request_start"/); + assert.match(proxyRecord, /"phase":"request_failed"/); - // Parity: the client on the daemon host localizes byte-identical content. - const direct = await localizeRemoteDaemonError(remoteError(), { - endpoint: { baseUrl: upstreamBaseUrl, token: UPSTREAM_TOKEN, tenantId: TENANT }, - stateDir: path.join(stateDir, 'direct-client'), - }); - assert.equal(direct.logPathUnavailable, undefined); - assert.equal(fs.readFileSync(direct.logPath!, 'utf8'), RECORD); + const expectedPath = `/sessions/${encodeURIComponent(PLAIN_SESSION)}/requests/${REQUEST_ID}/diagnostics`; + const forwardedRequests = proxy.forwarded.map((entry) => [ + entry.method, + entry.route, + entry.authorization, + entry.tenantId, + entry.statusCode, + ]); + assert.deepEqual(forwardedRequests, [ + ['POST', '/rpc', `Bearer ${daemon.token}`, undefined, 404], + ['GET', expectedPath, `Bearer ${daemon.token}`, undefined, 200], + ]); + const forwardedRpc = proxy.forwarded[0]; + assert.ok(forwardedRpc); + const forwardedRpcPayload = JSON.parse(forwardedRpc.requestBody) as { + params?: { command?: string; session?: string }; + }; + assert.equal(forwardedRpcPayload.params?.command, 'trace'); + assert.equal(forwardedRpcPayload.params?.session, PLAIN_SESSION); + const forwardedDiagnostics = proxy.forwarded.filter( + (entry) => entry.method === 'GET' && entry.route.endsWith('/diagnostics'), + ); + assert.deepEqual( + forwardedDiagnostics.map((entry) => [ + entry.route, + entry.authorization, + entry.tenantId, + entry.statusCode, + ]), + [[expectedPath, `Bearer ${daemon.token}`, undefined, 200]], + ); + + const directRequestSpy = vi.spyOn(http, 'request'); + try { + const direct = await localizeRemoteDaemonError(error!, { + endpoint: { baseUrl: daemon.baseUrl, token: daemon.token }, + stateDir: path.join(stateDir, 'direct-client'), + }); + assert.equal(direct.logPathUnavailable, undefined); + assert.equal(fs.readFileSync(direct.logPath!, 'utf8'), proxyRecord); + + const directRequest = directRequestSpy.mock.calls[0]?.[0]; + assert.ok( + directRequest && typeof directRequest === 'object' && !(directRequest instanceof URL), + ); + const directOptions = directRequest as { + protocol?: string; + host?: string; + port?: string; + method?: string; + path?: string; + headers?: Record; + }; + assert.equal(directRequestSpy.mock.calls.length, 1); + assert.deepEqual(directOptions, { + protocol: 'http:', + host: '127.0.0.1', + port: String(daemon.httpPort), + method: 'GET', + path: expectedPath, + headers: { + authorization: `Bearer ${daemon.token}`, + 'x-agent-device-token': daemon.token, + }, + }); + assert.deepEqual( + proxy.forwarded.map((entry) => [ + entry.method, + entry.route, + entry.authorization, + entry.tenantId, + entry.statusCode, + ]), + forwardedRequests, + 'direct diagnostics lookup must bypass the proxy', + ); + } finally { + directRequestSpy.mockRestore(); + } } finally { - await closeLoopbackServer(proxy); - await closeLoopbackServer(upstream); - fs.rmSync(stateDir, { recursive: true, force: true }); + await proxy.close(); + await daemon.close(); } }); diff --git a/test/integration/provider-scenarios/remote-test-artifacts-materialization.test.ts b/test/integration/provider-scenarios/remote-test-artifacts-materialization.test.ts index 420b10d655..a54d6eeed0 100644 --- a/test/integration/provider-scenarios/remote-test-artifacts-materialization.test.ts +++ b/test/integration/provider-scenarios/remote-test-artifacts-materialization.test.ts @@ -1,17 +1,10 @@ import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import http from 'node:http'; -import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; -import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; -import { - cleanupDownloadableArtifact, - trackDownloadableArtifact, -} from '@agent-device/daemon/testing/artifact-tracking'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { attachRemoteReplayTestArtifacts } from '@agent-device/replay-port/test-command'; import { downloadRemoteArtifact, materializeRemoteArtifacts, @@ -19,105 +12,141 @@ import { import { closeLoopbackServer, listenOnLoopback, + mkdtempForTestSync, skipWhenLoopbackUnavailable, } from '@agent-device/test-kit'; +import { + createProviderScenarioHarness, + withProviderScenarioResource, + withProviderScenarioTempDir, +} from './harness.ts'; +import { createProviderScenarioProxy } from './harness-proxy.ts'; -const SUITE_INVOCATION_ID = 'cd5f9c01feec8d70'; +const PROXY_TOKEN = 'remote-test-artifacts-proxy-token'; -test('the composed daemon and client path owners report the directory that was materialized', async (t) => { +test('the composed daemon and client path materializes the directory produced by a test command', async (t) => { if (await skipWhenLoopbackUnavailable(t, 'remote test-artifacts materialization coverage')) return; - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-remote-suite-response-')); - const daemonSuiteDir = path.join(tempDir, 'daemon-side', SUITE_INVOCATION_ID); - const daemonAttemptDir = path.join(daemonSuiteDir, 'flow.ad', 'attempt-1'); - const clientRoot = path.join(tempDir, 'caller-side', 'artifacts'); - const clientSuiteDir = path.join(clientRoot, SUITE_INVOCATION_ID); - fs.mkdirSync(daemonAttemptDir, { recursive: true }); - fs.writeFileSync(path.join(daemonAttemptDir, 'replay.ad'), 'open "Demo"\n'); - fs.writeFileSync(path.join(daemonAttemptDir, 'result.txt'), 'status: passed\n'); + await withProviderScenarioResource( + async () => await createProviderScenarioHarness({ deviceInventoryProvider: async () => [] }), + async (daemon) => { + await withProviderScenarioTempDir('agent-device-remote-suite-response-', async (tempDir) => { + const sourceDir = path.join(tempDir, 'source'); + const daemonArtifactsRoot = path.join(tempDir, 'daemon-side'); + const clientRoot = path.join(tempDir, 'caller-side', 'artifacts'); + const scriptPath = path.join(sourceDir, 'flow.ad'); + const source = ''; + fs.mkdirSync(sourceDir, { recursive: true }); + fs.writeFileSync(scriptPath, source); - const request: DaemonWireRequest = { - token: 'daemon-token', - session: 'default', - command: 'test', - positionals: [], - meta: { clientArtifactPaths: { artifactsDir: clientRoot } }, - }; - const suite: ReplaySuiteResult = { - total: 1, - executed: 1, - passed: 1, - failed: 0, - skipped: 0, - notRun: 0, - durationMs: 1, - failures: [], - artifactsDir: daemonSuiteDir, - tests: [ - { - file: 'flow.ad', - session: 'default:test:1', - status: 'passed', - durationMs: 1, - attempts: 1, - artifactsDir: daemonAttemptDir, - replayed: 1, - healed: 0, - }, - ], - }; - const response = { - ok: true as const, - data: attachRemoteReplayTestArtifacts(suite, request), - }; - const artifact = response.data.artifacts?.[0]; - assert.ok(artifact?.path); - const artifactId = trackDownloadableArtifact({ - artifactPath: artifact.path, - artifactType: artifact.artifactType, - fileName: artifact.fileName, - }); - response.data.artifacts = [{ ...artifact, artifactId }]; + const proxy = await createProviderScenarioProxy(daemon, { clientToken: PROXY_TOKEN }); + const request: DaemonWireRequest = { + token: PROXY_TOKEN, + session: 'default', + command: 'test', + positionals: [scriptPath], + flags: { artifactsDir: daemonArtifactsRoot }, + meta: { + cwd: sourceDir, + requestId: 'remote-test-artifact-materialization', + clientArtifactPaths: { artifactsDir: clientRoot }, + }, + }; - const server = await createDaemonHttpServer({ - token: 'daemon-token', - handleRequest: async () => { - throw new Error('not exercised: this test only hits the /artifacts route'); - }, - }); + try { + const rpc = await proxy.callCommand( + PROXY_TOKEN, + request.command, + request.positionals, + request.flags, + { meta: request.meta, session: request.session }, + ); + assert.equal(rpc.statusCode, 200, JSON.stringify(rpc.json)); + const data = rpc.json.result?.data as + | { + total?: number; + passed?: number; + failed?: number; + artifactsDir?: string; + artifacts?: Array<{ + artifactId?: string; + artifactType?: string; + }>; + } + | undefined; + assert.equal(data?.total, 1, JSON.stringify(rpc.json)); + assert.equal(data?.passed, 1, JSON.stringify(rpc.json)); + assert.equal(data?.failed, 0, JSON.stringify(rpc.json)); + const artifact = data?.artifacts?.[0]; + assert.ok(artifact, 'the test command did not register its directory artifact'); + assert.equal(artifact?.artifactType, 'test-artifacts'); + assert.equal(typeof artifact?.artifactId, 'string'); + const artifactId = artifact?.artifactId as string; - try { - const port = await listenOnLoopback(server); - const materialized = await materializeRemoteArtifacts( - { baseUrl: `http://127.0.0.1:${port}`, token: 'daemon-token' }, - request, - response, - ); + assert.equal(proxy.forwarded.length, 1); + const forwardedCommand = proxy.forwarded[0]; + assert.ok(forwardedCommand); + assert.equal(forwardedCommand.method, 'POST'); + assert.equal(forwardedCommand.route, '/rpc'); + assert.equal(forwardedCommand.authorization, `Bearer ${daemon.token}`); + assert.equal(forwardedCommand.statusCode, 200); + const forwardedRpc = JSON.parse(forwardedCommand.requestBody) as { + method?: string; + params?: { command?: string }; + }; + assert.equal(forwardedRpc.method, 'agent_device.command'); + assert.equal(forwardedRpc.params?.command, 'test'); - assert.equal(materialized.ok, true); - if (!materialized.ok) return; - assert.equal(materialized.data?.artifactsDir, clientSuiteDir); - assert.equal(materialized.data?.artifacts?.[0]?.localPath, clientSuiteDir); - assert.equal( - fs.readFileSync(path.join(clientSuiteDir, 'flow.ad', 'attempt-1', 'replay.ad'), 'utf8'), - 'open "Demo"\n', - ); - assert.equal( - fs.readFileSync(path.join(clientSuiteDir, 'flow.ad', 'attempt-1', 'result.txt'), 'utf8'), - 'status: passed\n', - ); - } finally { - cleanupDownloadableArtifact(artifactId); - await closeLoopbackServer(server); - fs.rmSync(tempDir, { recursive: true, force: true }); - } + const response: Extract = { + ok: true, + data: data as NonNullable['data']>, + }; + const materialized = await materializeRemoteArtifacts( + { baseUrl: proxy.baseUrl, token: PROXY_TOKEN }, + request, + response, + ); + assert.equal(materialized.ok, true); + if (!materialized.ok) return; + + const materializedSuiteDir = materialized.data?.artifactsDir; + if (typeof materializedSuiteDir !== 'string') { + throw new Error('the artifact route did not materialize a suite directory'); + } + assert.equal(path.dirname(materializedSuiteDir), clientRoot); + assert.ok(fs.existsSync(materializedSuiteDir)); + const attemptDir = path.join(materializedSuiteDir, 'flow.ad', 'attempt-1'); + assert.equal(fs.readFileSync(path.join(attemptDir, 'replay.ad'), 'utf8'), source); + const result = fs.readFileSync(path.join(attemptDir, 'result.txt'), 'utf8'); + assert.match(result, /status: passed/); + + assert.deepEqual( + proxy.forwarded.map((entry) => [ + entry.method, + entry.route, + entry.authorization, + entry.tenantId, + entry.statusCode, + ]), + [ + ['POST', '/rpc', `Bearer ${daemon.token}`, undefined, 200], + ['GET', `/artifacts/${artifactId}`, `Bearer ${daemon.token}`, undefined, 200], + ], + 'the real daemon must produce and serve the artifact through the shared proxy', + ); + } finally { + await proxy.close(); + } + }); + }, + ); }); test('directory downloads stage beside the destination and finish timeout cleanup before rejecting', async (t) => { if (await skipWhenLoopbackUnavailable(t, 'remote test-artifacts atomic staging coverage')) return; - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-remote-suite-timeout-')); + const tempDir = mkdtempForTestSync('agent-device-remote-suite-timeout-'); const clientRoot = path.join(tempDir, 'caller-side', 'artifacts'); const server = http.createServer((_req, res) => { res.statusCode = 200; @@ -149,6 +178,5 @@ test('directory downloads stage beside the destination and finish timeout cleanu ); } finally { await closeLoopbackServer(server); - fs.rmSync(tempDir, { recursive: true, force: true }); } }); diff --git a/test/integration/provider-scenarios/replay-repair-abort-close.test.ts b/test/integration/provider-scenarios/replay-repair-abort-close.test.ts index 9b1a88e6db..d047cbda3f 100644 --- a/test/integration/provider-scenarios/replay-repair-abort-close.test.ts +++ b/test/integration/provider-scenarios/replay-repair-abort-close.test.ts @@ -63,9 +63,9 @@ test('Provider-backed integration: repair close paths (abort, plain close, commi await armRepair(client, selection); const abortedClose = await daemon.callCommand('close', [], { saveScript: true }); - assertRpcError(abortedClose, 'COMMAND_FAILED', /repair was aborted/); + assertRpcError(abortedClose, 500, 'COMMAND_FAILED', /repair was aborted/); assert.equal(fs.existsSync(path.join(tempRoot, 'repair.healed.ad')), false); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); await daemon.close(); // 2. Aborted repair with plain close succeeds and leaves no artifact @@ -80,7 +80,7 @@ test('Provider-backed integration: repair close paths (abort, plain close, commi const plainClose = await daemon.callCommand('close', []); assertRpcOk(plainClose); assert.equal(fs.existsSync(path.join(tempRoot, 'repair.healed.ad')), false); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); await daemon.close(); // 3. Committed repair with close --save-script succeeds and writes artifact @@ -103,7 +103,7 @@ test('Provider-backed integration: repair close paths (abort, plain close, commi const committedClose = await daemon.callCommand('close', [], { saveScript: true }); assertRpcOk(committedClose); assert.equal(fs.existsSync(path.join(tempRoot, 'repair.healed.ad')), true); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); await daemon.close(); } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); diff --git a/test/integration/provider-scenarios/replay-repair-record-exclusion.test.ts b/test/integration/provider-scenarios/replay-repair-record-exclusion.test.ts index c04d2b1b2a..6386244ec0 100644 --- a/test/integration/provider-scenarios/replay-repair-record-exclusion.test.ts +++ b/test/integration/provider-scenarios/replay-repair-record-exclusion.test.ts @@ -98,7 +98,7 @@ async function armRepairAndAssertHeld(params: { repairPath: string; selection: ScenarioSelection; }): Promise<{ from: number; planDigest: string }> { - const { client, daemon, repairPath, selection } = params; + const { client, repairPath, selection } = params; const divergenceError = await client.replay .run({ path: repairPath, saveScript: true, ...selection }) .then( @@ -114,11 +114,6 @@ async function armRepairAndAssertHeld(params: { // R7/C1: the divergence reports the repair transaction as held — the exact // signal the #1271 stage-1 guidance clause is gated on. assert.equal(divergenceReport.resume.repairSessionHeld, true); - // The session is repair-armed: the repair variant's boundary is what the - // exclusion keys off (an ordinary `open --save-script` never sets it). - const publication = daemon.session()?.scriptPublication; - const armedBoundary = publication?.kind === 'repair' ? publication.boundary : undefined; - assert.equal(typeof armedBoundary, 'number'); return divergenceReport.resume; } @@ -131,7 +126,6 @@ async function assertRepairSegmentReadExclusion(params: { selection: ScenarioSelection; }): Promise { const { daemon, selection } = params; - const actionsBeforeReads = daemon.session()?.actions.length ?? 0; // (a) A diagnostic read used to LOCATE the target: excluded by default, // with no `--no-record` needed (the #1271 stage-1 foot-gun). @@ -139,11 +133,6 @@ async function assertRepairSegmentReadExclusion(params: { ...selection, }); assertRpcOk(diagnosticRead); - assert.equal( - daemon.session()?.actions.length, - actionsBeforeReads, - 'a diagnostic read inside a repair segment must not be recorded', - ); // (b) The corrective read (the wave-3 E3 shape: the diverged step is itself // a read), forced into the heal with `--record`. @@ -152,9 +141,6 @@ async function assertRepairSegmentReadExclusion(params: { record: true, }); assertRpcOk(correctiveRead); - const armedActions = daemon.session()?.actions ?? []; - assert.equal(armedActions.length, actionsBeforeReads + 1); - assert.equal(armedActions.at(-1)?.command, 'get'); // (c) `--record` and `--no-record` are opposite intents for one action. const conflictingFlags = await daemon.callCommand('get', ['text', SEARCH_SELECTOR], { @@ -162,7 +148,7 @@ async function assertRepairSegmentReadExclusion(params: { record: true, noRecord: true, }); - assertRpcError(conflictingFlags, 'INVALID_ARGS', /--record and --no-record are mutually/); + assertRpcError(conflictingFlags, 400, 'INVALID_ARGS', /--record and --no-record are mutually/); } /** What must (and must not) have landed in the healed script. */ diff --git a/test/integration/provider-scenarios/scroll-movement-observation.test.ts b/test/integration/provider-scenarios/scroll-movement-observation.test.ts index 6cd1cc4ed8..87ff8b17f6 100644 --- a/test/integration/provider-scenarios/scroll-movement-observation.test.ts +++ b/test/integration/provider-scenarios/scroll-movement-observation.test.ts @@ -215,7 +215,7 @@ test('Provider-backed integration scroll down that never moves a hidden-content const callsBeforeScroll = runnerTranscript.calls.length; const scroll = await daemon.callCommand('scroll', ['down']); - const errorData = assertRpcError(scroll, 'COMMAND_FAILED', /scroll down moved nothing/); + const errorData = assertRpcError(scroll, 500, 'COMMAND_FAILED', /scroll down moved nothing/); assert.equal( errorData.details && (errorData.details as Record).reason, 'scroll_no_progress', diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index 4054ca5f2b..14bed0c9d8 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -1,7 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { SCROLL_UNTIL_PASS_LIMIT } from '@agent-device/daemon/testing/scroll-until'; import { createAndroidSettingsWorld } from './android-world.ts'; import { withProviderScenarioResource } from './harness.ts'; @@ -60,7 +59,6 @@ test('Provider-backed integration scroll --until stops on the capture that bring // An exact count is what proves repeated scrolling on valid geometry, rather than a lucky // first capture or a budget burned to the limit. assert.equal(result.passes, ARRIVAL_PASSES); - assert.ok(ARRIVAL_PASSES < SCROLL_UNTIL_PASS_LIMIT); assert.match( String(result.message), new RegExp(`Scrolled down ${ARRIVAL_PASSES} passes until text=Terms was visible`), diff --git a/test/integration/provider-scenarios/stale-provider-runtime-admission.fixtures.ts b/test/integration/provider-scenarios/stale-provider-runtime-admission.fixtures.ts index 307e240d79..29ee2c6d90 100644 --- a/test/integration/provider-scenarios/stale-provider-runtime-admission.fixtures.ts +++ b/test/integration/provider-scenarios/stale-provider-runtime-admission.fixtures.ts @@ -1,112 +1,44 @@ -import { composedPlatformServicesFixture } from '../../../src/__tests__/daemon-composition/platform-services-fixture.ts'; -import fs from 'node:fs'; -import path from 'node:path'; -import { vi } from 'vitest'; import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; -import type { DeviceRuntimeGateway, RuntimeFacts } from '@agent-device/contracts/platform-runtime'; +import type { RuntimeFacts } from '@agent-device/contracts/platform-runtime'; import type { PlatformRuntimeHost, PlatformRuntimeOperations, PlatformRuntimeProviderModule, } from '@agent-device/contracts/platform-runtime-operations'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { handleAppDeploymentCommand } from '@agent-device/daemon/testing/handlers/session-app-deployment'; -import { createRequestRuntimeBindings } from '@agent-device/daemon/testing/request-runtime-binding'; -import { createRequestDispatchLedger } from '@agent-device/daemon/testing/request-dispatch-ledger'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { mkdtempForTestSync } from '@agent-device/test-kit'; import { createComposedPlatformRuntimeGateway } from '../../../src/platform-runtime-gateway.ts'; -function createProviderDeploymentAdmission(params: { +export async function inspectProviderDeploymentAdmission(params: { runtime: ProviderDeviceRuntime; module: PlatformRuntimeProviderModule; device: DeviceInfo; -}) { - const localLoad = vi.fn(async () => { - throw new Error('provider-owned device must not load a local runtime'); - }); +}): Promise> { + const localLoadCalls: string[] = []; const gateway = createComposedPlatformRuntimeGateway({ - modules: new Map([['android', { family: 'android', loadRuntime: localLoad }]]), + modules: new Map([ + [ + 'android', + { + family: 'android', + loadRuntime: async () => { + localLoadCalls.push('android'); + throw new Error('provider-owned device must not load a local runtime'); + }, + }, + ], + ]), loadHost: async () => ({}) as PlatformRuntimeHost, providerRuntimes: [params.runtime], providerModules: [{ runtime: params.runtime, module: params.module }], }); - let inspectedFacts: RuntimeFacts | undefined; - const inspectFacts = vi.fn(async (device: DeviceInfo) => { - inspectedFacts = await gateway.inspectFacts(device); - return inspectedFacts; - }); - const bind = vi.fn( - async (...args: Parameters['bind']>) => - await gateway.bind(...args), - ); - const bindings = createRequestRuntimeBindings({ - dispatchLedger: createRequestDispatchLedger(), - gateway: { inspectFacts, bind, shutdown: async () => {} }, - scope: { - signal: new AbortController().signal, - diagnostics: { emit: () => {} }, - progress: { report: () => {} }, - }, - admitDeviceClaim: async () => {}, - }); - return { - bindings, - inspectFacts, - bind, - localLoad, - facts: () => inspectedFacts, - close: async () => { - await bindings[Symbol.asyncDispose](); - await gateway.shutdown(); - }, - }; -} - -export async function inspectProviderDeploymentAdmission(params: { - runtime: ProviderDeviceRuntime; - module: PlatformRuntimeProviderModule; - device: DeviceInfo; -}) { - const admission = createProviderDeploymentAdmission(params); - const sessionStore = makeSessionStore('agent-device-provider-admission-'); - const appDirectory = mkdtempForTestSync('agent-device-provider-app-'); - const appPath = path.join(appDirectory, 'demo.apk'); - fs.writeFileSync(appPath, 'fixture'); try { - sessionStore.publish('default', { - name: 'default', - createdAt: 1, - actions: [], - device: params.device, - }); - const response = await handleAppDeploymentCommand({ - platformServices: composedPlatformServicesFixture(), - req: { - token: 'test', - session: 'default', - command: 'install', - positionals: [appPath], - flags: {}, - }, - command: 'install', - sessionName: 'default', - sessionStore, - inspectFacts: admission.bindings.inspectFacts, - bindDevice: admission.bindings.bindDevice, - }); - const facts = admission.facts(); - if (!facts) throw new Error('Expected provider facts inspection'); - return { - response, - facts, - inspectFactCalls: admission.inspectFacts.mock.calls.length, - bindCalls: admission.bind.mock.calls.length, - localLoadCalls: admission.localLoad.mock.calls.length, - }; + const facts = await gateway.inspectFacts(params.device); + if (localLoadCalls.length > 0) { + throw new Error('provider-owned facts inspection loaded a local runtime'); + } + return facts; } finally { - await admission.close(); - fs.rmSync(appDirectory, { recursive: true, force: true }); + await gateway.shutdown(); } } diff --git a/test/integration/provider-scenarios/stale-provider-runtime-admission.test.ts b/test/integration/provider-scenarios/stale-provider-runtime-admission.test.ts index 69c8cce22c..847ba7d369 100644 --- a/test/integration/provider-scenarios/stale-provider-runtime-admission.test.ts +++ b/test/integration/provider-scenarios/stale-provider-runtime-admission.test.ts @@ -1,10 +1,16 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { expect, test, vi } from 'vitest'; -import type { DeviceLease, ProviderDeviceRuntime } from '@agent-device/contracts/device'; -import type { PlatformRuntimeProviderModule } from '@agent-device/contracts/platform-runtime-operations'; +import type { DeviceLease } from '@agent-device/contracts/device'; import { createBundledCloudWebDriverRuntimes } from '@agent-device/provider-webdriver'; import { createLimrunRuntime, type LimrunRuntimeDependencies } from '@agent-device/provider-limrun'; import { limrunTestDependencies } from '../../../src/platform-runtime-gateway.fixtures.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { createProviderDeviceRuntimeRequestProviders } from '../../../src/provider-device-runtime.ts'; +import { assertRpcError, assertRpcOk } from './assertions.ts'; +import { createProviderScenarioHarness } from './harness.ts'; import { CloudWebDriverTestServer, cloudWebDriverTestJson, @@ -57,7 +63,7 @@ vi.mock('@limrun/api/instance-client', () => ({ })), })); -test('a stale WebDriver device remains provider-owned and fails admission before local execution', async () => { +test('a released WebDriver device remains provider-owned with deployment unavailable', async () => { const server = await startCloudWebDriverTestServer(new StaleWebDriverServer()); const runtime = webDriverRuntime(server.url); const lease = providerLease(CLOUD_WEBDRIVER_PROVIDERS.browserStack, 'webdriver-stale'); @@ -77,7 +83,7 @@ test('a stale WebDriver device remains provider-owned and fails admission before await runtime.leaseLifecycle.release?.(lease); expect(runtime.ownsDevice(device)).toBe(true); - const facts = await expectUnsupportedDeploymentAdmission({ + const facts = await inspectProviderDeploymentAdmission({ runtime, module: runtime.platformRuntimeModule, device, @@ -95,7 +101,7 @@ test('a stale WebDriver device remains provider-owned and fails admission before } }); -test('a stale Limrun session is rejected by admission facts before a deployment bind', async () => { +test('a released Limrun session exposes unavailable deployment facts before binding', async () => { const registration = createLimrunRuntime({ apiKey: 'limrun-test-key' }, limrunDependencies(), { includePlatformModule: true, }); @@ -104,10 +110,13 @@ test('a stale Limrun session is rejected by admission facts before a deployment try { const allocation = await registration.runtime.leaseLifecycle.allocate?.(lease); const device = readAllocatedDevice(allocation); + expect(limrunState.androidCreate).toHaveBeenCalledTimes(1); await registration.runtime.leaseLifecycle.release?.(lease); + expect(limrunState.disconnect).toHaveBeenCalled(); expect(registration.runtime.ownsDevice(device)).toBe(true); - const facts = await expectUnsupportedDeploymentAdmission({ + const createCallsBeforeAdmission = limrunState.androidCreate.mock.calls.length; + const facts = await inspectProviderDeploymentAdmission({ runtime: registration.runtime, module: registration.platformModule, device, @@ -119,6 +128,7 @@ test('a stale Limrun session is rejected by admission facts before a deployment deployApp: { available: false, reason: 'owner-capability-missing' }, }, }); + expect(limrunState.androidCreate).toHaveBeenCalledTimes(createCallsBeforeAdmission); } finally { await registration.runtime.shutdown(); } @@ -128,58 +138,97 @@ test('an active AWS WebDriver device denies unsupported install from facts befor const server = await ProviderRegressionServer.start(); const host = new AwsRemoteAccessHost({ appiumEndpoint: `${server.url}/wd/hub/` }); const runtime = awsWebDriverRuntime(host); - const lease = providerLease(CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, 'aws-active-install'); + const provider = runtime.platformRuntimeModule; + if (!provider) throw new Error('AWS WebDriver runtime requires its platform runtime module'); + let bindCalls = 0; + const countedProvider = Object.freeze({ + ...provider, + loadRuntime: async (...args: Parameters) => { + const owner = await provider.loadRuntime(...args); + return Object.freeze({ + ...owner, + bind: async (request: Parameters[0]) => { + bindCalls += 1; + return await owner.bind(request); + }, + }); + }, + }); + const providers = createProviderDeviceRuntimeRequestProviders([runtime]); + const daemon = await createProviderScenarioHarness({ + ...providers, + deviceInventorySource: providers.deviceInventorySource!, + platformRuntime: { + providerRuntimes: [runtime], + providerModules: [{ runtime, module: countedProvider }], + }, + }); + const leaseId = 'a'.repeat(32); + const flags = { + ...awsRegressionContext().flags, + tenant: 'team-a', + runId: 'run-a', + leaseId, + leaseProvider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + }; + const meta = { + tenantId: 'team-a', + runId: 'run-a', + leaseId, + leaseBackend: 'android-instance' as const, + leaseProvider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + deviceKey: 'provider-device-regression', + clientId: 'client-a', + }; + let lease: DeviceLease | undefined; try { - await runtime.leaseLifecycle.allocate?.(lease, awsRegressionContext()); + lease = assertRpcOk<{ lease: DeviceLease }>( + await daemon.callCommand('lease_allocate', [], flags, { meta }), + ).lease; const device = await allocatedDevice(runtime, lease); + const tempDir = mkdtempForTestSync('agent-device-provider-install-admission-'); + const appPath = path.join(tempDir, 'demo.apk'); + fs.writeFileSync(appPath, 'fake apk'); const providerCallsBeforeAdmission = server.calls.length; const hostCallsBeforeAdmission = host.calls.length; - - const facts = await expectUnsupportedDeploymentAdmission({ - runtime, - module: runtime.platformRuntimeModule, - device, - }); - - expect(facts).toMatchObject({ - device: { providerMode: 'provider-runtime' }, - operations: { - ensureReady: { available: true }, - deployApp: { - available: false, - reason: 'owner-capability-missing', - hint: expect.stringMatching(/local artifact upload\/install is not implemented/), - }, - materializeAppSource: { available: false, reason: 'owner-capability-missing' }, - deployMaterializedApp: { available: false, reason: 'owner-capability-missing' }, - }, - }); + const bindCallsBeforeAdmission = bindCalls; + const response = await daemon.callCommand( + 'install', + [appPath], + { ...flags, serial: device.id }, + { meta }, + ); + const error = assertRpcError( + response, + 500, + 'UNSUPPORTED_OPERATION', + /not supported on this device/, + ); + assert.equal( + (error.details as Record | undefined)?.reason, + 'owner-capability-missing', + ); + expect(bindCalls).toBe(bindCallsBeforeAdmission); expect(server.calls).toHaveLength(providerCallsBeforeAdmission); expect(host.calls).toHaveLength(hostCallsBeforeAdmission); } finally { - await runtime.leaseLifecycle.release?.(lease); - await runtime.shutdown(); - await server.close(); + try { + if (lease) await daemon.callCommand('lease_release', [], flags, { meta }); + } finally { + try { + await daemon.close(); + } finally { + try { + await runtime.shutdown(); + } finally { + await server.close(); + } + } + } } }); -async function expectUnsupportedDeploymentAdmission(params: { - runtime: ProviderDeviceRuntime; - module: PlatformRuntimeProviderModule; - device: DeviceInfo; -}) { - const result = await inspectProviderDeploymentAdmission(params); - expect(result.response).toMatchObject({ - ok: false, - error: { code: 'UNSUPPORTED_OPERATION', details: { reason: 'owner-capability-missing' } }, - }); - expect(result.inspectFactCalls).toBe(1); - expect(result.bindCalls).toBe(0); - expect(result.localLoadCalls).toBe(0); - return result.facts; -} - function webDriverRuntime(endpoint: string) { const runtimes = createBundledCloudWebDriverRuntimes({ env: { diff --git a/test/integration/provider-scenarios/stale-ref-warning.test.ts b/test/integration/provider-scenarios/stale-ref-warning.test.ts index 6354a00cb5..327b0df46e 100644 --- a/test/integration/provider-scenarios/stale-ref-warning.test.ts +++ b/test/integration/provider-scenarios/stale-ref-warning.test.ts @@ -128,7 +128,7 @@ test('Provider-backed integration iOS @refs reject after a selector press replac assert.equal(selectorData.warning, undefined); const stalePress = await daemon.callCommand('press', ['@e2'], {}); - assertRpcError(stalePress, 'COMMAND_FAILED', /Ref @e2 belongs to an expired ref frame/); + assertRpcError(stalePress, 500, 'COMMAND_FAILED', /Ref @e2 belongs to an expired ref frame/); assert.equal( (stalePress.json?.error?.data?.details as Record)?.reason, 'ref_frame_expired', @@ -191,7 +191,7 @@ test('Provider-backed iOS press rejects a stale ref after navigation', async () assertRpcOk(await daemon.callCommand('press', ['label=Back'], {})); const stalePress = await daemon.callCommand('press', ['@e3'], {}); - assertRpcError(stalePress, 'COMMAND_FAILED', /Ref @e3 belongs to an expired ref frame/); + assertRpcError(stalePress, 500, 'COMMAND_FAILED', /Ref @e3 belongs to an expired ref frame/); assert.equal( (stalePress.json?.error?.data?.details as Record)?.reason, 'ref_frame_expired', diff --git a/test/integration/provider-scenarios/vega-tv-remote.test.ts b/test/integration/provider-scenarios/vega-tv-remote.test.ts index 3ddad7f116..15b5b3e72d 100644 --- a/test/integration/provider-scenarios/vega-tv-remote.test.ts +++ b/test/integration/provider-scenarios/vega-tv-remote.test.ts @@ -54,13 +54,6 @@ test('Provider-backed Vega TV flow routes lifecycle and every remote control as deviceInventoryProvider: async () => [VEGA_TV], }), async (daemon) => { - daemon.publishSession('default', { - name: 'default', - device: VEGA_TV, - createdAt: Date.now(), - actions: [], - }); - const open = await daemon.callCommand('open', [APP_ID], SELECTION); assertRpcOk(open); @@ -83,30 +76,9 @@ test('Provider-backed Vega TV flow routes lifecycle and every remote control as assertRpcOk(await daemon.callCommand('back', [], SELECTION)); assertRpcOk(await daemon.callCommand('home', [], SELECTION)); - assert.deepEqual( - daemon.session()?.actions.map((action) => action.command), - [ - 'open', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'tv-remote', - 'back', - 'home', - ], - ); - assertRpcOk(await daemon.callCommand('close', [APP_ID], SELECTION)); assert.deepEqual(calls, expectedVegaCalls()); - assert.equal(daemon.session(), undefined); + assert.equal(await daemon.hasSession(), false); }, ); }); diff --git a/test/integration/provider-scenarios/versioned-refs.test.ts b/test/integration/provider-scenarios/versioned-refs.test.ts index 6049f9e700..171ae7d074 100644 --- a/test/integration/provider-scenarios/versioned-refs.test.ts +++ b/test/integration/provider-scenarios/versioned-refs.test.ts @@ -132,6 +132,7 @@ test('Provider-backed integration rejects stale pinned @refs and accepts current const pinnedStale = await daemon.callCommand('press', [`@e2~s${g1}`], {}); const pinnedStaleData = assertRpcError( pinnedStale, + 500, 'COMMAND_FAILED', /Ref @e2 was minted from a superseded snapshot generation/, ); diff --git a/test/integration/provider-scenarios/web-desktop.test.ts b/test/integration/provider-scenarios/web-desktop.test.ts index 4c7febe27e..2090c5d958 100644 --- a/test/integration/provider-scenarios/web-desktop.test.ts +++ b/test/integration/provider-scenarios/web-desktop.test.ts @@ -229,45 +229,6 @@ test('Provider-backed integration web desktop flow uses semantic web provider ca JSON.stringify(refLessClick.json), ); - const actions = daemon.session()?.actions ?? []; - assert.ok( - actions.some( - (action) => - action.command === 'hover' && - action.positionals.join(' ') === '@e4' && - action.result?.x === undefined && - action.result?.y === undefined, - ), - 'Expected ref hover action to be recorded on the session without fabricated coordinates', - ); - assert.ok( - actions.some( - (action) => - action.command === 'click' && - action.positionals.join(' ') === '@e4' && - action.result?.x === undefined && - action.result?.y === undefined, - ), - 'Expected ref click action to be recorded on the session without fabricated coordinates', - ); - assert.ok( - actions.some( - (action) => - action.command === 'fill' && - action.positionals.join(' ') === '@e3 qa@example.test' && - action.flags.delayMs === 1 && - action.result?.x === undefined && - action.result?.y === undefined, - ), - 'Expected ref fill action to be recorded on the session without fabricated coordinates', - ); - assert.ok( - actions.some( - (action) => action.command === 'type' && action.positionals.join(' ') === ' ok', - ), - 'Expected type action to be recorded on the session', - ); - const close = await daemon.callCommand('close', [WEB_URL]); assert.equal(close.statusCode, 200, JSON.stringify(close.json)); diff --git a/test/integration/provider-scenarios/web-provider.test.ts b/test/integration/provider-scenarios/web-provider.test.ts index 4e36b4e5db..477b4ffa35 100644 --- a/test/integration/provider-scenarios/web-provider.test.ts +++ b/test/integration/provider-scenarios/web-provider.test.ts @@ -108,7 +108,6 @@ test('web provider is scoped through the request router and dispatch path', asyn const harness = await createProviderScenarioHarness({ deviceInventoryProvider: async () => [WEB_DESKTOP_DEVICE], - platformRuntime: true, webProvider: ({ device, session }) => { calls.push(`scope:${session?.name ?? 'none'}:${device.id}`); return webProvider; @@ -139,7 +138,6 @@ test('web provider is scoped through the request router and dispatch path', asyn label: 'main', rect: { x: 0, y: 0, width: 320, height: 240 }, depth: 0, - parentIndex: undefined, ref: 'e1', kind: 'section', }, @@ -252,7 +250,6 @@ test('non-dense browser refs survive snapshot storage and routed fill and click' async () => { const harness = await createProviderScenarioHarness({ deviceInventoryProvider: async () => [WEB_DESKTOP_DEVICE], - platformRuntime: true, webProvider: () => provider, }); try { diff --git a/test/integration/provider-scenarios/web-world.ts b/test/integration/provider-scenarios/web-world.ts index d1985af0a0..845ea82218 100644 --- a/test/integration/provider-scenarios/web-world.ts +++ b/test/integration/provider-scenarios/web-world.ts @@ -128,7 +128,6 @@ export async function createWebDesktopWorld(): Promise { }; const daemon = await createProviderScenarioHarness({ - platformRuntime: true, webProvider: () => provider, deviceInventoryProvider: async () => [PROVIDER_SCENARIO_WEB], }); diff --git a/test/output-economy/daemon-route-fixtures.ts b/test/output-economy/daemon-route-fixtures.ts new file mode 100644 index 0000000000..8a68483e6a --- /dev/null +++ b/test/output-economy/daemon-route-fixtures.ts @@ -0,0 +1,393 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { PNG } from '@agent-device/capture-kit/png'; +import type { BackendDumpNetworkResult } from '@agent-device/contracts/backend-diagnostics'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { WebProvider } from '@agent-device/platform-web'; +import { WEB_DESKTOP_DEVICE } from '@agent-device/test-kit'; +import { createProviderScenarioHarness } from '../integration/provider-scenarios/harness.ts'; +import { NETWORK_SELECTION_RESULT } from './selection-fixtures.ts'; + +const SCREENSHOT_PATH = path.join(os.tmpdir(), `agent-device-output-economy-${process.pid}.png`); +const STABLE_SCREENSHOT_PATH = '/tmp/agent-device/economy-fixture.png'; + +const beforeOrder: RawSnapshotNode[] = [ + { + index: 0, + ref: 'e1', + type: 'Window', + role: 'window', + label: 'Checkout', + depth: 0, + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + ref: 'e2', + type: 'TextField', + role: 'text-field', + label: 'Email', + value: 'qa@example.com', + identifier: 'checkout-email', + hittable: true, + enabled: true, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 120, width: 350, height: 44 }, + }, + { + index: 2, + ref: 'e3', + type: 'Button', + role: 'button', + label: 'Place order', + identifier: 'submit-order', + hittable: true, + enabled: true, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 720, width: 350, height: 48 }, + }, + { + index: 3, + type: 'StaticText', + role: 'text', + label: 'Shipping address', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 420, width: 350, height: 24 }, + }, + { + index: 4, + type: 'StaticText', + role: 'text', + label: 'Payment method', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 480, width: 350, height: 24 }, + }, + { + index: 5, + type: 'StaticText', + role: 'text', + label: 'Order total', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 540, width: 350, height: 24 }, + }, +]; + +const afterOrder: RawSnapshotNode[] = [ + beforeOrder[0]!, + beforeOrder[1]!, + { + index: 2, + ref: 'e4', + type: 'StaticText', + role: 'text', + label: 'Order confirmed', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 660, width: 350, height: 32 }, + }, + { + index: 3, + ref: 'e5', + type: 'Button', + role: 'button', + label: 'View receipt', + hittable: true, + enabled: true, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 720, width: 350, height: 48 }, + }, + { + index: 4, + type: 'StaticText', + role: 'text', + label: 'Shipping address', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 420, width: 350, height: 24 }, + }, + { + index: 5, + type: 'StaticText', + role: 'text', + label: 'Payment method', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 480, width: 350, height: 24 }, + }, + { + index: 6, + type: 'StaticText', + role: 'text', + label: 'Order total', + hittable: false, + enabled: false, + depth: 1, + parentIndex: 0, + rect: { x: 20, y: 540, width: 350, height: 24 }, + }, +]; + +function makePng(): Buffer { + const png = new PNG({ width: 64, height: 64 }); + png.data.fill(255); + return PNG.sync.write(png); +} + +function stableResponseData(response: { json: any }, operation: string): Record { + const error = response.json?.error; + if (error) throw new Error(`${operation} failed: ${JSON.stringify(error)}`); + const data = response.json?.result?.data; + if (!data || typeof data !== 'object') { + throw new Error(`${operation} returned no response data`); + } + return stabilizeVolatileFields(data) as Record; +} + +function stabilizeVolatileFields(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stabilizeVolatileFields); + if (typeof value === 'string') return value.replaceAll(SCREENSHOT_PATH, STABLE_SCREENSHOT_PATH); + if (value === null || typeof value !== 'object') return value; + const record = value as Record; + return Object.fromEntries( + Object.entries(record).map(([key, entry]) => [ + key, + key === 'artifactId' && typeof entry === 'string' + ? 'artifact-economy-fixture' + : record.artifactType === 'screenshot' && key === 'fileName' && typeof entry === 'string' + ? 'economy-fixture.png' + : ['maxMs', 'p50Ms', 'p95Ms', 'waitedMs'].includes(key) && typeof entry === 'number' + ? 100 + : stabilizeVolatileFields(entry), + ]), + ); +} + +export async function captureOutputEconomyDaemonResponses() { + let page: 'before-order' | 'after-order' | 'receipt-open' = 'before-order'; + const routeTrace: string[] = []; + const png = makePng(); + const networkEntries = + NETWORK_SELECTION_RESULT.entries as unknown as BackendDumpNetworkResult['entries']; + const webProvider: WebProvider = { + async open() {}, + async close() {}, + async snapshot() { + routeTrace.push(`snapshot:${page}`); + const nodes = page === 'before-order' ? beforeOrder : afterOrder; + return { + nodes: + page === 'receipt-open' + ? nodes.filter((node) => node.ref !== 'e5' && node.ref !== 'e4') + : nodes, + truncated: false, + }; + }, + async screenshot(outPath) { + fs.writeFileSync(outPath, png); + }, + async setViewport() {}, + async click(x, y) { + routeTrace.push(`click:${x}:${y}`); + if (y >= 700 && page === 'before-order') page = 'after-order'; + else if (y >= 700 && page === 'after-order') page = 'receipt-open'; + }, + async fill() {}, + async typeText() {}, + async scroll() {}, + async dumpNetwork() { + return { + entries: networkEntries, + backend: 'agent-browser', + redacted: false, + notes: NETWORK_SELECTION_RESULT.notes as string[], + }; + }, + }; + const daemon = await createProviderScenarioHarness({ + deviceInventoryProvider: async () => [WEB_DESKTOP_DEVICE], + webProvider: () => webProvider, + }); + + try { + const open = async (session: string) => { + page = 'before-order'; + stableResponseData( + await daemon.callCommand( + 'open', + ['https://example.test/checkout'], + { platform: 'web' }, + { session }, + ), + `open ${session}`, + ); + }; + const close = async (session: string) => { + stableResponseData( + await daemon.callCommand('close', [], {}, { session }), + `close ${session}`, + ); + }; + const clickAndSettle = async ( + session: string, + responseLevel?: 'digest', + ): Promise> => { + const response = await daemon.callCommand( + 'click', + ['195', '744'], + { platform: 'web', settle: true, settleQuietMs: 25, timeoutMs: 10_000 }, + { + session, + ...(responseLevel ? { meta: { responseLevel } } : {}), + }, + ); + if (response.json?.error) { + throw new Error( + `click ${responseLevel ?? 'default'} failed: ${JSON.stringify(response.json.error)}; ${routeTrace.join(', ')}`, + ); + } + const data = stableResponseData(response, `click ${responseLevel ?? 'default'}`); + if ((data.settle as { settled?: unknown } | undefined)?.settled !== true) { + throw new Error( + `click ${responseLevel ?? 'default'} did not settle: ${JSON.stringify(data.settle)}; ${routeTrace.join(', ')}`, + ); + } + return data; + }; + + await open('default'); + + const snapshotDefault = stableResponseData( + await daemon.callCommand('snapshot', [], { platform: 'web' }), + 'snapshot default', + ); + const snapshotDigest = stableResponseData( + await daemon.callCommand( + 'snapshot', + [], + { platform: 'web' }, + { meta: { responseLevel: 'digest' } }, + ), + 'snapshot digest', + ); + const selectorDefault = stableResponseData( + await daemon.callCommand('get', ['text', '@e2'], { platform: 'web' }), + 'selector read default', + ); + const selectorDigest = stableResponseData( + await daemon.callCommand( + 'get', + ['text', '@e2'], + { platform: 'web' }, + { meta: { responseLevel: 'digest' } }, + ), + 'selector read digest', + ); + + const settleDefault = await clickAndSettle('default'); + const postOrderSnapshotDefault = stableResponseData( + await daemon.callCommand('snapshot', [], { platform: 'web' }, { session: 'default' }), + 'post-order snapshot', + ); + const settleTailDefault = await clickAndSettle('default'); + await close('default'); + + await open('digest'); + stableResponseData( + await daemon.callCommand('snapshot', [], { platform: 'web' }, { session: 'digest' }), + 'digest session snapshot', + ); + const screenshotDefault = stableResponseData( + await daemon.callCommand( + 'screenshot', + [SCREENSHOT_PATH], + { platform: 'web', overlayRefs: true }, + { session: 'digest', meta: { clientArtifactPaths: { path: SCREENSHOT_PATH } } }, + ), + 'screenshot default', + ); + const screenshotDigest = stableResponseData( + await daemon.callCommand( + 'screenshot', + [SCREENSHOT_PATH], + { platform: 'web', overlayRefs: true }, + { + session: 'digest', + meta: { + responseLevel: 'digest', + clientArtifactPaths: { path: SCREENSHOT_PATH }, + }, + }, + ), + 'screenshot digest', + ); + const snapshotDigestBeforeSettle = stableResponseData( + await daemon.callCommand('snapshot', [], { platform: 'web' }, { session: 'digest' }), + 'digest pre-click snapshot', + ); + const settleDigest = await clickAndSettle('digest', 'digest'); + const postOrderSnapshotDigest = stableResponseData( + await daemon.callCommand('snapshot', [], { platform: 'web' }, { session: 'digest' }), + 'digest post-order snapshot', + ); + const settleTailDigest = await clickAndSettle('digest', 'digest'); + + const networkDefault = stableResponseData( + await daemon.callCommand( + 'network', + ['dump', '25', 'all'], + { platform: 'web' }, + { session: 'digest' }, + ), + 'network default', + ); + const networkDigest = stableResponseData( + await daemon.callCommand( + 'network', + ['dump', '25', 'all'], + { platform: 'web' }, + { session: 'digest', meta: { responseLevel: 'digest' } }, + ), + 'network digest', + ); + + return { + snapshotDefault, + snapshotDigest, + snapshotDigestBeforeSettle, + selectorDefault, + selectorDigest, + settleDefault, + settleDigest, + settleTailDefault, + settleTailDigest, + postOrderSnapshotDefault, + postOrderSnapshotDigest, + screenshotDefault, + screenshotDigest, + networkDefault, + networkDigest, + }; + } finally { + await daemon.close(); + fs.rmSync(SCREENSHOT_PATH, { force: true }); + } +} diff --git a/test/output-economy/output-economy.baseline.json b/test/output-economy/output-economy.baseline.json index 85c83d9bac..bc7f624dc5 100644 --- a/test/output-economy/output-economy.baseline.json +++ b/test/output-economy/output-economy.baseline.json @@ -14,7 +14,7 @@ "shape": "$.nodes:array|$.nodes[].depth:number|$.nodes[].enabled:boolean|$.nodes[].hittable:boolean|$.nodes[].identifier:string|$.nodes[].index:number|$.nodes[].kind:string|$.nodes[].label:string|$.nodes[].parentIndex:number|$.nodes[].rect.height:number|$.nodes[].rect.width:number|$.nodes[].rect.x:number|$.nodes[].rect.y:number|$.nodes[].rect:object|$.nodes[].ref:string|$.nodes[].role:string|$.nodes[].type:string|$.nodes[].value:string|$.nodes[]:object|$.truncated:boolean|$.visibility.partial:boolean|$.visibility.reasons:array|$.visibility.totalNodeCount:number|$.visibility.visibleNodeCount:number|$.visibility:object|$:object" }, "snapshot.digest.json": { - "bytes": 210, + "bytes": 214, "lines": 1, "refs": 2, "hints": 0, @@ -35,11 +35,11 @@ "shape": "$.message:string|$.ref:string|$.settle.captures:number|$.settle.diff.lines:array|$.settle.diff.lines[].kind:string|$.settle.diff.lines[].ref:string|$.settle.diff.lines[].text:string|$.settle.diff.lines[]:object|$.settle.diff.summary.additions:number|$.settle.diff.summary.removals:number|$.settle.diff.summary.unchanged:number|$.settle.diff.summary:object|$.settle.diff:object|$.settle.quietMs:number|$.settle.refsGeneration:number|$.settle.settled:boolean|$.settle.timeoutMs:number|$.settle.waitedMs:number|$.settle:object|$.x:number|$.y:number|$:object" }, "settle.digest.json": { - "bytes": 264, + "bytes": 274, "lines": 1, - "refs": 3, + "refs": 2, "hints": 0, - "shape": "$.message:string|$.ref:string|$.settle.captures:number|$.settle.diff.summary.additions:number|$.settle.diff.summary.removals:number|$.settle.diff.summary.unchanged:number|$.settle.diff.summary:object|$.settle.diff:object|$.settle.quietMs:number|$.settle.refs:array|$.settle.refsGeneration:number|$.settle.refs[].ref:string|$.settle.refs[]:object|$.settle.settled:boolean|$.settle.timeoutMs:number|$.settle.waitedMs:number|$.settle:object|$.x:number|$.y:number|$:object" + "shape": "$.message:string|$.settle.captures:number|$.settle.diff.summary.additions:number|$.settle.diff.summary.removals:number|$.settle.diff.summary.unchanged:number|$.settle.diff.summary:object|$.settle.diff:object|$.settle.quietMs:number|$.settle.refs:array|$.settle.refsGeneration:number|$.settle.refs[].ref:string|$.settle.refs[]:object|$.settle.settled:boolean|$.settle.timeoutMs:number|$.settle.waitedMs:number|$.settle:object|$.targetKind:string|$.x:number|$.y:number|$:object" }, "settle-tail.default.text": { "bytes": 188, @@ -49,11 +49,11 @@ "shape": "text" }, "settle-tail.digest.json": { - "bytes": 325, + "bytes": 306, "lines": 1, - "refs": 3, + "refs": 1, "hints": 0, - "shape": "$.message:string|$.ref:string|$.settle.captures:number|$.settle.diff.summary.additions:number|$.settle.diff.summary.removals:number|$.settle.diff.summary.unchanged:number|$.settle.diff.summary:object|$.settle.diff:object|$.settle.quietMs:number|$.settle.refsGeneration:number|$.settle.settled:boolean|$.settle.tail:array|$.settle.tail[].label:string|$.settle.tail[].ref:string|$.settle.tail[].role:string|$.settle.tail[]:object|$.settle.timeoutMs:number|$.settle.waitedMs:number|$.settle:object|$.x:number|$.y:number|$:object" + "shape": "$.message:string|$.settle.captures:number|$.settle.diff.summary.additions:number|$.settle.diff.summary.removals:number|$.settle.diff.summary.unchanged:number|$.settle.diff.summary:object|$.settle.diff:object|$.settle.quietMs:number|$.settle.refsGeneration:number|$.settle.settled:boolean|$.settle.tail:array|$.settle.tail[].label:string|$.settle.tail[].ref:string|$.settle.tail[].role:string|$.settle.tail[]:object|$.settle.timeoutMs:number|$.settle.waitedMs:number|$.settle:object|$.targetKind:string|$.x:number|$.y:number|$:object" }, "not-settled.default.text": { "bytes": 119, @@ -70,25 +70,25 @@ "shape": "$.node.appName:string|$.node.bundleId:string|$.node.depth:number|$.node.enabled:boolean|$.node.hittable:boolean|$.node.identifier:string|$.node.index:number|$.node.kind:string|$.node.label:string|$.node.parentIndex:number|$.node.pid:number|$.node.rect.height:number|$.node.rect.width:number|$.node.rect.x:number|$.node.rect.y:number|$.node.rect:object|$.node.ref:string|$.node.role:string|$.node.surface:string|$.node.type:string|$.node.value:string|$.node.windowTitle:string|$.node:object|$.ref:string|$.text:string|$.warning:string|$:object" }, "selector-read.digest.json": { - "bytes": 89, + "bytes": 36, "lines": 1, "refs": 1, "hints": 0, - "shape": "$.ref:string|$.text:string|$.warning:string|$:object" + "shape": "$.ref:string|$.text:string|$:object" }, "screenshot.default.json": { - "bytes": 603, + "bytes": 650, "lines": 1, "refs": 2, "hints": 0, - "shape": "$.artifacts:array|$.artifacts[].artifactId:string|$.artifacts[].artifactType:string|$.artifacts[].field:string|$.artifacts[].fileName:string|$.artifacts[]:object|$.height:number|$.logicalHeight:number|$.logicalWidth:number|$.overlayRefs:array|$.overlayRefs[].center.x:number|$.overlayRefs[].center.y:number|$.overlayRefs[].center:object|$.overlayRefs[].label:string|$.overlayRefs[].overlayRect.height:number|$.overlayRefs[].overlayRect.width:number|$.overlayRefs[].overlayRect.x:number|$.overlayRefs[].overlayRect.y:number|$.overlayRefs[].overlayRect:object|$.overlayRefs[].rect.height:number|$.overlayRefs[].rect.width:number|$.overlayRefs[].rect.x:number|$.overlayRefs[].rect.y:number|$.overlayRefs[].rect:object|$.overlayRefs[].ref:string|$.overlayRefs[]:object|$.path:string|$.pixelDensity:number|$.width:number|$:object" + "shape": "$.artifacts:array|$.artifacts[].artifactId:string|$.artifacts[].artifactType:string|$.artifacts[].field:string|$.artifacts[].fileName:string|$.artifacts[].localPath:string|$.artifacts[]:object|$.height:number|$.message:string|$.overlayRefs:array|$.overlayRefs[].center.x:number|$.overlayRefs[].center.y:number|$.overlayRefs[].center:object|$.overlayRefs[].label:string|$.overlayRefs[].overlayRect.height:number|$.overlayRefs[].overlayRect.width:number|$.overlayRefs[].overlayRect.x:number|$.overlayRefs[].overlayRect.y:number|$.overlayRefs[].overlayRect:object|$.overlayRefs[].rect.height:number|$.overlayRefs[].rect.width:number|$.overlayRefs[].rect.x:number|$.overlayRefs[].rect.y:number|$.overlayRefs[].rect:object|$.overlayRefs[].ref:string|$.overlayRefs[]:object|$.path:string|$.width:number|$:object" }, "screenshot.digest.json": { - "bytes": 360, + "bytes": 352, "lines": 1, "refs": 2, "hints": 0, - "shape": "$.artifacts:array|$.artifacts[].artifactId:string|$.artifacts[].artifactType:string|$.artifacts[].field:string|$.artifacts[].fileName:string|$.artifacts[]:object|$.height:number|$.logicalHeight:number|$.logicalWidth:number|$.overlayCount:number|$.overlayRefs:array|$.overlayRefs[].label:string|$.overlayRefs[].ref:string|$.overlayRefs[]:object|$.path:string|$.pixelDensity:number|$.width:number|$:object" + "shape": "$.artifacts:array|$.artifacts[].artifactId:string|$.artifacts[].artifactType:string|$.artifacts[].field:string|$.artifacts[].fileName:string|$.artifacts[].localPath:string|$.artifacts[]:object|$.height:number|$.overlayCount:number|$.overlayRefs:array|$.overlayRefs[].label:string|$.overlayRefs[].ref:string|$.overlayRefs[]:object|$.path:string|$.width:number|$:object" }, "error.normalized.json": { "bytes": 212, @@ -119,11 +119,11 @@ "shape": "$.active:boolean|$.backend:string|$.entries:array|$.entries[].durationMs:number|$.entries[].headers:string|$.entries[].line:number|$.entries[].method:string|$.entries[].packetId:string|$.entries[].raw:string|$.entries[].requestBody:string|$.entries[].requestHeaders.authorization:string|$.entries[].requestHeaders.content-type:string|$.entries[].requestHeaders.x-request-id:string|$.entries[].requestHeaders:object|$.entries[].responseBody:string|$.entries[].responseHeaders.content-type:string|$.entries[].responseHeaders.x-request-id:string|$.entries[].responseHeaders.x-retry-after-ms:string|$.entries[].responseHeaders:object|$.entries[].status:number|$.entries[].timestamp:string|$.entries[].url:string|$.entries[]:object|$.exists:boolean|$.include:string|$.limits.maxEntries:number|$.limits.maxPayloadChars:number|$.limits.maxScanLines:number|$.limits:object|$.matchedLines:number|$.notes:array|$.notes[]:string|$.path:string|$.scannedLines:number|$.state:string|$:object" }, "selection.network.digest.json": { - "bytes": 1665, + "bytes": 1594, "lines": 1, "refs": 0, "hints": 0, - "shape": "$.active:boolean|$.backend:string|$.entries:array|$.entries[].durationMs:number|$.entries[].line:number|$.entries[].method:string|$.entries[].packetId:string|$.entries[].status:number|$.entries[].timestamp:string|$.entries[].url:string|$.entries[]:object|$.exists:boolean|$.include:string|$.limits.maxEntries:number|$.limits.maxPayloadChars:number|$.limits.maxScanLines:number|$.limits:object|$.matchedLines:number|$.notes:array|$.notes[]:string|$.path:string|$.scannedLines:number|$.state:string|$:object" + "shape": "$.active:boolean|$.backend:string|$.entries:array|$.entries[].durationMs:number|$.entries[].line:number|$.entries[].method:string|$.entries[].packetId:string|$.entries[].status:number|$.entries[].timestamp:string|$.entries[].url:string|$.entries[]:object|$.include:string|$.limits.maxEntries:number|$.limits.maxPayloadChars:number|$.limits.maxScanLines:number|$.limits:object|$.matchedLines:number|$.notes:array|$.notes[]:string|$.scannedLines:number|$.state:string|$:object" }, "selection.events.default.json": { "bytes": 7686, diff --git a/test/output-economy/output-economy.baseline.test.ts b/test/output-economy/output-economy.baseline.test.ts index 37ee5ddafb..cfb392f075 100644 --- a/test/output-economy/output-economy.baseline.test.ts +++ b/test/output-economy/output-economy.baseline.test.ts @@ -102,35 +102,54 @@ function readBaselineAtRef(ref: string): EconomyBaseline | null { describe('actionability and reliability floors', () => { test('snapshot digest preserves actionable refs and their generation', () => { + expect(rendered.snapshotDigest.refsGeneration).toEqual(expect.any(Number)); + expect(rendered.snapshotDigest.refs).toEqual([ + { ref: 'e2', label: 'Email' }, + { ref: 'e3', label: 'Place order' }, + ]); expect(rendered.snapshotDigest).toMatchObject({ - refsGeneration: 12, - refs: [ - { ref: 'e2', label: 'Email' }, - { ref: 'e3', label: 'Place order' }, - ], + nodeCount: 6, }); }); test('settle digests preserve new targets and removals-only fallback targets', () => { expect(rendered.settleDigest.settle).toMatchObject({ - refsGeneration: 13, refs: [{ ref: 'e4' }, { ref: 'e5' }], }); + expect(rendered.settleDigest.settle.refsGeneration).toEqual(expect.any(Number)); expect(rendered.settleTailDigest.settle).toMatchObject({ - refsGeneration: 14, - tail: [ - { ref: 'e7', role: 'button', label: 'Continue' }, - { ref: 'e9', role: 'tab', label: 'Home' }, - ], + tail: [{ ref: 'e2', role: 'text-field', label: 'qa@example.com' }], }); + expect(rendered.settleTailDigest.settle.refsGeneration).toEqual(expect.any(Number)); + }); + + test('settle generations advance from the corresponding pre-action snapshots', () => { + expect(rendered.settleDefault.settle.refsGeneration).toBeGreaterThan( + rendered.daemonSnapshotDefault.refsGeneration, + ); + expect(rendered.settleTailDefault.settle.refsGeneration).toBeGreaterThan( + rendered.postOrderSnapshotDefault.refsGeneration, + ); + expect(rendered.settleTailDefault.settle.refsGeneration).toBeGreaterThan( + rendered.settleDefault.settle.refsGeneration, + ); + expect(rendered.settleDigest.settle.refsGeneration).toBeGreaterThan( + rendered.snapshotDigestBeforeSettle.refsGeneration, + ); + expect(rendered.settleTailDigest.settle.refsGeneration).toBeGreaterThan( + rendered.postOrderSnapshotDigest.refsGeneration, + ); + expect(rendered.settleTailDigest.settle.refsGeneration).toBeGreaterThan( + rendered.settleDigest.settle.refsGeneration, + ); }); - test('selector digest keeps the answer and recovery warning', () => { - expect(rendered.selectorDigest).toEqual({ - ref: '@e2', + test('selector digest keeps the answer and drops the redundant node', () => { + expect(rendered.selectorDigest).toMatchObject({ + ref: 'e2', text: 'qa@example.com', - warning: 'Recovered from a blocking system dialog', }); + expect(rendered.selectorDigest).not.toHaveProperty('node'); }); test('network digest preserves every entry identity, notes, and follow-up actionability', () => { @@ -200,16 +219,26 @@ describe('baseline-independent actionability floors', () => { expect(settleText).toContain('Order confirmed'); }); - test('screenshot digest keeps overlay refs and the artifact retrieval handle', () => { + test('screenshot digest keeps overlay refs and the screenshot path', () => { expect(rendered.screenshotDigest).toMatchObject({ + path: '/tmp/agent-device/economy-fixture.png', + overlayCount: 2, overlayRefs: [ { ref: 'e2', label: 'Email' }, { ref: 'e3', label: 'Place order' }, ], - artifacts: [{ artifactType: 'screenshot', artifactId: 'artifact-economy-fixture' }], }); }); + test('the screenshot route preserves its artifact retrieval handle', () => { + expect(rendered.screenshotDefault.artifacts).toContainEqual( + expect.objectContaining({ + artifactType: 'screenshot', + artifactId: 'artifact-economy-fixture', + }), + ); + }); + test('non-settled output carries the actual recovery guidance, not just a hint count', () => { const sample = rendered.samples['not-settled.default.text']; const text = 'text' in sample ? sample.text : ''; diff --git a/test/output-economy/output-economy.waivers.json b/test/output-economy/output-economy.waivers.json index 384d2d6e2e..812f7ddec0 100644 --- a/test/output-economy/output-economy.waivers.json +++ b/test/output-economy/output-economy.waivers.json @@ -1,4 +1,16 @@ { + "snapshot.digest.json": { + "bytes": 214, + "reason": "The public daemon route now supplies the live response-view input. Its production six-digit refsGeneration is four bytes larger than the old two-digit static fixture, matching the generation seed that prevents common cross-lifetime ref collisions." + }, + "settle.digest.json": { + "bytes": 274, + "reason": "The public daemon route now supplies the live settled-click response. Its retained targetKind and production six-digit refsGeneration make the measured digest 10 bytes larger than the prior static response fixture." + }, + "screenshot.default.json": { + "bytes": 650, + "reason": "The public screenshot route includes its request-finalized artifact retrieval handle, local path and response message; the prior static response fixture did not include the live route metadata." + }, "settle-tail.default.text": { "bytes": 188, "reason": "ADR 0014: a settled diff's reusable tail refs render in ready-to-copy pinned @eN~s form so a human CLI caller can paste them straight into the next mutation (+8 bytes for the two tail pins). The response-level refsGeneration stays the token-economical authority for JSON and MCP; only human-CLI text carries the pins." diff --git a/test/output-economy/render-fixtures.ts b/test/output-economy/render-fixtures.ts index 64b33d93c4..ae8e79748a 100644 --- a/test/output-economy/render-fixtures.ts +++ b/test/output-economy/render-fixtures.ts @@ -1,16 +1,15 @@ import { snapshotCliOutput } from '../../src/commands/capture/output.ts'; import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts'; import type { AgentDeviceClient } from '../../src/client/client-types.ts'; -import { RESPONSE_VIEWS } from '@agent-device/daemon/testing/response-views'; import { normalizeError } from '@agent-device/kernel/errors'; import { createCommandToolExecutor } from '../../src/mcp/command-tools.ts'; import type { EconomySample } from './economy-metrics.ts'; import { renderRoutineWorkflow } from './routine-workflow.ts'; +import { captureOutputEconomyDaemonResponses } from './daemon-route-fixtures.ts'; import { ACTIONABLE_ERROR, NOT_SETTLED_RESULT, POLICY_NORMALIZED_ERROR, - SCREENSHOT_RESULT, SELECTOR_READ_RESULT, SETTLE_ADDED_REF_RESULT, SETTLE_TAIL_RESULT, @@ -24,21 +23,15 @@ import { RECORDING_SELECTION_RESULT, } from './selection-fixtures.ts'; -function interactionText(result: typeof SETTLE_ADDED_REF_RESULT): string { +function interactionText(result: Record): string { return interactionCliOutputFormatters.press({ input: {}, result }).text ?? ''; } export async function renderOutputFixtures() { + const daemon = await captureOutputEconomyDaemonResponses(); const snapshot = await snapshotCliOutput({ result: SNAPSHOT_RESULT, }); - const snapshotDigest = RESPONSE_VIEWS.snapshot!(SNAPSHOT_DAEMON_RESULT, 'digest'); - const settleDigest = RESPONSE_VIEWS.press!(SETTLE_ADDED_REF_RESULT, 'digest'); - const settleTailDigest = RESPONSE_VIEWS.press!(SETTLE_TAIL_RESULT, 'digest'); - const selectorDigest = RESPONSE_VIEWS.find!(SELECTOR_READ_RESULT, 'digest'); - const screenshotDefault = RESPONSE_VIEWS.screenshot!(SCREENSHOT_RESULT, 'default'); - const screenshotDigest = RESPONSE_VIEWS.screenshot!(SCREENSHOT_RESULT, 'digest'); - const networkDigest = RESPONSE_VIEWS.network!(NETWORK_SELECTION_RESULT, 'digest'); const error = normalizeError(ACTIONABLE_ERROR); const errorPolicyNormalized = normalizeError(POLICY_NORMALIZED_ERROR); const mcpSnapshot = await createCommandToolExecutor({ @@ -49,35 +42,41 @@ export async function renderOutputFixtures() { return { snapshot, - snapshotDigest, - settleDigest, - settleTailDigest, - selectorDigest, - screenshotDefault, - screenshotDigest, - networkDigest, + daemonSnapshotDefault: daemon.snapshotDefault, + snapshotDigest: daemon.snapshotDigest, + settleDigest: daemon.settleDigest, + settleTailDigest: daemon.settleTailDigest, + settleDefault: daemon.settleDefault, + settleTailDefault: daemon.settleTailDefault, + postOrderSnapshotDefault: daemon.postOrderSnapshotDefault, + postOrderSnapshotDigest: daemon.postOrderSnapshotDigest, + snapshotDigestBeforeSettle: daemon.snapshotDigestBeforeSettle, + selectorDigest: daemon.selectorDigest, + screenshotDefault: daemon.screenshotDefault, + screenshotDigest: daemon.screenshotDigest, + networkDigest: daemon.networkDigest, error, errorPolicyNormalized, mcpSnapshot, samples: { 'snapshot.default.text': { text: snapshot.text ?? '' }, 'snapshot.default.json': { data: snapshot.jsonData }, - 'snapshot.digest.json': { data: snapshotDigest }, + 'snapshot.digest.json': { data: daemon.snapshotDigest }, 'settle.default.text': { text: interactionText(SETTLE_ADDED_REF_RESULT) }, 'settle.default.json': { data: SETTLE_ADDED_REF_RESULT }, - 'settle.digest.json': { data: settleDigest }, + 'settle.digest.json': { data: daemon.settleDigest }, 'settle-tail.default.text': { text: interactionText(SETTLE_TAIL_RESULT) }, - 'settle-tail.digest.json': { data: settleTailDigest }, + 'settle-tail.digest.json': { data: daemon.settleTailDigest }, 'not-settled.default.text': { text: interactionText(NOT_SETTLED_RESULT) }, 'selector-read.default.json': { data: SELECTOR_READ_RESULT }, - 'selector-read.digest.json': { data: selectorDigest }, - 'screenshot.default.json': { data: screenshotDefault }, - 'screenshot.digest.json': { data: screenshotDigest }, + 'selector-read.digest.json': { data: daemon.selectorDigest }, + 'screenshot.default.json': { data: daemon.screenshotDefault }, + 'screenshot.digest.json': { data: daemon.screenshotDigest }, 'error.normalized.json': { data: error }, 'error.policy-normalized.json': { data: errorPolicyNormalized }, 'mcp.snapshot.default.json': { data: mcpSnapshot }, 'selection.network.default.json': { data: NETWORK_SELECTION_RESULT }, - 'selection.network.digest.json': { data: networkDigest }, + 'selection.network.digest.json': { data: daemon.networkDigest }, 'selection.events.default.json': { data: EVENTS_SELECTION_RESULT }, 'selection.debug.default.json': { data: DEBUG_SELECTION_RESULT }, 'selection.recording.default.json': { data: RECORDING_SELECTION_RESULT },