From 5495cfab3ad4a3b8efbfe94144aad333e4df7d94 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:10:57 +0530 Subject: [PATCH 1/9] fix(daemon): start a lease's TTL when its allocation completes The registry records a lease, and stamps its expiry, before the lease lifecycle provider allocates the session behind it. Hosted providers can spend 30-80 seconds creating that session, so a lease on the default one-minute inactivity window was expired, or nearly so, by the time its client received it: the next command failed with "Lease is not active", release reported nothing to release, and the paid provider session was left running. The expiry sweep could also reap the lease while the provider was still allocating. Hold a lease work pass for the duration of the provider allocation, the same protection admitted request work gets. The lease cannot expire underneath the allocation, and ending the pass while the requester is still waiting renews the lease for its own TTL from that moment. The response now carries the renewed lease. A requester that hung up still protects nothing, and its allocation is released as before. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0007-remote-device-leases.md | 10 +++ .../__tests__/lease-artifacts.test.ts | 57 +++++++++++-- src/daemon/handlers/__tests__/lease.test.ts | 79 +++++++++++++++++++ src/daemon/handlers/lease.ts | 13 ++- website/docs/docs/remote-proxy.md | 2 +- 5 files changed, 151 insertions(+), 10 deletions(-) diff --git a/docs/adr/0007-remote-device-leases.md b/docs/adr/0007-remote-device-leases.md index d64f08b6ae..d386648e08 100644 --- a/docs/adr/0007-remote-device-leases.md +++ b/docs/adr/0007-remote-device-leases.md @@ -67,6 +67,16 @@ one minute, while a cloud WebDriver connection profile asks for ten. A single co longer than its own lease is therefore ordinary on the default and only reachable through a profile on the longer one. +## Provider allocation + +Allocation is admitted work too. The registry records a lease before a hosted provider creates the +session behind it, and creating that session can take longer than the lease's inactivity TTL, so a +lease timed from its record was already expired, or nearly so, when its client first received it, and +the paid session it pointed at was orphaned. The allocation therefore holds a work pass while the +provider allocates: the lease cannot expire underneath it, and a successful allocation still wanted by +its requester starts the inactivity TTL from the moment allocation completed. A requester that hung up +preserves nothing, and its allocation is released as before. + ## Client-side work that precedes admission Protecting admitted work covers nothing that happens before a request is admitted. Installing an diff --git a/src/daemon/handlers/__tests__/lease-artifacts.test.ts b/src/daemon/handlers/__tests__/lease-artifacts.test.ts index 8067a4fdae..a6fd63a376 100644 --- a/src/daemon/handlers/__tests__/lease-artifacts.test.ts +++ b/src/daemon/handlers/__tests__/lease-artifacts.test.ts @@ -2,7 +2,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { CloudArtifactsQuery } from '@agent-device/contracts/observability'; import type { DeviceLease } from '@agent-device/contracts/device'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import { clearRequestCanceled, markRequestCanceled } from '@agent-device/host-kit/request'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; import { handleLeaseCommands } from '../lease.ts'; @@ -117,7 +118,7 @@ test('artifacts refuses an expired provider session after retention before lazy assert.deepEqual(world.providerCalls, []); }); -test('artifacts refuses a provider session returned after allocation expiry retention', async () => { +test('artifacts lists a provider session whose allocation outlasted the lease TTL', async () => { let now = 1_000; const world = createWorld({ now: () => now, @@ -128,16 +129,60 @@ test('artifacts refuses a provider session returned after allocation expiry rete }); world.lifecycle.allocate = async (lease) => { now = lease.expiresAt + 51; - return { providerSessionId: 'late-allocation-session' }; + return { providerSessionId: 'slow-allocation-session' }; }; await allocateLease(world, 'tenant-a', 'run-a'); - await assertProviderSessionNotOwned(world, { + const listed = await listArtifacts(world, { tenantId: 'tenant-a', runId: 'run-a', - providerSessionId: 'late-allocation-session', + providerSessionId: 'slow-allocation-session', }); - assert.deepEqual(world.providerCalls, []); + assert.equal(listed.ok, true); +}); + +test('artifacts refuses a provider session returned after a canceled allocation outlived retention', async () => { + let now = 1_000; + const world = createWorld({ + now: () => now, + defaultLeaseTtlMs: 100, + minLeaseTtlMs: 1, + maxLeaseTtlMs: 100, + providerSessionRetentionMs: 50, + }); + const requestId = 'late-canceled-allocation'; + world.lifecycle.allocate = async (lease) => { + markRequestCanceled(requestId); + now = lease.expiresAt + 51; + return { providerSessionId: 'late-allocation-session' }; + }; + + try { + await assert.rejects( + handleLeaseCommands({ + req: leaseRequest('lease_allocate', { + requestId, + tenantId: 'tenant-a', + runId: 'run-a', + leaseBackend: 'android-instance', + leaseProvider: CLOUD_PROVIDER, + }), + sessionName: 'artifact-test', + sessionStore: world.sessionStore, + leaseRegistry: world.leaseRegistry, + leaseLifecycleProvider: world.lifecycle, + }), + isRequestCanceledError, + ); + await assertProviderSessionNotOwned(world, { + tenantId: 'tenant-a', + runId: 'run-a', + providerSessionId: 'late-allocation-session', + }); + assert.deepEqual(world.providerCalls, []); + } finally { + clearRequestCanceled(requestId); + } }); test('artifacts refuses a provider session returned after release expiry retention', async () => { diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts index 08fdb22c7f..5cbd378ea6 100644 --- a/src/daemon/handlers/__tests__/lease.test.ts +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -2,7 +2,9 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { handleLeaseCommands } from '../lease.ts'; import { LeaseRegistry } from '../../lease-registry.ts'; +import type { DaemonRequest } from '../../daemon-request.ts'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; import { clearRequestCanceled, markRequestCanceled } from '@agent-device/host-kit/request'; import { @@ -93,3 +95,80 @@ test('activation drains canceled provider allocation and its release cleanup', a clearRequestCanceled(requestId); } }); + +function allocateRequest(): DaemonRequest { + return { + token: 'test-token', + session: 'lease-ttl-test', + command: 'lease_allocate', + positionals: [], + flags: {}, + meta: { + tenantId: 'tenant-a', + runId: 'run-a', + clientId: 'client-a', + leaseBackend: 'android-instance', + leaseProvider: 'cloud', + }, + }; +} + +// A hosted provider can spend longer creating its session than the lease's inactivity +// TTL. Stamping the TTL when the registry record is created handed the client a lease +// that was already expired, and the paid session behind it was orphaned. +test('a lease whose provider allocation outlasts its TTL is active when allocation returns', async () => { + let now = 0; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 60_000 }); + const response = await handleLeaseCommands({ + req: allocateRequest(), + sessionName: 'lease-ttl-test', + sessionStore: makeSessionStore('agent-device-slow-provider-'), + leaseRegistry: registry, + leaseLifecycleProvider: { + allocate: async (lease) => { + now = 80_000; + assert.deepEqual( + registry.consumeExpiredLeases(), + [], + 'the sweeper must not reap a lease mid-allocation', + ); + now = 90_000; + return { providerSessionId: `session-${lease.leaseId}` }; + }, + }, + }); + + assert.equal(response?.ok, true); + const lease = (response?.ok ? response.data?.lease : undefined) as DeviceLease; + assert.equal(lease.expiresAt, 150_000); + assert.deepEqual( + registry.listActiveLeases().map((entry) => [entry.leaseId, entry.expiresAt]), + [[lease.leaseId, 150_000]], + ); + now = 149_999; + registry.assertLeaseAdmission({ + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + leaseProvider: lease.leaseProvider, + }); +}); + +test('a lease allocated without a provider keeps the TTL it was created with', async () => { + const now = 5_000; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 60_000 }); + const response = await handleLeaseCommands({ + req: allocateRequest(), + sessionName: 'lease-ttl-test', + sessionStore: makeSessionStore('agent-device-no-provider-'), + leaseRegistry: registry, + }); + + assert.equal(response?.ok, true); + const lease = (response?.ok ? response.data?.lease : undefined) as DeviceLease; + assert.equal(lease.createdAt, 5_000); + assert.equal(lease.heartbeatAt, 5_000); + assert.equal(lease.expiresAt, 65_000); + assert.deepEqual(registry.listActiveLeases(), [lease]); +}); diff --git a/src/daemon/handlers/lease.ts b/src/daemon/handlers/lease.ts index fa597823e2..f0826bb87f 100644 --- a/src/daemon/handlers/lease.ts +++ b/src/daemon/handlers/lease.ts @@ -71,20 +71,26 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise { let providerData: Record | undefined; + // A hosted provider can take longer than the lease TTL to create its session; the work + // pass keeps the lease alive until it does, and ending the pass restarts the TTL then. + const work = leaseRegistry.retainLeaseWork(lease, () => !isRequestCanceled(requestId)); try { providerData = await leaseLifecycleProvider?.allocate?.(lease, { ...leaseLifecycleContext(req), - signal: getRequestSignal(req.meta?.requestId), + signal: getRequestSignal(requestId), deadline: Date.now() + LEASE_ALLOCATION_BUDGET_MS, }); recordProviderSession(leaseRegistry, lease, providerData); } catch (error) { leaseRegistry.releaseLease(leaseReleaseRequestFor(lease)); throw error; + } finally { + work.release(); } - if (isRequestCanceled(req.meta?.requestId)) { + if (isRequestCanceled(requestId)) { // The requester left while the provider was allocating; the lease it // produced is real (and billed) and nobody will ever release it. throw await releaseAllocationForGoneRequester( @@ -93,9 +99,10 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise` instead of exporting the environment variable also works, but only authenticates the single command it is passed to; subsequent commands need the token again through the env var, a `daemonAuthToken` entry in your remote config profile, or a repeated `--daemon-auth-token` flag. -`connect proxy` stores the proxy profile and client identity. Device leases are automatic on `open` and expire after five minutes without commands. That five minutes is the window `open` asks for; a lease allocated directly over the RPC without `ttlMs` keeps the daemon's one-minute inactivity default instead. `close` releases the active session and device lease; `disconnect` clears local connection state. +`connect proxy` stores the proxy profile and client identity. Device leases are automatic on `open` and expire after five minutes without commands. That five minutes is the window `open` asks for; a lease allocated directly over the RPC without `ttlMs` keeps the daemon's one-minute inactivity default instead. Either window starts when allocation completes, not when it was requested. `close` releases the active session and device lease; `disconnect` clears local connection state. Multiple agents can share one proxy when each uses the normal `connect proxy`, `open`, commands, `close`, and `disconnect` flow. A busy device error means another agent owns the device until it closes or its inactivity lease expires. From 313a7d71aebee3b745278120515ae12229d35d0a Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:25:12 +0530 Subject: [PATCH 2/9] refactor(provider-webdriver): declare the profile fields each provider consumes Lease allocation forwards every provider profile field, but only some routes checked which ones a provider could act on. The connect builders and the BrowserStack/AWS session preparation refused flags they did not own, while the Limrun lease runtime and a remote-config profile skipped that check. A typed client asking Limrun for providerDeviceType 'real' was therefore given a simulator without an error. Each lease provider now declares, for every Cloud provider profile field, whether it consumes or refuses it. The declaration is a total record over CloudProviderProfileFields, so adding a field fails to build until every provider decides. One refusal, derived from that declaration, runs at lease preparation (the WebDriver session manager for BrowserStack, AWS Device Farm and TestMu AI, and Limrun allocation) and at connect, so connect, leases.allocate and remote-config profiles all fail the same way with INVALID_ARGS naming the flag and the provider. This replaces the per-provider TestMu-only, TestMu-unsupported and BrowserStack-only checks, the hand-copied complement of the BrowserStack feature table, and the ./testmu-device-features package subpath. AWS Device Farm now also refuses the hub-only app, OS version, project and build flags it never read. Co-Authored-By: Claude Opus 5.5 --- packages/contracts/package.json | 4 + .../src/provider-profile-fields.test.ts | 73 ++++++++++ .../contracts/src/provider-profile-fields.ts | 66 +++++++++ packages/provider-limrun/src/device.ts | 29 ++++ packages/provider-limrun/src/index.ts | 2 +- packages/provider-limrun/src/runtime.ts | 2 + .../provider-limrun/src/session-allocation.ts | 7 +- packages/provider-webdriver/package.json | 4 - .../src/browserstack-device-features.ts | 41 ------ packages/provider-webdriver/src/index.ts | 4 +- .../src/provider-definitions.ts | 117 +++++++++++++--- .../src/provider-profile-fields.test.ts | 56 ++++++++ .../provider-webdriver/src/runtime-session.ts | 6 + packages/provider-webdriver/src/runtime.ts | 3 + .../src/testmu-device-features.test.ts | 67 +--------- .../src/testmu-device-features.ts | 56 -------- .../layering/contracts-exports.snapshot.json | 1 + scripts/layering/package-boundaries.test.ts | 1 - src/__tests__/cloud-connect-profile.test.ts | 4 +- src/__tests__/cloud-connect-testmu.test.ts | 25 +++- .../lease-provider-profile-fields.test.ts | 126 ++++++++++++++++++ src/cli/connection/cloud-webdriver-profile.ts | 18 ++- src/cli/connection/limrun-profile.ts | 5 +- .../cloud-webdriver-provider-adapters.test.ts | 8 +- website/docs/docs/device-clouds.md | 2 + website/docs/docs/testmu.md | 3 +- 26 files changed, 514 insertions(+), 216 deletions(-) create mode 100644 packages/contracts/src/provider-profile-fields.test.ts create mode 100644 packages/contracts/src/provider-profile-fields.ts create mode 100644 packages/provider-webdriver/src/provider-profile-fields.test.ts create mode 100644 src/__tests__/lease-provider-profile-fields.test.ts diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 328cdce13a..bee7b0f6f9 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -364,6 +364,10 @@ "types": "./src/runtime-operation-names.ts", "default": "./src/runtime-operation-names.ts" }, + "./provider-profile-fields": { + "types": "./src/provider-profile-fields.ts", + "default": "./src/provider-profile-fields.ts" + }, "./progress": { "types": "./src/facades/progress.ts", "default": "./src/facades/progress.ts" diff --git a/packages/contracts/src/provider-profile-fields.test.ts b/packages/contracts/src/provider-profile-fields.test.ts new file mode 100644 index 0000000000..000d72a020 --- /dev/null +++ b/packages/contracts/src/provider-profile-fields.test.ts @@ -0,0 +1,73 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { + rejectRefusedProviderProfileFields, + type ProviderProfileFieldDeclaration, +} from './provider-profile-fields.ts'; + +const DECLARATION: ProviderProfileFieldDeclaration = { + provider: 'fake', + label: 'Fake Cloud', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'refused', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'refused', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'refused', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', + }, +}; + +test('consumed, unset, empty, and false fields pass', () => { + assert.doesNotThrow(() => rejectRefusedProviderProfileFields(undefined, DECLARATION)); + assert.doesNotThrow(() => + rejectRefusedProviderProfileFields( + { + providerApp: 'app', + providerDeviceType: '', + providerGeoLocation: undefined, + providerNoResignApp: false, + unrelated: 'x', + }, + DECLARATION, + ), + ); +}); + +test('a refused field fails with its flag and the provider named', () => { + assert.throws( + () => rejectRefusedProviderProfileFields({ providerDeviceType: 'real' }, DECLARATION), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === '--provider-device-type is not supported by Fake Cloud.' && + error.details?.provider === 'fake' && + JSON.stringify(error.details?.flags) === '["--provider-device-type"]', + ); +}); + +test('every refused field is reported at once', () => { + assert.throws( + () => + rejectRefusedProviderProfileFields( + { providerGeoLocation: 'US', providerNoResignApp: true, providerDeviceType: 'virtual' }, + DECLARATION, + ), + /--provider-device-type, --provider-geo-location, --provider-no-resign-app are not supported by Fake Cloud\./, + ); +}); diff --git a/packages/contracts/src/provider-profile-fields.ts b/packages/contracts/src/provider-profile-fields.ts new file mode 100644 index 0000000000..800dd145a8 --- /dev/null +++ b/packages/contracts/src/provider-profile-fields.ts @@ -0,0 +1,66 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { CloudProviderProfileFields } from './remote-config-fields.ts'; + +export type ProviderProfileField = keyof CloudProviderProfileFields; + +/** + * What one lease provider does with every Cloud provider profile field. The record is total, so a + * field added to the profile fails to build until each provider says whether it consumes it; a + * field a provider neither reads nor refuses would otherwise ride the profile and be dropped. + */ +export type ProviderProfileFieldDeclaration = Readonly<{ + provider: string; + label: string; + fields: Readonly>; +}>; + +const PROVIDER_PROFILE_FIELD_FLAGS: Readonly> = { + providerApp: '--provider-app', + providerOsVersion: '--provider-os-version', + providerDeviceType: '--provider-device-type', + providerProject: '--provider-project', + providerBuild: '--provider-build', + providerSessionName: '--provider-session-name', + providerDeviceOrientation: '--provider-device-orientation', + providerGeoLocation: '--provider-geo-location', + providerTimezone: '--provider-timezone', + providerAppiumVersion: '--provider-appium-version', + providerLanguage: '--provider-language', + providerLocale: '--provider-locale', + providerNetworkProfile: '--provider-network-profile', + providerCustomNetwork: '--provider-custom-network', + providerNoResignApp: '--provider-no-resign-app', + awsProjectArn: '--aws-project-arn', + awsDeviceArn: '--aws-device-arn', + awsAppArn: '--aws-app-arn', + awsRegion: '--aws-region', + awsInteractionMode: '--aws-interaction-mode', +}; + +/** + * Fails when `flags` set a profile field the provider refuses. Every route to a provider — connect, + * `leases.allocate`, a hand-authored remote-config profile — runs this against the same declaration. + */ +export function rejectRefusedProviderProfileFields( + flags: Readonly> | undefined, + declaration: ProviderProfileFieldDeclaration, +): void { + const refused = (Object.keys(declaration.fields) as ProviderProfileField[]) + .filter((field) => declaration.fields[field] === 'refused' && isSet(flags?.[field])) + .map((field) => PROVIDER_PROFILE_FIELD_FLAGS[field]); + if (refused.length === 0) return; + const plural = refused.length !== 1; + throw new AppError( + 'INVALID_ARGS', + `${refused.join(', ')} ${plural ? 'are' : 'is'} not supported by ${declaration.label}.`, + { + hint: `Drop ${plural ? 'those flags' : 'the flag'} or use a provider that supports ${plural ? 'them' : 'it'}.`, + provider: declaration.provider, + flags: refused, + }, + ); +} + +function isSet(value: unknown): boolean { + return value !== undefined && value !== null && value !== false && value !== ''; +} diff --git a/packages/provider-limrun/src/device.ts b/packages/provider-limrun/src/device.ts index a61e41b3e2..2d19e5e59b 100644 --- a/packages/provider-limrun/src/device.ts +++ b/packages/provider-limrun/src/device.ts @@ -1,10 +1,39 @@ import type { DeviceLease } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; export type LimrunPlatform = 'ios' | 'android'; export const LIMRUN_PROVIDER = 'limrun'; +/** Limrun reads only the app to preinstall; it picks the instance itself. */ +export const LIMRUN_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: LIMRUN_PROVIDER, + label: 'Limrun', + fields: { + providerApp: 'consumed', + providerOsVersion: 'refused', + providerDeviceType: 'refused', + providerProject: 'refused', + providerBuild: 'refused', + providerSessionName: 'refused', + providerDeviceOrientation: 'refused', + providerGeoLocation: 'refused', + providerTimezone: 'refused', + providerAppiumVersion: 'refused', + providerLanguage: 'refused', + providerLocale: 'refused', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + awsProjectArn: 'refused', + awsDeviceArn: 'refused', + awsAppArn: 'refused', + awsRegion: 'refused', + awsInteractionMode: 'refused', + }, +}; + const LIMRUN_DEVICE_ID_PREFIX = LIMRUN_PROVIDER; export function platformForLimrunLeaseBackend(backend: string): LimrunPlatform | undefined { diff --git a/packages/provider-limrun/src/index.ts b/packages/provider-limrun/src/index.ts index d29e531c0c..f3ee00a35e 100644 --- a/packages/provider-limrun/src/index.ts +++ b/packages/provider-limrun/src/index.ts @@ -1,4 +1,4 @@ -export { LIMRUN_PROVIDER } from './device.ts'; +export { LIMRUN_PROFILE_FIELDS, LIMRUN_PROVIDER } from './device.ts'; export { createLimrunRuntime, type LimrunRuntime, type LimrunRuntimeOptions } from './runtime.ts'; export { verifyLimrunConnection } from './connection-verification.ts'; diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index d3528ea1b1..f1fb902b1e 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -237,9 +237,11 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { const { allocateLimrunAndroidSession, allocateLimrunIosSession, + rejectRefusedLimrunProfileFields, resolvePreinstalledAppId, resolveRequestedLimrunAppAsset, } = await import('./session-allocation.ts'); + rejectRefusedLimrunProfileFields(context); const requestedAsset = await resolveRequestedLimrunAppAsset(this.limrun, platform, context); const session = platform === 'ios' diff --git a/packages/provider-limrun/src/session-allocation.ts b/packages/provider-limrun/src/session-allocation.ts index de83a71366..a78bb4edd2 100644 --- a/packages/provider-limrun/src/session-allocation.ts +++ b/packages/provider-limrun/src/session-allocation.ts @@ -1,8 +1,9 @@ import type Limrun from '@limrun/api'; import type { DeviceLease, LeaseLifecycleContext } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import { createLimrunAndroidSession, type LimrunAndroidSession } from './android.ts'; -import { buildLimrunDevice } from './device.ts'; +import { buildLimrunDevice, LIMRUN_PROFILE_FIELDS } from './device.ts'; import { createLimrunIosSession, type LimrunIosSession } from './ios.ts'; import { assertLimrunUploadedAppAccess, @@ -34,6 +35,10 @@ type SessionAllocationParams = Readonly<{ dependencies: LimrunRuntimeDependencies; }>; +export function rejectRefusedLimrunProfileFields(context?: LeaseLifecycleContext): void { + rejectRefusedProviderProfileFields(context?.flags, LIMRUN_PROFILE_FIELDS); +} + export async function resolveRequestedLimrunAppAsset( limrun: Limrun, platform: 'android' | 'ios', diff --git a/packages/provider-webdriver/package.json b/packages/provider-webdriver/package.json index a9f01ebacb..d4d56b8d0e 100644 --- a/packages/provider-webdriver/package.json +++ b/packages/provider-webdriver/package.json @@ -19,10 +19,6 @@ "./providers": { "types": "./src/providers.ts", "default": "./src/providers.ts" - }, - "./testmu-device-features": { - "types": "./src/testmu-device-features.ts", - "default": "./src/testmu-device-features.ts" } } } diff --git a/packages/provider-webdriver/src/browserstack-device-features.ts b/packages/provider-webdriver/src/browserstack-device-features.ts index 3ea041add7..275ae64ff3 100644 --- a/packages/provider-webdriver/src/browserstack-device-features.ts +++ b/packages/provider-webdriver/src/browserstack-device-features.ts @@ -130,47 +130,6 @@ export function buildBrowserStackDeviceFeatureCapabilities( return capabilities; } -/** - * Canonical CLI flags for every device-feature capability set on `flags`. - * - * These capabilities are BrowserStack-owned. Other providers have no equivalent, so a caller who - * passes them to AWS Device Farm would otherwise have them accepted, persisted into the profile, - * and then silently dropped — the session runs with provider defaults and nothing says why. - * Callers use this to reject them at the point the provider is known. - */ -function browserStackOnlyDeviceFeatureFlags(flags: Record | undefined): string[] { - return BROWSERSTACK_DEVICE_FEATURE_SPECS.filter((spec) => { - const value = flags?.[spec.field]; - return value !== undefined && value !== false && value !== ''; - }).map((spec) => spec.flag); -} - -/** - * Fails when a non-BrowserStack provider was given BrowserStack-owned device features. - * - * Called from both the CLI profile builder and the provider's own session preparation. The second - * is the one that actually closes the hole: the typed client and hand-authored remote-config - * profiles reach session preparation without passing through `connect`, so a CLI-only check leaves - * those routes accepting the capabilities and dropping them. - */ -export function rejectBrowserStackOnlyDeviceFeatures( - flags: Record | undefined, - provider: string, -): void { - const configured = browserStackOnlyDeviceFeatureFlags(flags); - if (configured.length === 0) return; - const plural = configured.length !== 1; - throw new AppError( - 'INVALID_ARGS', - `${configured.join(', ')} ${plural ? 'are' : 'is'} only supported by BrowserStack, not ${provider}.`, - { - hint: `Drop ${plural ? 'those flags' : 'the flag'} or use the browserstack provider.`, - provider, - flags: configured, - }, - ); -} - /** * Reads device-feature fields off an untyped flag bag (a daemon request), so the daemon-side * capability build and the CLI-side profile build stay driven by the same table. diff --git a/packages/provider-webdriver/src/index.ts b/packages/provider-webdriver/src/index.ts index cba755ce82..12c30acf60 100644 --- a/packages/provider-webdriver/src/index.ts +++ b/packages/provider-webdriver/src/index.ts @@ -4,6 +4,7 @@ import type { } from '@agent-device/contracts/observability'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; import { + CLOUD_WEBDRIVER_PROFILE_FIELDS, createCloudWebDriverProviderDefinitions, type DefaultCloudWebDriverArtifactEnv, type DefaultCloudWebDriverProviderRuntimeEnv, @@ -17,9 +18,8 @@ import { } from './connection-verification.ts'; import type { CloudWebDriverRuntime } from './runtime.ts'; -export { CLOUD_WEBDRIVER_PROVIDERS }; +export { CLOUD_WEBDRIVER_PROFILE_FIELDS, CLOUD_WEBDRIVER_PROVIDERS }; export { readAwsDeviceFarmRegionFromArn }; -export { rejectBrowserStackOnlyDeviceFeatures } from './browserstack-device-features.ts'; export type { CloudWebDriverKnownProviderName } from './providers.ts'; export type { ProviderWebDriverDependencies, RunHostCommand } from './dependencies.ts'; export type { diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index dbc2092f7a..6981043fc5 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,6 +1,7 @@ import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; import type { ProviderDeviceType } from '@agent-device/contracts/remote'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; import { @@ -21,7 +22,6 @@ import { import { buildBrowserStackDeviceFeatureCapabilities, readBrowserStackDeviceFeatureFields, - rejectBrowserStackOnlyDeviceFeatures, } from './browserstack-device-features.ts'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import { CLOUD_WEBDRIVER_PROVIDERS, type CloudWebDriverKnownProviderName } from './providers.ts'; @@ -82,8 +82,100 @@ const TESTMU_CAPABILITY_OVERRIDES = { const loadTestMu = async () => await import('./testmu.ts'); const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); +const AWS_DEVICE_FARM_FIELDS_REFUSED = { + awsProjectArn: 'refused', + awsDeviceArn: 'refused', + awsAppArn: 'refused', + awsRegion: 'refused', + awsInteractionMode: 'refused', +} as const; + +const BROWSERSTACK_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + label: 'BrowserStack', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'refused', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'consumed', + ...AWS_DEVICE_FARM_FIELDS_REFUSED, + }, +}; + +const AWS_DEVICE_FARM_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + label: 'AWS Device Farm', + fields: { + providerApp: 'refused', + providerOsVersion: 'refused', + providerDeviceType: 'refused', + providerProject: 'refused', + providerBuild: 'refused', + providerSessionName: 'consumed', + providerDeviceOrientation: 'refused', + providerGeoLocation: 'refused', + providerTimezone: 'refused', + providerAppiumVersion: 'refused', + providerLanguage: 'refused', + providerLocale: 'refused', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', + }, +}; + +const TESTMU_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + label: 'TestMu AI', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'consumed', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + ...AWS_DEVICE_FARM_FIELDS_REFUSED, + }, +}; + +/** The profile fields each hub provider reads, for routes that check them before a runtime exists. */ +export const CLOUD_WEBDRIVER_PROFILE_FIELDS: Readonly< + Record +> = { + [CLOUD_WEBDRIVER_PROVIDERS.browserStack]: BROWSERSTACK_PROFILE_FIELDS, + [CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm]: AWS_DEVICE_FARM_PROFILE_FIELDS, + [CLOUD_WEBDRIVER_PROVIDERS.testMu]: TESTMU_PROFILE_FIELDS, +}; + export type CloudWebDriverProviderDefinition = { provider: CloudWebDriverKnownProviderName; + /** Every profile field, consumed or refused; session preparation refuses the refused ones. */ + profileFields: ProviderProfileFieldDeclaration; createRuntime: (env: DefaultCloudWebDriverProviderRuntimeEnv) => CloudWebDriverRuntime; listArtifactsFromEnv: ( providerSessionId: string, @@ -97,10 +189,12 @@ export function createCloudWebDriverProviderDefinitions( return [ { provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + profileFields: BROWSERSTACK_PROFILE_FIELDS, createRuntime: (env) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + profileFields: BROWSERSTACK_PROFILE_FIELDS, platform: 'android', deviceName: 'BrowserStack device', endpoint: env.BROWSERSTACK_WEBDRIVER_ENDPOINT ?? BROWSERSTACK_APP_AUTOMATE_ENDPOINT, @@ -125,10 +219,6 @@ export function createCloudWebDriverProviderDefinitions( }, prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'BrowserStack'); - (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( - request.flags, - CLOUD_WEBDRIVER_PROVIDERS.browserStack, - ); const username = requireEnv(env, 'BROWSERSTACK_USERNAME', 'BrowserStack'); const accessKey = requireEnv(env, 'BROWSERSTACK_ACCESS_KEY', 'BrowserStack'); const platform = requireRequestPlatform(request, 'BrowserStack'); @@ -208,10 +298,12 @@ export function createCloudWebDriverProviderDefinitions( }, { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, createRuntime: (env) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, endpoint: 'http://127.0.0.1/', platform: 'android', deviceName: 'AWS Device Farm device', @@ -228,17 +320,6 @@ export function createCloudWebDriverProviderDefinitions( }, prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'AWS Device Farm'); - // Enforced here, not only in the CLI profile builder: the typed client and - // hand-authored remote-config profiles both reach session preparation without passing - // through `connect`, and would otherwise have these capabilities silently dropped. - rejectBrowserStackOnlyDeviceFeatures( - request.flags, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - ); - (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( - request.flags, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - ); const platform = requireRequestPlatform(request, 'AWS Device Farm'); const sessionOptions = { client: createAwsCliDeviceFarmClient({ @@ -288,10 +369,12 @@ export function createCloudWebDriverProviderDefinitions( }, { provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + profileFields: TESTMU_PROFILE_FIELDS, createRuntime: (env) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + profileFields: TESTMU_PROFILE_FIELDS, platform: 'android', deviceName: 'TestMu AI device', endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, @@ -306,9 +389,7 @@ export function createCloudWebDriverProviderDefinitions( buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, readTestMuDeviceType, - rejectUnsupportedTestMuDeviceFeatures, } = await loadTestMuDeviceFeatures(); - rejectUnsupportedTestMuDeviceFeatures(request.flags); const deviceType = readTestMuDeviceType(request.flags); const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); const credentials = requireTestMuCredentials(env, 'TestMu AI'); diff --git a/packages/provider-webdriver/src/provider-profile-fields.test.ts b/packages/provider-webdriver/src/provider-profile-fields.test.ts new file mode 100644 index 0000000000..84f3221ab3 --- /dev/null +++ b/packages/provider-webdriver/src/provider-profile-fields.test.ts @@ -0,0 +1,56 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { ProviderProfileField } from '@agent-device/contracts/provider-profile-fields'; +import { CLOUD_WEBDRIVER_PROFILE_FIELDS } from './provider-definitions.ts'; +import { CLOUD_WEBDRIVER_PROVIDERS } from './providers.ts'; +import { BROWSERSTACK_DEVICE_FEATURE_SPECS } from './browserstack-device-features.ts'; +import { TESTMU_DEVICE_FEATURE_SPECS } from './testmu-device-features.ts'; + +// Fields a hub reads directly while building its session, outside the device-feature tables. +const HUB_SESSION_FIELDS: readonly ProviderProfileField[] = [ + 'providerApp', + 'providerOsVersion', + 'providerDeviceType', + 'providerProject', + 'providerBuild', + 'providerSessionName', +]; + +function consumedFields(provider: keyof typeof CLOUD_WEBDRIVER_PROFILE_FIELDS): string[] { + const { fields } = CLOUD_WEBDRIVER_PROFILE_FIELDS[provider]; + return (Object.keys(fields) as ProviderProfileField[]) + .filter((field) => fields[field] === 'consumed') + .sort(); +} + +test('every declaration names the provider it is registered under', () => { + for (const [provider, declaration] of Object.entries(CLOUD_WEBDRIVER_PROFILE_FIELDS)) { + assert.equal(declaration.provider, provider); + } +}); + +// A consumed device feature with no capability row would be accepted and then dropped at the hub. +test('hub declarations consume exactly the fields their capability builders read', () => { + assert.deepEqual( + consumedFields(CLOUD_WEBDRIVER_PROVIDERS.browserStack), + [ + ...HUB_SESSION_FIELDS.filter((field) => field !== 'providerDeviceType'), + ...BROWSERSTACK_DEVICE_FEATURE_SPECS.map((spec) => spec.field), + ].sort(), + ); + assert.deepEqual( + consumedFields(CLOUD_WEBDRIVER_PROVIDERS.testMu), + [...HUB_SESSION_FIELDS, ...TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field)].sort(), + ); +}); + +test('AWS Device Farm consumes only its own fields and the session name', () => { + assert.deepEqual(consumedFields(CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm), [ + 'awsAppArn', + 'awsDeviceArn', + 'awsInteractionMode', + 'awsProjectArn', + 'awsRegion', + 'providerSessionName', + ]); +}); diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index cce3f7ca86..293aa51b0d 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -203,6 +203,12 @@ export class WebDriverSessionManager { lease: DeviceLease, req: LeaseLifecycleContext | undefined, ): Promise { + if (this.options.profileFields) { + // Loaded on first allocation so the package entry's eager closure stays unchanged. + const { rejectRefusedProviderProfileFields } = + await import('@agent-device/contracts/provider-profile-fields'); + rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); + } const base = this.baseSessionForLease(lease); return this.options.prepareSession ? await this.options.prepareSession({ lease, req, base }) diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 3ee5aab125..146cb89787 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -18,6 +18,7 @@ import type { PlatformRuntimeProviderModule, } from '@agent-device/contracts/platform-runtime-operations'; import { providerRuntimeOwner } from '@agent-device/contracts/platform-runtime'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { createCloudWebDriverCapabilities, @@ -100,6 +101,8 @@ export type CloudWebDriverRuntimeOptions = { deviceId?: (lease: DeviceLease) => string; prepareSession?: CloudWebDriverPrepareSession; capabilityOverrides?: CloudWebDriverCapabilityOverrides; + /** Profile fields this provider reads; any field it refuses fails session preparation. */ + profileFields?: ProviderProfileFieldDeclaration; }; export function createCloudWebDriverRuntime( diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-webdriver/src/testmu-device-features.test.ts index 33eb5afd49..2faba2e427 100644 --- a/packages/provider-webdriver/src/testmu-device-features.test.ts +++ b/packages/provider-webdriver/src/testmu-device-features.test.ts @@ -7,29 +7,9 @@ import { buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, readTestMuDeviceType, - rejectTestMuOnlyProviderFlags, - rejectUnsupportedTestMuDeviceFeatures, } from './testmu-device-features.ts'; -// Every hosted-provider device-feature field is either a TestMu spec row or an explicit rejection: -// a field in neither parses off the CLI, rides the profile, and is silently dropped at the hub. -const SUPPORTED_FIELDS = [ - 'providerDeviceOrientation', - 'providerGeoLocation', - 'providerTimezone', - 'providerAppiumVersion', - 'providerLanguage', - 'providerLocale', -] as const; -const REJECTED_FIELDS = [ - 'providerNetworkProfile', - 'providerCustomNetwork', - 'providerNoResignApp', -] as const; - -test('every supported device-feature field maps to exactly one lt:options key', () => { - const fields = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field); - assert.deepEqual([...fields].sort(), [...SUPPORTED_FIELDS].sort()); +test('every device-feature spec maps to exactly one lt:options key', () => { const capabilities = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.capability); assert.equal(new Set(capabilities).size, capabilities.length); }); @@ -77,33 +57,6 @@ test('daemon flag bags are read through the same table with orientation validate ); }); -test('BrowserStack-only flags are rejected by flag name instead of being dropped', () => { - assert.doesNotThrow(() => rejectUnsupportedTestMuDeviceFeatures(undefined)); - assert.doesNotThrow(() => - rejectUnsupportedTestMuDeviceFeatures({ - providerGeoLocation: 'US', - providerNoResignApp: false, - }), - ); - for (const field of REJECTED_FIELDS) { - assert.throws( - () => - rejectUnsupportedTestMuDeviceFeatures({ - [field]: field === 'providerNoResignApp' ? true : 'x', - }), - (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', - ); - } - assert.throws( - () => - rejectUnsupportedTestMuDeviceFeatures({ - providerNetworkProfile: '4g-lte-good', - providerCustomNetwork: '1000', - }), - /--provider-network-profile, --provider-custom-network are not supported by TestMu AI/, - ); -}); - test('the device type defaults to the virtual pool and rejects unknown values', () => { assert.equal(readTestMuDeviceType(undefined), 'virtual'); assert.equal(readTestMuDeviceType({ providerDeviceType: '' }), 'virtual'); @@ -117,21 +70,3 @@ test('the device type defaults to the virtual pool and rejects unknown values', error.details?.flag === '--provider-device-type', ); }); - -test('other providers refuse --provider-device-type by flag name', () => { - assert.doesNotThrow(() => rejectTestMuOnlyProviderFlags(undefined, 'browserstack')); - assert.doesNotThrow(() => - rejectTestMuOnlyProviderFlags({ providerGeoLocation: 'US' }, 'browserstack'), - ); - for (const providerDeviceType of ['real', 'virtual']) { - assert.throws( - () => rejectTestMuOnlyProviderFlags({ providerDeviceType }, 'aws-device-farm'), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /--provider-device-type is only supported by TestMu AI, not aws-device-farm/.test( - error.message, - ), - ); - } -}); diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-webdriver/src/testmu-device-features.ts index 84ea7d8e95..6d886b8438 100644 --- a/packages/provider-webdriver/src/testmu-device-features.ts +++ b/packages/provider-webdriver/src/testmu-device-features.ts @@ -50,16 +50,6 @@ export const TESTMU_DEVICE_FEATURE_SPECS: readonly TestMuDeviceFeatureSpec[] = [ { field: 'providerLocale', capability: 'locale', flag: '--provider-locale' }, ]; -/** Hosted-provider flags other vendors own and TestMu has no capability for. */ -const TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS: ReadonlyArray<{ - field: keyof CloudProviderProfileFields; - flag: string; -}> = [ - { field: 'providerNetworkProfile', flag: '--provider-network-profile' }, - { field: 'providerCustomNetwork', flag: '--provider-custom-network' }, - { field: 'providerNoResignApp', flag: '--provider-no-resign-app' }, -]; - /** Builds the `lt:options` fragment for the configured device features. */ export function buildTestMuDeviceFeatureCapabilities( fields: TestMuDeviceFeatureFields, @@ -73,31 +63,6 @@ export function buildTestMuDeviceFeatureCapabilities( return capabilities; } -/** - * Fails when flags TestMu cannot act on were given. Called from both `connect testmu` (through the - * `./testmu-device-features` subpath, so the package entry stays lazy) and session preparation, - * since the typed client and hand-authored profiles skip `connect`. - */ -export function rejectUnsupportedTestMuDeviceFeatures( - flags: Record | undefined, -): void { - const configured = TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS.filter(({ field }) => { - const value = flags?.[field]; - return value !== undefined && value !== false && value !== ''; - }).map(({ flag }) => flag); - if (configured.length === 0) return; - const plural = configured.length !== 1; - throw new AppError( - 'INVALID_ARGS', - `${configured.join(', ')} ${plural ? 'are' : 'is'} not supported by TestMu AI.`, - { - hint: `Drop ${plural ? 'those flags' : 'the flag'}; TestMu AI has no equivalent capability.`, - provider: 'testmu', - flags: configured, - }, - ); -} - /** * Reads device-feature fields off an untyped flag bag (a daemon request). Enum values are * validated here rather than forwarded to the hub, where an unrecognized value is ignored. @@ -131,24 +96,3 @@ export function readTestMuDeviceType( flag: '--provider-device-type', }); } - -/** - * Fails when another provider was given a TestMu-only flag. Like the BrowserStack-only check, it - * runs in both the connect profile builder and session preparation. - */ -export function rejectTestMuOnlyProviderFlags( - flags: Record | undefined, - provider: string, -): void { - const value = flags?.providerDeviceType; - if (value === undefined || value === '') return; - throw new AppError( - 'INVALID_ARGS', - `--provider-device-type is only supported by TestMu AI, not ${provider}.`, - { - hint: 'Drop the flag or use the testmu provider.', - provider, - flags: ['--provider-device-type'], - }, - ); -} diff --git a/scripts/layering/contracts-exports.snapshot.json b/scripts/layering/contracts-exports.snapshot.json index 840c9d49cb..d5f7f69d10 100644 --- a/scripts/layering/contracts-exports.snapshot.json +++ b/scripts/layering/contracts-exports.snapshot.json @@ -87,6 +87,7 @@ "@agent-device/contracts/platform-runtime-operations", "@agent-device/contracts/platform-runtime-unavailable", "@agent-device/contracts/progress", + "@agent-device/contracts/provider-profile-fields", "@agent-device/contracts/react-native-overlay", "@agent-device/contracts/record-runtime-execution", "@agent-device/contracts/recording", diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index de24acb872..3f41c6c817 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -756,7 +756,6 @@ test('the real tree parses, declares, and passes R11', () => { assert.deepEqual([...providerWebDriverPackage.exportTargets.keys()].sort(), [ '@agent-device/provider-webdriver', '@agent-device/provider-webdriver/providers', - '@agent-device/provider-webdriver/testmu-device-features', ]); assert.deepEqual([...providerWebDriverPackage.workspaceDependencies].sort(), [ '@agent-device/capture-kit', diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index 3b0118d9e9..e91650c8b2 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -747,7 +747,7 @@ test('connect does not activate provider state when verification fails', async ( } }); -test('connect aws-device-farm rejects BrowserStack-only device-feature flags', () => { +test('connect aws-device-farm rejects device-feature flags it does not read', () => { const tempRoot = mkdtempForTestSync('agent-device-connect-aws-reject-'); try { @@ -774,7 +774,7 @@ test('connect aws-device-farm rejects BrowserStack-only device-feature flags', ( // Names every offending flag, and fires before the provider's own required-arg checks so // the caller is told what is unsupported rather than what else is missing. assert.match(error.message, /--provider-device-orientation, --provider-timezone/); - assert.match(error.message, /only supported by BrowserStack, not aws-device-farm/); + assert.match(error.message, /are not supported by AWS Device Farm/); assert.deepEqual(error.details?.flags, [ '--provider-device-orientation', '--provider-timezone', diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index 1d882c96ff..df9358b53d 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -238,10 +238,8 @@ test('providers other than TestMu refuse --provider-device-type before saving a (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.code, 'INVALID_ARGS'); - assert.match( - error.message, - new RegExp(`--provider-device-type is only supported by TestMu AI, not ${provider}`), - ); + assert.match(error.message, /^--provider-device-type is not supported by /); + assert.equal(error.details?.provider, provider); return true; }, ); @@ -252,16 +250,29 @@ test('providers other than TestMu refuse --provider-device-type before saving a } }); -test('connect limrun refuses the TestMu device type', async () => { +test('connect limrun refuses profile fields Limrun does not read', async () => { const result = await runCliCapture( - ['connect', 'limrun', '--platform', 'ios', '--provider-device-type', 'real', '--json'], + [ + 'connect', + 'limrun', + '--platform', + 'ios', + '--provider-device-type', + 'real', + '--provider-os-version', + '18', + '--json', + ], { env: { LIMRUN_API_KEY: 'lim_test_key' }, stateDirPrefix: 'agent-device-connect-limrun-device-type-', }, ); assert.equal(result.code, 1); - assert.match(result.stdout, /--provider-device-type is only supported by TestMu AI, not limrun/); + assert.match( + result.stdout, + /--provider-os-version, --provider-device-type are not supported by Limrun/, + ); }); async function connectWithGeneratedProviderProfile(options: { diff --git a/src/__tests__/lease-provider-profile-fields.test.ts b/src/__tests__/lease-provider-profile-fields.test.ts new file mode 100644 index 0000000000..141e8d3234 --- /dev/null +++ b/src/__tests__/lease-provider-profile-fields.test.ts @@ -0,0 +1,126 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { + errorResponse, + type DaemonRequest, + type DaemonResponse, +} from '@agent-device/kernel/contracts'; +import { LimrunRuntime } from '../sdk/limrun.ts'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { handleLeaseCommands } from '../daemon/handlers/lease.ts'; +import { LeaseRegistry } from '../daemon/lease-registry.ts'; +import { createProviderDeviceRuntimeRequestProviders } from '../provider-device-runtime.ts'; +import { + hashRemoteConfigFile, + writeRemoteConnectionState, +} from '../remote/remote-connection-state.ts'; +import { createTransport } from './client-transport-fixture.ts'; +import { runCliCapture } from './cli-capture.ts'; +import { makeTempWorkspace } from './cli-config-fixtures.ts'; +import { makeSessionStore } from './test-utils/store-factory.ts'; + +const limrunInstances = vi.hoisted(() => ({ + iosCreate: vi.fn(async () => { + throw new Error('a refused allocation must not create a Limrun instance'); + }), +})); + +vi.mock('@limrun/api', () => ({ + default: class MockLimrun { + readonly iosInstances = { create: limrunInstances.iosCreate }; + }, +})); + +afterEach(() => { + vi.clearAllMocks(); +}); + +// The daemon's lease handler over the same provider composition a daemon builds, so a request +// reaches the Limrun runtime exactly as it does in production. +function limrunDaemon(): (req: Omit) => Promise { + const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); + const providers = createProviderDeviceRuntimeRequestProviders([runtime]); + const leaseRegistry = new LeaseRegistry(); + const sessionStore = makeSessionStore('agent-device-limrun-profile-fields-'); + return async (req) => { + try { + return (await handleLeaseCommands({ + req: { ...req, token: 'test-token' } as Parameters[0]['req'], + sessionName: req.session ?? 'default', + sessionStore, + leaseRegistry, + providerRuntimeIds: providers.providerRuntimeIds, + providerRuntimeRequiredIds: providers.providerRuntimeRequiredIds, + leaseLifecycleProvider: providers.leaseLifecycleProvider, + })) as DaemonResponse; + } catch (error) { + const normalized = normalizeError(error); + return errorResponse(normalized.code, normalized.message, normalized.details); + } + }; +} + +test('leases.allocate refuses a real device from Limrun instead of handing out a simulator', async () => { + const setup = createTransport(limrunDaemon()); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await assert.rejects( + client.leases.allocate({ + tenant: 'limrun', + runId: 'run-real', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + platform: 'ios', + providerDeviceType: 'real', + }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-device-type is not supported by Limrun/.test(error.message), + ); + assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); +}); + +test('a remote-config profile cannot carry a field Limrun refuses past lease allocation', async () => { + const { root, home, project } = makeTempWorkspace(); + const stateDir = path.join(root, 'state'); + const remoteConfig = path.join(project, 'limrun.remote.json'); + fs.writeFileSync(remoteConfig, JSON.stringify({ providerDeviceType: 'real' }), 'utf8'); + const now = new Date().toISOString(); + writeRemoteConnectionState({ + stateDir, + state: { + version: 1, + session: 'limrun-profile', + remoteConfigPath: remoteConfig, + remoteConfigHash: hashRemoteConfigFile(remoteConfig), + tenant: 'limrun', + runId: 'run-profile', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + platform: 'ios', + connectedAt: now, + updatedAt: now, + }, + }); + const daemon = limrunDaemon(); + + try { + const result = await runCliCapture(['open', '--state-dir', stateDir, '--json'], { + cwd: project, + env: { HOME: home }, + sendToDaemon: async (req) => await daemon(req), + }); + + assert.equal(result.code, 1); + assert.equal(result.calls[0]?.command, 'lease_allocate'); + assert.equal(result.calls[0]?.flags?.providerDeviceType, 'real'); + assert.match(result.stdout, /--provider-device-type is not supported by Limrun/); + assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index c6ed0ebbb9..fe4c8b31ab 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -1,13 +1,10 @@ import { + CLOUD_WEBDRIVER_PROFILE_FIELDS, CLOUD_WEBDRIVER_PROVIDERS, readAwsDeviceFarmRegionFromArn, - rejectBrowserStackOnlyDeviceFeatures, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; -import { - rejectTestMuOnlyProviderFlags, - rejectUnsupportedTestMuDeviceFeatures, -} from '@agent-device/provider-webdriver/testmu-device-features'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformSelector } from '@agent-device/kernel/device'; @@ -27,7 +24,12 @@ export function resolveCloudWebDriverConnectProfile(options: { cwd: string; env?: EnvMap; }): { flags: CliFlags; remoteConfigPath: string } { - const providerConfig = requireConnectProfileBuilder(options.provider)(options); + const buildProfileFields = requireConnectProfileBuilder(options.provider); + rejectRefusedProviderProfileFields( + options.flags, + CLOUD_WEBDRIVER_PROFILE_FIELDS[options.provider], + ); + const providerConfig = buildProfileFields(options); const clientId = buildConnectClientId( options.provider, options.stateDir, @@ -121,7 +123,6 @@ function browserStackProfileFields(options: { env?: EnvMap; cwd: string; }): RemoteConfigProfile { - rejectTestMuOnlyProviderFlags(options.flags, CLOUD_WEBDRIVER_PROVIDERS.browserStack); return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); } @@ -130,7 +131,6 @@ function testMuProfileFields(options: { env?: EnvMap; cwd: string; }): RemoteConfigProfile { - rejectUnsupportedTestMuDeviceFeatures(options.flags); return { ...hubProviderProfileFields(TESTMU_HUB_PROFILE, options), providerDeviceType: options.flags.providerDeviceType, @@ -187,8 +187,6 @@ function awsDeviceFarmProfileFields(options: { env?: EnvMap; }): RemoteConfigProfile { const { env, flags } = options; - rejectBrowserStackOnlyDeviceFeatures(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); - rejectTestMuOnlyProviderFlags(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); const platform = requireCloudWebDriverPlatform( flags.platform, 'connect aws-device-farm requires --platform ios|android.', diff --git a/src/cli/connection/limrun-profile.ts b/src/cli/connection/limrun-profile.ts index 2033dec412..a50c5dcbeb 100644 --- a/src/cli/connection/limrun-profile.ts +++ b/src/cli/connection/limrun-profile.ts @@ -3,7 +3,8 @@ import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { CliFlags } from '@agent-device/contracts/command'; import { type EnvMap } from '@agent-device/kernel/source-value'; -import { rejectTestMuOnlyProviderFlags } from '@agent-device/provider-webdriver/testmu-device-features'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; +import { LIMRUN_PROFILE_FIELDS } from '@agent-device/provider-limrun'; import { readMetroProfileFields } from './profile-fields.ts'; import { persistAndResolveGeneratedProfile } from './generated-config.ts'; import { resolveRequestedLeaseBackend } from '../commands/connection-runtime.ts'; @@ -54,7 +55,7 @@ function buildLimrunRemoteProfile(options: { flags: CliFlags }): RemoteConfigPro } function validateLimrunConnectFlags(flags: CliFlags): 'android-instance' | 'ios-instance' { - rejectTestMuOnlyProviderFlags(flags, 'limrun'); + rejectRefusedProviderProfileFields(flags, LIMRUN_PROFILE_FIELDS); if (flags.platform !== 'android' && flags.platform !== 'ios') { throw new AppError('INVALID_ARGS', 'connect limrun requires --platform ios or android.'); } diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index 0fae68261d..f66534ba6b 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -142,7 +142,7 @@ test('BrowserStack facade nests device-feature capabilities inside bstack:option }); }, 15_000); -test('AWS Device Farm facade rejects BrowserStack-owned device features at session preparation', async () => { +test('AWS Device Farm facade rejects device features it does not read at session preparation', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const host = new FakeAwsHostCommand(`${server.url}/wd/hub/`); const provider = createProviderWebDriver({ @@ -172,7 +172,7 @@ test('AWS Device Farm facade rejects BrowserStack-owned device features at sessi (error: unknown) => { assert.match( (error as Error).message, - /--provider-device-orientation, --provider-network-profile are only supported by BrowserStack, not aws-device-farm/, + /--provider-device-orientation, --provider-network-profile are not supported by AWS Device Farm/, ); return true; }, @@ -231,7 +231,7 @@ test('TestMu facade routes a real-device session to the real pool and its upload }); }, 15_000); -test('BrowserStack facade rejects the TestMu device type at session preparation', async () => { +test('BrowserStack facade rejects the device type at session preparation', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const provider = createProviderWebDriver({ clientVersion: CLIENT_VERSION, @@ -253,7 +253,7 @@ test('BrowserStack facade rejects the TestMu device type at session preparation' await runtime.leaseLifecycle.allocate?.(lease, { flags: { ...context.flags, providerDeviceType: 'real' }, }), - /--provider-device-type is only supported by TestMu AI, not browserstack/, + /--provider-device-type is not supported by BrowserStack/, ); assert.deepEqual(server.calls, []); } finally { diff --git a/website/docs/docs/device-clouds.md b/website/docs/docs/device-clouds.md index 0e4333c012..fc1923adba 100644 --- a/website/docs/docs/device-clouds.md +++ b/website/docs/docs/device-clouds.md @@ -21,4 +21,6 @@ For each provider, the standard lifecycle is: 3. Follow the printed next command to install or open the app. 4. Run normal device commands, then `agent-device close` and `agent-device disconnect`. +Each provider reads only its own provider flags (`--provider-*` and `--aws-*`). A flag the provider does not use fails with `INVALID_ARGS` naming the flag, whether it arrives through `connect`, `client.leases.allocate()`, or a remote-config profile, so a setting is never silently dropped. + Each provider guide covers its connection selectors, client configuration, MCP setup, artifacts, and troubleshooting. Generated remote profiles are safe to store as non-secret configuration. They may include app IDs, ARNs, device names, OS versions, and labels, but never provider API keys, access keys, or AWS secret keys. diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index c2adc8266d..40d80a9717 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -105,7 +105,8 @@ agent-device connect testmu \ pool you connect to; when in doubt, pass the local path or URL and let agent-device upload it. - `TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` redirects real-device uploads, as `TESTMU_APP_UPLOAD_ENDPOINT` does for virtual-device uploads. -- `--provider-device-type` applies only to TestMu AI; other providers refuse it. +- `--provider-device-type` applies only to TestMu AI; BrowserStack, AWS Device Farm, and Limrun + refuse it on every route, including `client.leases.allocate()`. ## CLI workflow From 9b5e8182c2998b2bc32072efcef8727af2b50f9b Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:28:39 +0530 Subject: [PATCH 3/9] fix(provider-webdriver): let the Apple materializer name the uploadable file The WebDriver deployment runtime guessed what to upload for a materialized iOS build from file extensions: an extracted .app with a .zip or .ipa archive uploaded that archive. The archive it saw is the outermost one the source arrived as, so a URL .zip wrapping an .ipa uploaded the wrapper zip instead of the .ipa, and a zip holding a .app.tar.gz uploaded a zip with no app bundle at its top. Materialization now records the archive an installable was extracted from directly, and the Apple materializer declares the file a hosted provider uploads: the .ipa itself, or the zip a simulator .app was extracted from. It names nothing when no such file exists. The deployment runtime uploads the declared file and otherwise the installable path, with no inference of its own. Co-Authored-By: Claude Opus 5.5 --- .../contracts/src/app-deployment-runtime.ts | 5 +++ .../src/core/install-artifact.ts | 27 ++++++++++++- .../src/runtime-deployment.test.ts | 32 +++++++++------- .../src/runtime-deployment.ts | 14 +------ .../src/install-source-archive.ts | 29 ++++++++++++-- packages/provision-kit/src/install-source.ts | 5 +++ src/__tests__/install-source.test.ts | 38 +++++++++++++++++++ 7 files changed, 120 insertions(+), 30 deletions(-) diff --git a/packages/contracts/src/app-deployment-runtime.ts b/packages/contracts/src/app-deployment-runtime.ts index 52e581c3c9..711a6858c6 100644 --- a/packages/contracts/src/app-deployment-runtime.ts +++ b/packages/contracts/src/app-deployment-runtime.ts @@ -13,6 +13,11 @@ export type AppDeploymentSource = export type MaterializedAppSource = Readonly<{ archivePath?: string; installablePath: string; + /** + * The single file a hosted provider uploads for this build, named by the materializer that knows + * its format. Absent when the installable is already that file or no such file exists. + */ + uploadPath?: string; bundleId?: string; packageName?: string; appName?: string; diff --git a/packages/platform-apple/src/core/install-artifact.ts b/packages/platform-apple/src/core/install-artifact.ts index 4217ed13b3..959c43ca7d 100644 --- a/packages/platform-apple/src/core/install-artifact.ts +++ b/packages/platform-apple/src/core/install-artifact.ts @@ -2,7 +2,11 @@ import path from 'node:path'; import type { LocalInstallSource } from '@agent-device/kernel/contracts'; import { readIosBundleInfo } from './bundle-info.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { extractArchiveSafely, ArchiveBudget } from '@agent-device/host-kit/archive'; +import { + archiveTypeFromPath, + extractArchiveSafely, + ArchiveBudget, +} from '@agent-device/host-kit/archive'; import { installArtifactArchiveBudget, @@ -35,6 +39,7 @@ type IosPayloadAppBundle = { export type PreparedIosInstallArtifact = { archivePath?: string; installablePath: string; + uploadPath?: string; bundleId?: string; appName?: string; cleanup: () => Promise; @@ -80,9 +85,11 @@ async function prepareIosInstallArtifactInScope( ? materialized.installablePath : undefined); + const uploadPath = iosUploadPath(materialized); return { archivePath, installablePath: resolvedInstallable.installPath, + ...(uploadPath ? { uploadPath } : {}), bundleId: bundleInfo.bundleId, appName: bundleInfo.appName, cleanup: async () => { @@ -102,6 +109,24 @@ async function prepareIosInstallArtifactInScope( export { readIosBundleInfo } from './bundle-info.ts'; +/** + * The installable is an extracted `.app` directory, which no hosted upload API accepts. The file + * that carries it is the `.ipa` it was unpacked from, or the zip the `.app` was extracted from + * directly; an outer archive that merely wrapped either is never it. + */ +function iosUploadPath(materialized: { + containingArchivePath?: string; + installablePath: string; +}): string | undefined { + if (materialized.installablePath.toLowerCase().endsWith('.ipa')) { + return materialized.installablePath; + } + const { containingArchivePath } = materialized; + return containingArchivePath && archiveTypeFromPath(containingArchivePath) === 'zip' + ? containingArchivePath + : undefined; +} + async function resolveIosInstallablePath( appPath: string, options?: InstallIosArtifactOptions, diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index 559f3c37c9..41aaa508d3 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -86,9 +86,9 @@ test('aborts a WebDriver provider deployment while its install is in flight', as expect(installApp).toHaveBeenCalledWith('bs://uploaded-app', controller.signal); }); -// Materialization extracts `App.app.zip` (or an .ipa) to a `.app` directory, which no hosted -// upload API accepts; the archive it came from is the uploadable build. -test('a hosted upload of a materialized iOS bundle sends the archive it was extracted from', async () => { +// The materializer knows which file carries the build; the deployment runtime must not second-guess +// it from extensions, or a URL zip wrapping an .ipa would upload the wrapper. +test('a hosted upload sends the file the materializer names, else the installable', async () => { const uploaded: string[] = []; const installApp = vi.fn(async () => undefined); const deployment = createWebDriverDeploymentRuntime({ @@ -109,20 +109,18 @@ test('a hosted upload of a materialized iOS bundle sends the archive it was extr const result = await deploy(iosDevice, { archivePath: '/m/App.app.zip', installablePath: '/m/extracted/App.app', + uploadPath: '/m/App.app.zip', bundleId: 'com.example.app', }); - await deploy(iosDevice, { archivePath: '/m/App.ipa', installablePath: '/m/x/Payload/App.app' }); - await deploy(iosDevice, { installablePath: '/m/App.app' }); + await deploy(iosDevice, { + archivePath: '/m/wrapper.zip', + installablePath: '/m/x/Payload/App.app', + uploadPath: '/m/x/App.ipa', + }); await deploy(iosDevice, { archivePath: '/m/App.tar.gz', installablePath: '/m/x/App.app' }); await deploy(device, { archivePath: '/m/build.zip', installablePath: '/m/x/app.apk' }); - expect(uploaded).toEqual([ - '/m/App.app.zip', - '/m/App.ipa', - '/m/App.app', - '/m/x/App.app', - '/m/x/app.apk', - ]); + expect(uploaded).toEqual(['/m/App.app.zip', '/m/x/App.ipa', '/m/x/App.app', '/m/x/app.apk']); expect(result).toEqual({ bundleId: 'com.example.app', launchTarget: 'com.example.app' }); expect(installApp).toHaveBeenNthCalledWith(1, 'lt://1', expect.any(AbortSignal)); }); @@ -139,6 +137,7 @@ test('a provider without an uploader still installs the materialized bundle path artifact: { archivePath: '/m/App.app.zip', installablePath: '/m/extracted/App.app', + uploadPath: '/m/App.app.zip', bundleId: 'com.example.app', cleanup: async () => {}, }, @@ -175,7 +174,14 @@ test('TestMu uploads the zipped simulator build that install-from-source extract await deployment.deployMaterializedApp( iosDevice, - { artifact: { archivePath, installablePath, cleanup: async () => {} } }, + { + artifact: { + archivePath, + installablePath, + uploadPath: archivePath, + cleanup: async () => {}, + }, + }, new AbortController().signal, ); diff --git a/packages/provider-webdriver/src/runtime-deployment.ts b/packages/provider-webdriver/src/runtime-deployment.ts index f88fe4f304..fb2a058b86 100644 --- a/packages/provider-webdriver/src/runtime-deployment.ts +++ b/packages/provider-webdriver/src/runtime-deployment.ts @@ -6,7 +6,6 @@ import type { AppDeploymentInput, AppDeploymentResult, DeployMaterializedAppInput, - MaterializedAppSource, } from '@agent-device/contracts/app-deployment-runtime'; import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; import { publicPlatformString, type DeviceInfo } from '@agent-device/kernel/device'; @@ -105,7 +104,7 @@ export function createWebDriverDeploymentRuntime( '', { appPath: input.artifact.installablePath, - uploadPath: materializedUploadPath(input.artifact), + uploadPath: input.artifact.uploadPath ?? input.artifact.installablePath, }, { appIdentifierHint: input.artifact.bundleId, @@ -117,17 +116,6 @@ export function createWebDriverDeploymentRuntime( }); } -/** - * Materialization extracts an iOS `.app` bundle out of a zipped simulator build or an .ipa, and no - * hosted upload API takes a directory, so the uploader gets the archive the bundle came from. - */ -function materializedUploadPath(artifact: MaterializedAppSource): string { - const { archivePath, installablePath } = artifact; - return archivePath && /\.(zip|ipa)$/i.test(archivePath) && /\.app\/?$/i.test(installablePath) - ? archivePath - : installablePath; -} - function deploymentFact(session: WebDriverProviderSession | undefined): RuntimeOperationFact { if (!session) { return Object.freeze({ diff --git a/packages/provision-kit/src/install-source-archive.ts b/packages/provision-kit/src/install-source-archive.ts index 7449c3d480..1f5fca61b8 100644 --- a/packages/provision-kit/src/install-source-archive.ts +++ b/packages/provision-kit/src/install-source-archive.ts @@ -15,10 +15,32 @@ type InstallableMatcher = ( stat: { isFile(): boolean; isDirectory(): boolean }, ) => boolean; +type ResolvedInstallableCandidate = { + /** The outermost archive the source arrived as. */ + archivePath?: string; + /** The innermost archive, the one the installable was extracted from directly. */ + containingArchivePath?: string; + installablePath: string; +}; + +function resolvedCandidate( + installablePath: string, + params: { archivePath: string | undefined; containingArchivePath?: string }, +): ResolvedInstallableCandidate { + return { + archivePath: params.archivePath, + ...(params.containingArchivePath + ? { containingArchivePath: params.containingArchivePath } + : {}), + installablePath, + }; +} + export async function resolveInstallableCandidate( candidatePath: string, params: { archivePath: string | undefined; + containingArchivePath?: string; isInstallablePath: InstallableMatcher; installableLabel: string; allowArchiveExtraction: boolean; @@ -27,11 +49,11 @@ export async function resolveInstallableCandidate( archiveDepth: number; onArchiveAccepted?: (depth: number) => void; }, -): Promise<{ archivePath?: string; installablePath: string }> { +): Promise { const stat = await fs.stat(candidatePath).catch(() => null); if (!stat) throw new AppError('INVALID_ARGS', `App source not found: ${candidatePath}`); if (params.isInstallablePath(candidatePath, stat)) { - return { archivePath: params.archivePath, installablePath: candidatePath }; + return resolvedCandidate(candidatePath, params); } if (stat.isFile() && isArchivePath(candidatePath)) { assertArchiveExtractionAllowed(candidatePath, params, false); @@ -40,7 +62,7 @@ export async function resolveInstallableCandidate( if (stat.isDirectory()) { const installables = await collectMatchingPaths(candidatePath, params.isInstallablePath); if (installables.length === 1) { - return { archivePath: params.archivePath, installablePath: installables[0]! }; + return resolvedCandidate(installables[0]!, params); } if (installables.length > 1) { throw new AppError( @@ -80,6 +102,7 @@ async function resolveExtractedArchive( return await resolveInstallableCandidate(extracted.outputPath, { ...params, archivePath: params.archivePath ?? archivePath, + containingArchivePath: archivePath, archiveDepth: params.archiveDepth + 1, }); } diff --git a/packages/provision-kit/src/install-source.ts b/packages/provision-kit/src/install-source.ts index e73c4aafdb..26119979e9 100644 --- a/packages/provision-kit/src/install-source.ts +++ b/packages/provision-kit/src/install-source.ts @@ -33,6 +33,8 @@ export type MaterializeInstallableOptions = { export type MaterializedInstallable = { archivePath?: string; + /** The archive the installable was extracted from directly, when it came out of one. */ + containingArchivePath?: string; installablePath: string; cleanup: () => Promise; }; @@ -69,6 +71,9 @@ export async function materializeInstallablePath( }); return { archivePath: resolved.archivePath, + ...(resolved.containingArchivePath + ? { containingArchivePath: resolved.containingArchivePath } + : {}), installablePath: resolved.installablePath, cleanup: async () => { await runCleanupTasks(cleanupTasks); diff --git a/src/__tests__/install-source.test.ts b/src/__tests__/install-source.test.ts index 58206934e5..974a6ba509 100644 --- a/src/__tests__/install-source.test.ts +++ b/src/__tests__/install-source.test.ts @@ -381,6 +381,8 @@ test('prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing try { assert.equal(path.basename(result.installablePath), 'Demo.app'); + // A tar has no hosted upload API, so nothing is named uploadable. + assert.equal(result.uploadPath, undefined); assert.equal(result.bundleId, 'com.example.githubtar'); assert.equal(result.appName, 'GitHub Tar'); } finally { @@ -420,6 +422,9 @@ test('prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing try { assert.equal(path.basename(result.installablePath), 'Demo.app'); + // The .ipa, not the artifact zip that wrapped it, is what a hosted provider uploads. + assert.equal(path.basename(result.uploadPath ?? ''), 'Demo.ipa'); + assert.equal((await fs.stat(result.uploadPath ?? '')).isFile(), true); assert.equal(result.bundleId, 'com.example.githubipa'); assert.equal(result.appName, 'GitHub IPA'); } finally { @@ -431,6 +436,39 @@ test('prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing ); }); +test('prepareIosInstallArtifact names the zip a simulator app arrived in as its upload', async () => { + await withArchiveFixture( + { + extractions: [ + { + command: 'unzip', + populate: async (outputPath) => { + await fs.mkdir(path.join(outputPath, 'Demo.app')); + }, + }, + ], + }, + async () => { + await withIosBundleInfo('com.example.githubapp', 'GitHub App', async () => { + await withMockedInstallSourceFetch(Buffer.from('artifact fixture'), async () => { + const result = await prepareIosInstallArtifact({ + kind: 'url', + url: 'https://api.github.com/repos/acme/app/actions/artifacts/989/zip', + }); + + try { + assert.equal(path.basename(result.installablePath), 'Demo.app'); + assert.equal(result.uploadPath, result.archivePath); + assert.equal((await fs.stat(result.uploadPath ?? '')).isFile(), true); + } finally { + await result.cleanup(); + } + }); + }); + }, + ); +}); + test('prepareIosInstallArtifact cleans URL materialization when IPA payload resolution fails', async () => { await withArchiveFixture( { From 402370e0f6dd13bcb5aacb02d143051140138ede Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:35:09 +0530 Subject: [PATCH 4/9] fix(provider-webdriver): tighten app-reference, endpoint and upload input handling Several inputs reached a hosted provider in a shape it could not use, or failed with an untyped error: - An empty `lt://` passed as a TestMu AI app reference and reached session creation. A reference is now validated against the app-id grammar, and an invalid one is INVALID_ARGS on connect and at session preparation. - `LT://APP1` or `BS://id` was treated as a file path. App schemes now match case-insensitively and are canonicalized to the spelling the hub accepts. - A directory given as `--provider-app` reached the upload and failed with a raw EISDIR; it is now INVALID_ARGS with a hint. - A missing path given straight to the TestMu AI upload failed with a bare ENOENT; it now gets the same typed refusal as a directory. - Session-detail URLs were built by string concatenation, so a query on TESTMU_API_ENDPOINT or the BrowserStack details endpoint swallowed the route. Routes are now appended to the URL path, keeping the query. - A whitespace-only artifact URL was reported as a ready artifact; it is now treated as absent, and URLs are trimmed. - A 2xx connection-verification answer that is not JSON was reported as a network failure. It is now COMMAND_FAILED with the HTTP status, and each hub supplies its own service-status hint. The connect help and the TestMu AI guide now say what connect checks for `--provider-app` and what is only validated when the session is created. Co-Authored-By: Claude Opus 5.5 --- .../src/artifact-results.ts | 5 +- .../browserstack-connection-verification.ts | 1 + .../src/browserstack.test.ts | 16 ++++ .../provider-webdriver/src/browserstack.ts | 7 +- .../src/testmu-connection-verification.ts | 21 ++--- .../provider-webdriver/src/testmu.test.ts | 34 ++++++- packages/provider-webdriver/src/testmu.ts | 33 ++++--- .../src/webdriver-utils.test.ts | 92 +++++++++++++++++++ .../provider-webdriver/src/webdriver-utils.ts | 63 +++++++++++-- src/__tests__/cloud-connect-testmu.test.ts | 38 ++++++++ src/cli/connection/cloud-webdriver-profile.ts | 10 +- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/testmu.md | 13 ++- 13 files changed, 287 insertions(+), 48 deletions(-) diff --git a/packages/provider-webdriver/src/artifact-results.ts b/packages/provider-webdriver/src/artifact-results.ts index de29b4ab3a..3aaabc69b5 100644 --- a/packages/provider-webdriver/src/artifact-results.ts +++ b/packages/provider-webdriver/src/artifact-results.ts @@ -38,7 +38,8 @@ export function urlArtifactFromDetails( kind: CloudArtifact['kind'], name: string, ): CloudArtifact | undefined { - const url = details[field]; - if (typeof url !== 'string' || url.length === 0) return undefined; + const value = details[field]; + const url = typeof value === 'string' ? value.trim() : ''; + if (url.length === 0) return undefined; return { provider, providerSessionId, kind, name, url, availability: 'ready' }; } diff --git a/packages/provider-webdriver/src/browserstack-connection-verification.ts b/packages/provider-webdriver/src/browserstack-connection-verification.ts index 56d36366cc..dfb18bd989 100644 --- a/packages/provider-webdriver/src/browserstack-connection-verification.ts +++ b/packages/provider-webdriver/src/browserstack-connection-verification.ts @@ -110,6 +110,7 @@ async function fetchBrowserStackJson( hints: { service: 'BrowserStack', unauthorizedHint: 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.', + serviceHint: 'Retry connect or check the BrowserStack service status.', networkHint: 'Check network access to api-cloud.browserstack.com and retry connect.', }, }); diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 1d8ca76b1a..7d7af5610a 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -135,3 +135,19 @@ test('BrowserStack session details lookup has a deadline and fails typed', async }); } }); + +test('BrowserStack session details keep a query on the endpoint override', async () => { + const calls: string[] = []; + globalThis.fetch = async (input) => { + calls.push(String(input)); + return new Response(JSON.stringify({ automation_session: { video_url: ' ' } }), { + status: 200, + }); + }; + const result = await listBrowserStackCloudArtifacts('browserstack', 'SESSION1', { + ...upload, + endpoint: 'https://api.example.test/sessions?region=eu', + }); + assert.deepEqual(calls, ['https://api.example.test/sessions/SESSION1.json?region=eu']); + assert.equal(result?.status, 'pending'); +}); diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index a5954eda00..ffd854052f 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -3,13 +3,13 @@ import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { + appendUrlPath, appFileUploadForm, asRecord, createHubUploadApp, fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, - trimTrailingSlash, } from './webdriver-utils.ts'; export const BROWSERSTACK_APP_AUTOMATE_ENDPOINT = 'https://hub-cloud.browserstack.com/wd/hub/'; @@ -151,8 +151,9 @@ async function fetchBrowserStackSessionDetails( sessionId: string, options: BrowserStackSessionDetailsOptions, ): Promise> { - const endpoint = new URL( - `${trimTrailingSlash(String(options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT))}/${sessionId}.json`, + const endpoint = appendUrlPath( + options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT, + `${sessionId}.json`, ); const json = await fetchProviderSessionDetails(endpoint, { clientVersion: options.clientVersion, diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-webdriver/src/testmu-connection-verification.ts index f54c28358f..7e37a4f2f5 100644 --- a/packages/provider-webdriver/src/testmu-connection-verification.ts +++ b/packages/provider-webdriver/src/testmu-connection-verification.ts @@ -1,7 +1,12 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { asRecord, fetchProviderVerificationJson, trimTrailingSlash } from './webdriver-utils.ts'; -import { TESTMU_API_ENDPOINT, TESTMU_APPS_ENDPOINT, isTestMuAppReference } from './testmu.ts'; +import { appendUrlPath, asRecord, fetchProviderVerificationJson } from './webdriver-utils.ts'; +import { + TESTMU_API_ENDPOINT, + TESTMU_APPS_ENDPOINT, + isTestMuAppReference, + testMuAppReferenceFromId, +} from './testmu.ts'; import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, @@ -34,7 +39,7 @@ export async function verifyTestMuConnection( const deviceType = options.deviceType ?? 'virtual'; const catalogUrl = options.devicesEndpoint ? new URL(options.devicesEndpoint) - : apiUrl(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); + : appendUrlPath(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); catalogUrl.searchParams.set('isVirtualDevice', String(deviceType === 'virtual')); const catalog = await fetchTestMuJson(catalogUrl, undefined, clientVersion); const namedDevices = readTestMuCatalogDevices(catalog, options.platform, deviceType).filter( @@ -118,13 +123,6 @@ async function verifyTestMuApp( }; } -/** Appends `route` to the base's path; the base may carry a query, which is kept. */ -function apiUrl(base: string | URL, route: string): URL { - const url = new URL(base); - url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; - return url; -} - async function fetchTestMuJson( endpoint: string | URL, auth: TestMuAuth | undefined, @@ -136,6 +134,7 @@ async function fetchTestMuJson( hints: { service: 'TestMu AI', unauthorizedHint: 'Check LT_USERNAME and LT_ACCESS_KEY.', + serviceHint: 'Retry connect or check the TestMu AI service status.', networkHint: 'Check network access to mobile-api.lambdatest.com and manual-api.lambdatest.com, then retry connect.', }, @@ -189,7 +188,7 @@ function readTestMuApps( return data.flatMap((entry) => { const app = asRecord(entry); if (!app || typeof app.app_id !== 'string') return []; - const reference = isTestMuAppReference(app.app_id) ? app.app_id : `lt://${app.app_id}`; + const reference = testMuAppReferenceFromId(app.app_id); return [ { reference, diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts index fd00b6782e..d14039a9fc 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -244,6 +244,20 @@ test('TestMu upload rejects an unzipped .app bundle before calling the upload AP } }); +// The install adapter reaches the upload without the resolver's existence check in front of it. +test('TestMu upload of a missing path fails typed, not with a bare ENOENT', async () => { + const fetchMock = vi.fn(); + globalThis.fetch = fetchMock; + await assert.rejects(uploadTestMuApp('/nonexistent/App.apk', auth), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.message, 'TestMu AI can only upload an app file: /nonexistent/App.apk'); + assert.ok(error.details?.hint); + return true; + }); + assert.equal(fetchMock.mock.calls.length, 0); +}); + test('the install adapter uploads the local build and launches the hinted app id', async () => { const tempDir = await mkdtempForTest('agent-device-testmu-install-'); const appPath = path.join(tempDir, 'Demo.apk'); @@ -277,6 +291,14 @@ test('TestMu passes lt:// ids through and has the upload API fetch a public URL' return jsonResponse({ app_id: 'APP9' }); }; assert.equal(await resolveTestMuAppReference('lt://APP1', auth), 'lt://APP1'); + assert.equal(await resolveTestMuAppReference('LT://APP1', auth), 'lt://APP1'); + await assert.rejects( + resolveTestMuAppReference('lt://', auth), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-app lt:\/\/ is not an lt:\/\/ app id/.test(error.message), + ); assert.equal(forms.length, 0); assert.equal( await resolveTestMuAppReference('https://builds.example/App.apk', auth), @@ -346,9 +368,10 @@ test('TestMu artifacts come from the jsend session payload and stay pending unti status: 'success', data: { test_id: 'SESSION1', - video_url: 'https://cdn.test/video.mp4', + video_url: ' https://cdn.test/video.mp4\n', appium_logs_url: 'https://api.test/sessions/SESSION1/log/appium', device_logs_url: '', + network_logs_url: ' ', }, }); }; @@ -356,7 +379,14 @@ test('TestMu artifacts come from the jsend session payload and stay pending unti ...auth, endpoint: 'https://api.test/mobile-automation/api/v1/', }); - assert.deepEqual(calls, ['https://api.test/mobile-automation/api/v1/sessions/SESSION1']); + await listTestMuCloudArtifacts('testmu', 'SESSION 2', { + ...auth, + endpoint: 'https://api.test/mobile-automation/api/v1?region=eu', + }); + assert.deepEqual(calls, [ + 'https://api.test/mobile-automation/api/v1/sessions/SESSION1', + 'https://api.test/mobile-automation/api/v1/sessions/SESSION%202?region=eu', + ]); assert.equal(result?.status, 'ready'); assert.deepEqual( result?.cloudArtifacts.map((artifact) => [artifact.kind, artifact.url]), diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts index a632cba50e..1217fd6c05 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-webdriver/src/testmu.ts @@ -6,13 +6,13 @@ import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime. import { AppError } from '@agent-device/kernel/errors'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { + appendUrlPath, appFileUploadForm, asRecord, createHubUploadApp, fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, - trimTrailingSlash, } from './webdriver-utils.ts'; /** @@ -83,7 +83,9 @@ export async function uploadTestMuApp( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - if (!(await fs.stat(appPath)).isFile()) { + // A missing path is the same caller mistake as a directory, so both get the typed refusal. + const stat = await fs.stat(appPath).catch(() => undefined); + if (!stat?.isFile()) { throw new AppError('INVALID_ARGS', `TestMu AI can only upload an app file: ${appPath}`, { appPath, hint: @@ -146,6 +148,7 @@ export async function resolveTestMuAppReference( cwd: options.cwd, referenceScheme: 'lt://', referenceLabel: 'an lt:// app id', + isReference: isTestMuAppReference, uploadFile: async (appPath, signal) => await uploadTestMuApp(appPath, options, signal), uploadUrl: async (url, signal) => await uploadTestMuAppFromUrl(url, options, signal), signal: options.signal, @@ -190,16 +193,24 @@ export function buildTestMuCapabilities( }; } +const TESTMU_APP_REFERENCE = /^lt:\/\/[\w.-]+$/; + export function isTestMuAppReference(value: string): boolean { - return value.startsWith('lt://'); + return TESTMU_APP_REFERENCE.test(value); +} + +/** The upload and app-list APIs answer with a bare app id or an `lt://` reference. */ +export function testMuAppReferenceFromId(id: string): string { + return id.startsWith('lt://') ? id : `lt://${id}`; } async function fetchTestMuSessionDetails( sessionId: string, options: TestMuSessionDetailsOptions, ): Promise> { - const endpoint = new URL( - `${trimTrailingSlash(String(options.endpoint ?? TESTMU_API_ENDPOINT))}/sessions/${encodeURIComponent(sessionId)}`, + const endpoint = appendUrlPath( + options.endpoint ?? TESTMU_API_ENDPOINT, + `sessions/${encodeURIComponent(sessionId)}`, ); let json: Record; try { @@ -259,17 +270,11 @@ function mapTestMuArtifacts( return ready.length > 0 ? [...ready, dashboard] : []; } -const TESTMU_APP_ID = /^[\w.-]+$/; - /** The upload answers with `app_url` (`lt://…`) and/or a bare `app_id`; anything else is a failed upload. */ function readTestMuAppReference(value: unknown): string | undefined { const { app_url: appUrl, app_id: appId } = asRecord(value) ?? {}; - if (typeof appUrl === 'string' && isValidTestMuAppReference(appUrl)) return appUrl; + if (typeof appUrl === 'string' && isTestMuAppReference(appUrl)) return appUrl; if (typeof appId !== 'string') return undefined; - const reference = isTestMuAppReference(appId) ? appId : `lt://${appId}`; - return isValidTestMuAppReference(reference) ? reference : undefined; -} - -function isValidTestMuAppReference(value: string): boolean { - return isTestMuAppReference(value) && TESTMU_APP_ID.test(value.slice('lt://'.length)); + const reference = testMuAppReferenceFromId(appId); + return isTestMuAppReference(reference) ? reference : undefined; } diff --git a/packages/provider-webdriver/src/webdriver-utils.test.ts b/packages/provider-webdriver/src/webdriver-utils.test.ts index 122aafaf80..58a182c789 100644 --- a/packages/provider-webdriver/src/webdriver-utils.test.ts +++ b/packages/provider-webdriver/src/webdriver-utils.test.ts @@ -4,8 +4,10 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { + appendUrlPath, asRecord, createHubUploadApp, + fetchProviderVerificationJson, postHubAppUpload, resolveHubAppReference, trimLeadingSlash, @@ -112,6 +114,7 @@ test('the hub app resolver passes references through, uploads local files, and r }); assert.equal(await resolve('hub://APP3'), 'hub://APP3'); + assert.equal(await resolve('HUB://APP3'), 'hub://APP3'); assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); assert.equal( await resolve('https://builds.example/App.apk', async (url) => `fetched:${url}`), @@ -132,3 +135,92 @@ test('the hub app resolver passes references through, uploads local files, and r await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test('the hub app resolver refuses an empty reference and a directory, typed', async () => { + const tempDir = await mkdtempForTest('agent-device-hub-resolve-invalid-'); + try { + await fs.mkdir(path.join(tempDir, 'App.app')); + const uploadFile = vi.fn(async () => 'hub://never'); + const resolve = (app: string) => + resolveHubAppReference({ + service: 'Hub', + app, + cwd: tempDir, + referenceScheme: 'hub://', + referenceLabel: 'a hub:// app id', + isReference: (reference) => /^hub:\/\/\w+$/.test(reference), + uploadFile, + }); + + for (const [app, message] of [ + ['hub://', /^Hub --provider-app hub:\/\/ is not a hub:\/\/ app id\.$/], + ['hub://a b', /is not a hub:\/\/ app id/], + ['App.app', /must be an app file, not a directory: .*App\.app$/], + ] as const) { + await assert.rejects( + resolve(app), + (error: unknown) => + error instanceof AppError && error.code === 'INVALID_ARGS' && message.test(error.message), + ); + } + assert.equal(uploadFile.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('appending a route keeps a query on the base endpoint', () => { + assert.equal( + appendUrlPath('https://api.example.test/v1/?region=eu', 'sessions/S%201').toString(), + 'https://api.example.test/v1/sessions/S%201?region=eu', + ); + assert.equal( + appendUrlPath('https://api.example.test/v1', 'sessions/S1').toString(), + 'https://api.example.test/v1/sessions/S1', + ); +}); + +const verificationHints = { + service: 'Hub', + unauthorizedHint: 'Check HUB_KEY.', + serviceHint: 'Retry connect or check the Hub service status.', + networkHint: 'Check network access to the hub.', +}; + +test('connection verification reports a non-JSON success typed, with its status', async () => { + globalThis.fetch = async () => new Response('maintenance', { status: 200 }); + await assert.rejects( + fetchProviderVerificationJson('https://api.example.test/apps', { + clientVersion: '0.0.0-test', + hints: verificationHints, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'Hub connection verification answer was not JSON.'); + assert.equal(error.details?.status, 200); + assert.equal(error.details?.hint, verificationHints.serviceHint); + return true; + }, + ); +}); + +test('connection verification gives each failure its provider hint', async () => { + for (const [status, code, hint] of [ + [401, 'UNAUTHORIZED', verificationHints.unauthorizedHint], + [503, 'COMMAND_FAILED', verificationHints.serviceHint], + ] as const) { + globalThis.fetch = async () => new Response('nope', { status }); + await assert.rejects( + fetchProviderVerificationJson('https://api.example.test/apps', { + clientVersion: '0.0.0-test', + hints: verificationHints, + }), + (error: unknown) => + error instanceof AppError && + error.code === code && + error.details?.status === status && + error.details?.hint === hint, + ); + } +}); diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 88724187dc..d75fd560ba 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -56,6 +56,13 @@ export function trimTrailingSlash(value: string): string { return lastNonSlash === value.length - 1 ? value : value.slice(0, lastNonSlash + 1); } +/** Appends `route` to the base's path; a query on the base is kept rather than swallowing the route. */ +export function appendUrlPath(base: string | URL, route: string): URL { + const url = new URL(base); + url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; + return url; +} + export function withTrailingSlash(url: URL): URL { if (url.pathname.endsWith('/')) return url; const copy = new URL(url); @@ -141,39 +148,71 @@ export async function resolveHubAppReference(options: { referenceScheme: string; /** How the scheme reads in the error message, e.g. `a bs:// app id`. */ referenceLabel: string; + /** Validates a canonical reference; by default any non-empty id after the scheme is accepted. */ + isReference?: (reference: string) => boolean; uploadFile: (appPath: string, signal?: AbortSignal) => Promise; uploadUrl?: (url: string, signal?: AbortSignal) => Promise; signal?: AbortSignal; }): Promise { const { app } = options; - if (app.startsWith(options.referenceScheme)) return app; + const reference = canonicalHubAppReference(app, options.referenceScheme); + if (reference !== undefined) { + const isReference = + options.isReference ?? ((value: string) => value.length > options.referenceScheme.length); + if (isReference(reference)) return reference; + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app ${app} is not ${options.referenceLabel}.`, + { providerApp: app }, + ); + } if (/^https?:\/\//i.test(app)) { return options.uploadUrl ? await options.uploadUrl(app, options.signal) : app; } const appPath = path.resolve(options.cwd ?? process.cwd(), app); - if (!fs.existsSync(appPath)) { + const stat = fs.statSync(appPath, { throwIfNoEntry: false }); + if (!stat) { throw new AppError( 'INVALID_ARGS', `${options.service} --provider-app must be ${options.referenceLabel}, URL, or existing local app path.`, { providerApp: app }, ); } + if (!stat.isFile()) { + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app must be an app file, not a directory: ${appPath}`, + { + providerApp: app, + hint: 'Zip an iOS simulator .app bundle and pass the .zip, or pass the .ipa, .apk, or .aab.', + }, + ); + } return await options.uploadFile(appPath, options.signal); } +/** URI schemes are case-insensitive, so `LT://id` is the hub reference `lt://id`. */ +function canonicalHubAppReference(app: string, scheme: string): string | undefined { + if (app.slice(0, scheme.length).toLowerCase() !== scheme) return undefined; + return `${scheme}${app.slice(scheme.length)}`; +} + const PROVIDER_API_TIMEOUT_MS = 15_000; /** The provider rejected or could not answer a verification call; typed so callers never sniff text. */ export type ProviderJsonFailureHints = { service: string; unauthorizedHint: string; + /** For any other non-2xx answer, or a 2xx answer that is not JSON. */ + serviceHint: string; networkHint: string; }; /** * Fetches JSON from a hosted provider's API during connection verification. A 401/403 is - * `UNAUTHORIZED` with a credential hint, any other non-2xx is `COMMAND_FAILED`, and a transport - * failure is wrapped so its cause survives without leaking the credentials. + * `UNAUTHORIZED` with a credential hint, any other non-2xx or a body that is not JSON is + * `COMMAND_FAILED` with the status, and a transport failure is wrapped so its cause survives + * without leaking the credentials. */ export async function fetchProviderVerificationJson( endpoint: string | URL, @@ -183,7 +222,7 @@ export async function fetchProviderVerificationJson( hints: ProviderJsonFailureHints; }, ): Promise { - const { service, unauthorizedHint, networkHint } = options.hints; + const { service, unauthorizedHint, serviceHint, networkHint } = options.hints; try { const response = await fetch(endpoint, { headers: { @@ -199,13 +238,19 @@ export async function fetchProviderVerificationJson( `${service} rejected connection verification.`, { status: response.status, - hint: unauthorized - ? unauthorizedHint - : `Retry connect or check the ${service} service status.`, + hint: unauthorized ? unauthorizedHint : serviceHint, }, ); } - return (await response.json()) as unknown; + const json = await readProviderJsonBody(response); + if (json === undefined) { + throw new AppError( + 'COMMAND_FAILED', + `${service} connection verification answer was not JSON.`, + { status: response.status, hint: serviceHint }, + ); + } + return json; } catch (error) { if (error instanceof AppError) throw error; throw new AppError( diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index df9358b53d..e9b7a3413c 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -85,6 +85,44 @@ test('connect testmu generates a local provider profile and verifies the virtual } }); +test('connect canonicalizes an upper-case app scheme and refuses an empty app id', () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-app-scheme-'); + const base = { json: false, help: false, version: false, platform: 'ios' as const }; + const connect = (provider: 'testmu' | 'browserstack', providerApp: string) => + resolveCloudWebDriverConnectProfile({ + provider, + stateDir: path.join(tempRoot, `.state-${provider}`), + cwd: tempRoot, + env: { + LT_USERNAME: 'u', + LT_ACCESS_KEY: 'k', + BROWSERSTACK_USERNAME: 'u', + BROWSERSTACK_ACCESS_KEY: 'k', + }, + flags: { ...base, device: 'iPhone 16', providerOsVersion: '18.0', providerApp }, + }); + + try { + assert.equal( + readGeneratedConfig(connect('testmu', 'LT://APP1').remoteConfigPath).providerApp, + 'lt://APP1', + ); + assert.equal( + readGeneratedConfig(connect('browserstack', 'Bs://abc').remoteConfigPath).providerApp, + 'bs://abc', + ); + assert.throws( + () => connect('testmu', 'lt://'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-app lt:\/\/ has no app id/.test(error.message), + ); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + test('connect testmu verifies against TESTMU_API_ENDPOINT', async () => { const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-endpoint-'); vi.stubEnv('LT_USERNAME', 'lt-user'); diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index fe4c8b31ab..5611c86eaa 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -172,7 +172,15 @@ function hubProviderProfileFields( } function normalizeHubAppReference(hub: HubProviderProfile, app: string, cwd: string): string { - if (app.startsWith(hub.appScheme) || /^https?:\/\//i.test(app)) return app; + if (/^https?:\/\//i.test(app)) return app; + // URI schemes are case-insensitive; the hub only matches the lower-case spelling. + if (app.slice(0, hub.appScheme.length).toLowerCase() === hub.appScheme) { + const id = app.slice(hub.appScheme.length); + if (id.length > 0) return `${hub.appScheme}${id}`; + throw new AppError('INVALID_ARGS', `${hub.command} --provider-app ${app} has no app id.`, { + hint: `Pass ${hub.appHint}.`, + }); + } const resolvedPath = path.resolve(cwd, app); try { if (fs.statSync(resolvedPath).isFile()) return resolvedPath; diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index b4d396fe9f..37c359bd63 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -577,7 +577,7 @@ Providers: Direct proxy: agent-device connect proxy --daemon-base-url stores the shared proxy profile and client identity. BrowserStack: agent-device connect browserstack verifies credentials, the exact device, and a bs:// app reference, then stores a local provider profile. It does not create an App Automate session. AWS Device Farm: agent-device connect aws-device-farm verifies credentials and the exact project, device, and optional app upload, then stores a local provider profile. It does not create a remote access session. - TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) or, with --provider-device-type real, real device and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. + TestMu AI: agent-device connect testmu verifies credentials and the exact virtual device (emulator or simulator) or, with --provider-device-type real, real device and OS version, then stores a local provider profile. --provider-app takes an lt:// app reference, an https URL, or a local path: connect looks an lt:// id up among your uploads for that device pool, while URL and local sources are uploaded and validated when open creates the session. It does not create a hub session. Limrun: agent-device connect limrun verifies access to the selected iOS or Android instance service, then stores a local provider profile. It does not create an instance. After direct-provider connect: diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index 40d80a9717..87912239ae 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -35,14 +35,17 @@ hub rejects `--provider-os-version 18` for a device listed with `18.0`, so `conn lists the versions the device offers. `--provider-app` accepts a TestMu AI app reference such as `lt://APP...`, an HTTP(S) app URL, or -an existing local app path (`.apk`, or a zipped simulator `.app` for iOS). TestMu AI uploads a local -path or fetches a URL when it creates the hosted session, through the virtual-device upload API. +an existing local app path (`.apk`, or a zipped simulator `.app` for iOS). When `open` creates the +hosted session, agent-device uploads a local path, and TestMu AI fetches a URL, through the +virtual-device upload API. During `connect`, agent-device checks the device/OS pair against TestMu AI's virtual-device catalog (`/capability/generator?isVirtualDevice=true`), verifies the credentials against your -uploaded-app listing, matches an `lt://` reference against that listing, and confirms that a local -artifact exists before saving its absolute path. `open` still needs the app's installed package or -bundle identifier, not the upload name or `lt://` id. +uploaded-app listing, looks an `lt://` reference up in that listing, and confirms that a local +artifact exists before saving its absolute path. `connect` does not prove the app usable: an `lt://` +id missing from the listing is still accepted, and TestMu AI validates it, like a URL or local +upload, only when the session is created. `open` still needs the app's installed package or bundle +identifier, not the upload name or `lt://` id. Optional labels: From 12bab7abe70715553eef8310bd18f35c7b05749c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:36:44 +0530 Subject: [PATCH 5/9] test(provider-webdriver): exercise in-flight deadlines and aborts for real The BrowserStack session-details timeout test threw a prebuilt TimeoutError from the fetch stub, so it passed even with the deadline removed. The fetch now stays pending until the signal the lookup armed aborts, and the test checks that signal is the 15 second deadline. The TestMu AI upload cancellation test aborted before the upload reached fetch, so it never covered a request in flight. It now waits for the stub to start before aborting. The connect suites shared a copied connectWithGeneratedProviderProfile helper; it now lives in the shared test utilities. Co-Authored-By: Claude Opus 5.5 --- .../src/browserstack.test.ts | 43 ++++++++++++++----- .../provider-webdriver/src/testmu.test.ts | 11 ++--- src/__tests__/cloud-connect-profile.test.ts | 24 +---------- src/__tests__/cloud-connect-testmu.test.ts | 26 +---------- src/__tests__/test-utils/connect-command.ts | 27 ++++++++++++ 5 files changed, 68 insertions(+), 63 deletions(-) create mode 100644 src/__tests__/test-utils/connect-command.ts diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 7d7af5610a..ff64a05c9e 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; import path from 'node:path'; -import { afterEach, test } from 'vitest'; +import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { listBrowserStackCloudArtifacts, @@ -109,22 +109,45 @@ test('BrowserStack passes bs:// ids and URLs to the hub and uploads only local p test('BrowserStack session details lookup has a deadline and fails typed', async () => { const lookup = async () => await listBrowserStackCloudArtifacts('browserstack', 'SESSION1', upload); - const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); - const transportFailures: unknown[] = [timeout, new TypeError('fetch failed')]; - for (const failure of transportFailures) { - globalThis.fetch = async (_input, init) => { - assert.ok(init?.signal instanceof AbortSignal); - throw failure; - }; - await assert.rejects(lookup(), (error: unknown) => { + // The deadline is the only thing that ends a hung lookup: the fetch stays pending until the + // signal the lookup armed for 15 s aborts, so removing the deadline would hang this test. + const deadline = new AbortController(); + const timeoutSpy = vi.spyOn(AbortSignal, 'timeout').mockImplementation(() => deadline.signal); + let started: () => void = () => {}; + const fetchStarted = new Promise((resolve) => { + started = resolve; + }); + globalThis.fetch = async (_input, init) => + await new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); + started(); + }); + try { + const pending = lookup(); + await fetchStarted; + assert.deepEqual(timeoutSpy.mock.calls, [[15_000]]); + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + deadline.abort(timeout); + await assert.rejects(pending, (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.code, 'COMMAND_FAILED'); assert.equal(error.message, 'BrowserStack session details lookup failed.'); - assert.equal(error.cause, failure); + assert.equal(error.cause, timeout); return true; }); + } finally { + timeoutSpy.mockRestore(); } + const networkFailure = new TypeError('fetch failed'); + globalThis.fetch = async () => { + throw networkFailure; + }; + await assert.rejects( + lookup(), + (error: unknown) => error instanceof AppError && error.cause === networkFailure, + ); + for (const body of ['gateway', '[]']) { globalThis.fetch = async () => new Response(body, { status: 200 }); await assert.rejects(lookup(), (error: unknown) => { diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts index d14039a9fc..084e7fb084 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -197,18 +197,19 @@ test('TestMu upload reads the lt:// reference and aborts while the request is in const abortReason = new Error('request cancelled during TestMu AI upload'); try { await fs.writeFile(appPath, 'placeholder'); + let started: () => void = () => {}; + const fetchStarted = new Promise((resolve) => { + started = resolve; + }); globalThis.fetch = async (_input, init) => await new Promise((_resolve, reject) => { assert.equal(init?.signal, controller.signal); - if (init?.signal?.aborted) { - reject(init.signal.reason); - return; - } init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); + started(); }); const pending = uploadTestMuApp(appPath, auth, controller.signal); - await Promise.resolve(); + await fetchStarted; controller.abort(abortReason); await assert.rejects(pending, (error: unknown) => error === abortReason); diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index e91650c8b2..5a0a3cd070 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -17,6 +17,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { verifyLimrunConnection } from '@agent-device/provider-limrun'; import { providerWebDriver } from '../provider-webdriver.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; +import { connectWithGeneratedProviderProfile } from './test-utils/connect-command.ts'; vi.mock('../cli/auth-session.ts', async (importOriginal) => ({ ...(await importOriginal()), @@ -857,29 +858,6 @@ async function captureConnectStdout(task: () => Promise): Promise { } } -async function connectWithGeneratedProviderProfile(options: { - stateDir: string; - positionals: string[]; - flags: Partial[0]['flags']>; -}): Promise { - const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - try { - await connectCommand({ - positionals: options.positionals, - flags: { - json: true, - help: false, - version: false, - stateDir: options.stateDir, - ...options.flags, - }, - client: {} as AgentDeviceClient, - }); - } finally { - stdoutWrite.mockRestore(); - } -} - function readGeneratedConfig(configPath: string): { tenant?: string; leaseProvider?: string; diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index e9b7a3413c..ee37923166 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -2,17 +2,16 @@ import { afterEach, beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { connectCommand } from '../cli/commands/connection.ts'; import { runCliCapture } from './cli-capture.ts'; import { readActiveConnectionState, type RemoteConnectionState, } from '../remote/remote-connection-state.ts'; -import type { AgentDeviceClient } from '../agent-device-client.ts'; import { resolveCloudWebDriverConnectProfile } from '../cli/connection/cloud-webdriver-profile.ts'; import { AppError } from '@agent-device/kernel/errors'; import { providerWebDriver } from '../provider-webdriver.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; +import { connectWithGeneratedProviderProfile } from './test-utils/connect-command.ts'; vi.mock('../provider-webdriver.ts', () => ({ providerWebDriver: { verifyConnection: vi.fn() }, @@ -313,29 +312,6 @@ test('connect limrun refuses profile fields Limrun does not read', async () => { ); }); -async function connectWithGeneratedProviderProfile(options: { - stateDir: string; - positionals: string[]; - flags: Partial[0]['flags']>; -}): Promise { - const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - try { - await connectCommand({ - positionals: options.positionals, - flags: { - json: true, - help: false, - version: false, - stateDir: options.stateDir, - ...options.flags, - }, - client: {} as AgentDeviceClient, - }); - } finally { - stdoutWrite.mockRestore(); - } -} - function readGeneratedConfig(configPath: string): { providerApp?: string; providerOsVersion?: string; diff --git a/src/__tests__/test-utils/connect-command.ts b/src/__tests__/test-utils/connect-command.ts new file mode 100644 index 0000000000..c16a40ca43 --- /dev/null +++ b/src/__tests__/test-utils/connect-command.ts @@ -0,0 +1,27 @@ +import { vi } from 'vitest'; +import { connectCommand } from '../../cli/commands/connection.ts'; +import type { AgentDeviceClient } from '../../agent-device-client.ts'; + +/** Runs `connect` the way the CLI does, writing its generated profile under `stateDir`. */ +export async function connectWithGeneratedProviderProfile(options: { + stateDir: string; + positionals: string[]; + flags: Partial[0]['flags']>; +}): Promise { + const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await connectCommand({ + positionals: options.positionals, + flags: { + json: true, + help: false, + version: false, + stateDir: options.stateDir, + ...options.flags, + }, + client: {} as AgentDeviceClient, + }); + } finally { + stdoutWrite.mockRestore(); + } +} From 0d70437b3cd9f810bdd610d856ef100c6f8df9aa Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:53:32 +0530 Subject: [PATCH 6/9] fix(provider-webdriver): refuse profile fields on a repeat allocation too The daemon hands a run's repeat lease_allocate the lease it already holds, and both the WebDriver runtime and the Limrun runtime returned the live session for a known lease before checking the request's profile fields. A second allocation on the same run that set a refused field, such as providerDeviceType on BrowserStack, therefore succeeded. Both runtimes now refuse before reusing a live session. Because a refusal of that repeat request is a provider allocate failure, the lease handler no longer releases a lease it reused when allocate throws; doing so would end the run's lease and leave the session the first allocation created without an owner. The runtime's profileFields option is now required, and each provider definition passes its own declaration into createRuntime, so a provider that forgets to wire its declaration fails to compile instead of silently refusing nothing. Co-Authored-By: Claude Opus 5.5 --- packages/provider-limrun/src/runtime.ts | 6 ++-- packages/provider-webdriver/src/index.ts | 2 +- .../src/profile-fields.fixtures.ts | 31 ++++++++++++++++ .../src/provider-definitions.ts | 20 ++++++----- .../src/runtime-session.test.ts | 2 ++ .../provider-webdriver/src/runtime-session.ts | 11 +++--- .../provider-webdriver/src/runtime.test.ts | 2 ++ packages/provider-webdriver/src/runtime.ts | 4 +-- src/__tests__/limrun-runtime.test.ts | 26 ++++++++++++++ src/daemon/handlers/__tests__/lease.test.ts | 36 +++++++++++++++++++ src/daemon/handlers/lease.ts | 8 ++++- .../cloud-webdriver-provider-adapters.test.ts | 36 +++++++++++++++++++ 12 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 packages/provider-webdriver/src/profile-fields.fixtures.ts diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index f1fb902b1e..df8b581e47 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -231,9 +231,6 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (lease.leaseProvider !== this.provider) return undefined; const platform = platformForLimrunLeaseBackend(lease.backend); if (!platform) return undefined; - const existing = this.sessions.get(lease.leaseId); - if (existing) return { limrunInstanceId: existing.instanceId, device: existing.device }; - const { allocateLimrunAndroidSession, allocateLimrunIosSession, @@ -241,7 +238,10 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { resolvePreinstalledAppId, resolveRequestedLimrunAppAsset, } = await import('./session-allocation.ts'); + // Before the reuse below: a repeat allocation of a live lease carries flags of its own. rejectRefusedLimrunProfileFields(context); + const existing = this.sessions.get(lease.leaseId); + if (existing) return { limrunInstanceId: existing.instanceId, device: existing.device }; const requestedAsset = await resolveRequestedLimrunAppAsset(this.limrun, platform, context); const session = platform === 'ios' diff --git a/packages/provider-webdriver/src/index.ts b/packages/provider-webdriver/src/index.ts index 12c30acf60..b8c785eb3c 100644 --- a/packages/provider-webdriver/src/index.ts +++ b/packages/provider-webdriver/src/index.ts @@ -50,7 +50,7 @@ export function createProviderWebDriver( return { providerIds: definitions.map((definition) => definition.provider), createDefaultRuntimes: (env = process.env) => - definitions.map((definition) => definition.createRuntime(env)), + definitions.map((definition) => definition.createRuntime(env, definition.profileFields)), listArtifactsFromEnv: async (query, env) => { if (!query.providerSessionId) return undefined; return await definitions diff --git a/packages/provider-webdriver/src/profile-fields.fixtures.ts b/packages/provider-webdriver/src/profile-fields.fixtures.ts new file mode 100644 index 0000000000..f52e710f25 --- /dev/null +++ b/packages/provider-webdriver/src/profile-fields.fixtures.ts @@ -0,0 +1,31 @@ +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; + +/** A test hub that reads every profile field, so no flag is refused. */ +export function consumeAllProfileFields(provider: string): ProviderProfileFieldDeclaration { + return { + provider, + label: provider, + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'consumed', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'consumed', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', + }, + }; +} diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 6981043fc5..d701f29595 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -174,9 +174,13 @@ export const CLOUD_WEBDRIVER_PROFILE_FIELDS: Readonly< export type CloudWebDriverProviderDefinition = { provider: CloudWebDriverKnownProviderName; - /** Every profile field, consumed or refused; session preparation refuses the refused ones. */ + /** Every profile field, consumed or refused; lease allocation refuses the refused ones. */ profileFields: ProviderProfileFieldDeclaration; - createRuntime: (env: DefaultCloudWebDriverProviderRuntimeEnv) => CloudWebDriverRuntime; + /** Receives `profileFields`, which the runtime requires, so the two cannot drift apart. */ + createRuntime: ( + env: DefaultCloudWebDriverProviderRuntimeEnv, + profileFields: ProviderProfileFieldDeclaration, + ) => CloudWebDriverRuntime; listArtifactsFromEnv: ( providerSessionId: string, env: DefaultCloudWebDriverArtifactEnv, @@ -190,11 +194,11 @@ export function createCloudWebDriverProviderDefinitions( { provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, profileFields: BROWSERSTACK_PROFILE_FIELDS, - createRuntime: (env) => + createRuntime: (env, profileFields) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, - profileFields: BROWSERSTACK_PROFILE_FIELDS, + profileFields, platform: 'android', deviceName: 'BrowserStack device', endpoint: env.BROWSERSTACK_WEBDRIVER_ENDPOINT ?? BROWSERSTACK_APP_AUTOMATE_ENDPOINT, @@ -299,11 +303,11 @@ export function createCloudWebDriverProviderDefinitions( { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, - createRuntime: (env) => + createRuntime: (env, profileFields) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, + profileFields, endpoint: 'http://127.0.0.1/', platform: 'android', deviceName: 'AWS Device Farm device', @@ -370,11 +374,11 @@ export function createCloudWebDriverProviderDefinitions( { provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, profileFields: TESTMU_PROFILE_FIELDS, - createRuntime: (env) => + createRuntime: (env, profileFields) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, - profileFields: TESTMU_PROFILE_FIELDS, + profileFields, platform: 'android', deviceName: 'TestMu AI device', endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, diff --git a/packages/provider-webdriver/src/runtime-session.test.ts b/packages/provider-webdriver/src/runtime-session.test.ts index e435e5571b..4429903b7e 100644 --- a/packages/provider-webdriver/src/runtime-session.test.ts +++ b/packages/provider-webdriver/src/runtime-session.test.ts @@ -3,6 +3,7 @@ import { afterEach, test } from 'vitest'; import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; import { createCloudWebDriverRuntime, type CloudWebDriverRuntimeOptions } from './runtime.ts'; +import { consumeAllProfileFields } from './profile-fields.fixtures.ts'; const realFetch = globalThis.fetch; @@ -106,6 +107,7 @@ function makeRuntime(overrides: Partial = {}) { endpoint: 'https://webdriver.test/wd/hub/', platform: 'android', deviceName: 'Test device', + profileFields: consumeAllProfileFields('webdriver-test'), requestPolicy: { retryAttempts: 0 }, ...overrides, }); diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index 293aa51b0d..7274789f6c 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -82,6 +82,11 @@ export class WebDriverSessionManager { async allocate(lease: DeviceLease, req?: LeaseLifecycleContext): Promise { if (lease.leaseProvider !== this.options.provider) return undefined; + // Before the reuse below: a repeat allocation of a live lease carries flags of its own. + // Loaded here rather than eagerly so the package entry's closure stays unchanged. + const { rejectRefusedProviderProfileFields } = + await import('@agent-device/contracts/provider-profile-fields'); + rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); if (this.sessionsByLeaseId.has(lease.leaseId)) return this.heartbeat(lease); const prepared = await this.prepareSession(lease, req); const client = new WebDriverClient({ @@ -203,12 +208,6 @@ export class WebDriverSessionManager { lease: DeviceLease, req: LeaseLifecycleContext | undefined, ): Promise { - if (this.options.profileFields) { - // Loaded on first allocation so the package entry's eager closure stays unchanged. - const { rejectRefusedProviderProfileFields } = - await import('@agent-device/contracts/provider-profile-fields'); - rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); - } const base = this.baseSessionForLease(lease); return this.options.prepareSession ? await this.options.prepareSession({ lease, req, base }) diff --git a/packages/provider-webdriver/src/runtime.test.ts b/packages/provider-webdriver/src/runtime.test.ts index 134e8a4a78..e94d1faece 100644 --- a/packages/provider-webdriver/src/runtime.test.ts +++ b/packages/provider-webdriver/src/runtime.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest'; import { createCloudWebDriverRuntime } from './runtime.ts'; +import { consumeAllProfileFields } from './profile-fields.fixtures.ts'; test('publishes eager provider metadata without loading a WebDriver session', async () => { const runtime = createCloudWebDriverRuntime({ @@ -8,6 +9,7 @@ test('publishes eager provider metadata without loading a WebDriver session', as endpoint: 'https://webdriver.test/wd/hub/', platform: 'android', deviceName: 'Test device', + profileFields: consumeAllProfileFields('webdriver-test'), }); try { diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 146cb89787..4b94ac2d72 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -101,8 +101,8 @@ export type CloudWebDriverRuntimeOptions = { deviceId?: (lease: DeviceLease) => string; prepareSession?: CloudWebDriverPrepareSession; capabilityOverrides?: CloudWebDriverCapabilityOverrides; - /** Profile fields this provider reads; any field it refuses fails session preparation. */ - profileFields?: ProviderProfileFieldDeclaration; + /** Profile fields this provider reads; any field it refuses fails lease allocation. */ + profileFields: ProviderProfileFieldDeclaration; }; export function createCloudWebDriverRuntime( diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index dd3446fbeb..5aa066d8b5 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -172,6 +172,32 @@ test('Limrun runtime identifies direct CLI usage to the Limrun API', async () => } }); +test('Limrun refuses a refused field on a repeat allocation of its live lease', async () => { + const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); + const lease: SimulatorLease = { + leaseId: 'lease-repeat', + tenantId: 'team-a', + runId: 'run-a', + backend: 'ios-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + }; + try { + const allocateLease = runtime.leaseLifecycle.allocate; + if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); + await allocateLease(lease); + await assert.rejects( + allocateLease(lease, { flags: { providerDeviceType: 'real' } }), + /--provider-device-type is not supported by Limrun/, + ); + assert.equal(limrunMockState.iosCreate.mock.calls.length, 1); + } finally { + await runtime.shutdown(); + } +}); + test('Limrun iOS uses shared deep-link classification', async () => { const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts index 5cbd378ea6..076cc0c07b 100644 --- a/src/daemon/handlers/__tests__/lease.test.ts +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -172,3 +172,39 @@ test('a lease allocated without a provider keeps the TTL it was created with', a assert.equal(lease.expiresAt, 65_000); assert.deepEqual(registry.listActiveLeases(), [lease]); }); + +// The registry hands a run's repeat allocation the lease it already holds. A provider refusing the +// repeat request (a profile field it does not read) must leave that lease and its session alone. +test("a refused repeat allocation keeps the run's live lease", async () => { + const registry = new LeaseRegistry(); + const sessionStore = makeSessionStore('agent-device-refused-repeat-'); + let calls = 0; + const allocate = async (req: DaemonRequest) => + await handleLeaseCommands({ + req, + sessionName: 'lease-ttl-test', + sessionStore, + leaseRegistry: registry, + leaseLifecycleProvider: { + allocate: async () => { + calls += 1; + if (calls === 1) return { providerSessionId: 'session-1' }; + throw new AppError('INVALID_ARGS', '--provider-device-type is not supported by Cloud.'); + }, + }, + }); + + const first = await allocate(allocateRequest()); + const lease = (first?.ok ? first.data?.lease : undefined) as DeviceLease; + const repeat = allocateRequest(); + repeat.flags = { providerDeviceType: 'real' }; + await assert.rejects( + allocate(repeat), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + assert.equal(calls, 2); + assert.deepEqual( + registry.listActiveLeases().map((entry) => entry.leaseId), + [lease.leaseId], + ); +}); diff --git a/src/daemon/handlers/lease.ts b/src/daemon/handlers/lease.ts index f0826bb87f..06d46126cd 100644 --- a/src/daemon/handlers/lease.ts +++ b/src/daemon/handlers/lease.ts @@ -70,7 +70,13 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise entry.leaseId), + ); const lease = leaseRegistry.allocateLease(leaseScopeToAllocateRequest(leaseScope)); + // A run's repeat allocation reuses its live lease; refusing that request must not end the + // lease, or the provider session the first allocation created is left without an owner. + const reused = activeLeaseIds.has(lease.leaseId); const requestId = req.meta?.requestId; return await leaseRegistry.runDeviceMutation(lease, async () => { let providerData: Record | undefined; @@ -85,7 +91,7 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + const provider = createProviderWebDriver({ + clientVersion: CLIENT_VERSION, + runHostCommand: unexpectedHostCommand, + }); + const runtime = runtimeFor( + provider.createDefaultRuntimes({ + BROWSERSTACK_USERNAME: 'user', + BROWSERSTACK_ACCESS_KEY: 'key', + BROWSERSTACK_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + }), + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + ); + const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.browserStack); + const context = browserStackContext(lease); + try { + await runtime.leaseLifecycle.allocate?.(lease, context); + const sessionCalls = server.calls.length; + await assert.rejects( + async () => + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { ...context.flags, providerDeviceType: 'real' }, + }), + /--provider-device-type is not supported by BrowserStack/, + ); + assert.equal(server.calls.length, sessionCalls); + assert.deepEqual(await runtime.leaseLifecycle.heartbeat?.(lease), { + provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + }); + } finally { + await runtime.shutdown(); + } + }); +}, 15_000); + test('AWS Device Farm facade uses the injected host-command capability for its full lifecycle', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const host = new FakeAwsHostCommand(`${server.url}/wd/hub/`); From 80aa05f862528ba4d0b300898ef381e6aa91cd16 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:54:48 +0530 Subject: [PATCH 7/9] fix(provider-webdriver): validate and canonicalize app references at connect Connect only checked that an lt:// or bs:// reference had something after the scheme, so `lt://a b` and `lt://a/b` were saved and failed only when the hub created the session. Both hubs' reference grammars now live in the light providers module, and connect, session preparation and the TestMu AI upload reader all validate against them. Connect also verified the raw spelling typed on the command line: the generated profile stored `lt://APP1` for `LT://APP1`, but the flags handed to verification were the raw CLI flags laid over the profile, so the uploaded-app lookup missed and reported a local artifact. The canonical reference now wins in the flags verification reads. Co-Authored-By: Claude Opus 5.5 --- .../provider-webdriver/src/browserstack.ts | 2 ++ packages/provider-webdriver/src/providers.ts | 12 ++++++++ packages/provider-webdriver/src/testmu.ts | 9 ++---- src/__tests__/cloud-connect-testmu.test.ts | 29 ++++++++++++------- src/cli/connection/cloud-webdriver-profile.ts | 25 ++++++++++++---- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index ffd854052f..dc55978c8f 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -2,6 +2,7 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contract import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { isBrowserStackAppReference } from './providers.ts'; import { appendUrlPath, appFileUploadForm, @@ -112,6 +113,7 @@ export async function resolveBrowserStackAppReference( cwd: options.cwd, referenceScheme: 'bs://', referenceLabel: 'a bs:// app id', + isReference: isBrowserStackAppReference, uploadFile: async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), signal: options.signal, }); diff --git a/packages/provider-webdriver/src/providers.ts b/packages/provider-webdriver/src/providers.ts index 4160d465a5..c64c3c7c00 100644 --- a/packages/provider-webdriver/src/providers.ts +++ b/packages/provider-webdriver/src/providers.ts @@ -14,3 +14,15 @@ export function isCloudWebDriverProviderName( ): provider is CloudWebDriverKnownProviderName { return provider !== undefined && CLOUD_WEBDRIVER_KNOWN_PROVIDERS.has(provider); } + +/** + * The app references each hub accepts. An id outside the grammar would otherwise pass every local + * check and fail only when the hub creates the session. + */ +export function isBrowserStackAppReference(value: string): boolean { + return /^bs:\/\/[\w.-]+$/.test(value); +} + +export function isTestMuAppReference(value: string): boolean { + return /^lt:\/\/[\w.-]+$/.test(value); +} diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts index 1217fd6c05..68b970d971 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-webdriver/src/testmu.ts @@ -4,6 +4,7 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contract import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; import { AppError } from '@agent-device/kernel/errors'; +import { isTestMuAppReference } from './providers.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { appendUrlPath, @@ -26,6 +27,8 @@ const TESTMU_APP_UPLOAD_ENDPOINTS: Record = { }; export const TESTMU_APPS_ENDPOINT = 'https://manual-api.lambdatest.com/app/data'; export const TESTMU_API_ENDPOINT = 'https://mobile-api.lambdatest.com/mobile-automation/api/v1'; +export { isTestMuAppReference }; + const TESTMU_DASHBOARD_TEST_URL = 'https://appautomation.lambdatest.com/test?testID='; export type TestMuCapabilitiesOptions = { @@ -193,12 +196,6 @@ export function buildTestMuCapabilities( }; } -const TESTMU_APP_REFERENCE = /^lt:\/\/[\w.-]+$/; - -export function isTestMuAppReference(value: string): boolean { - return TESTMU_APP_REFERENCE.test(value); -} - /** The upload and app-list APIs answer with a bare app id or an `lt://` reference. */ export function testMuAppReferenceFromId(id: string): string { return id.startsWith('lt://') ? id : `lt://${id}`; diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index ee37923166..ea4b2f5dfa 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -102,21 +102,28 @@ test('connect canonicalizes an upper-case app scheme and refuses an empty app id }); try { - assert.equal( - readGeneratedConfig(connect('testmu', 'LT://APP1').remoteConfigPath).providerApp, - 'lt://APP1', - ); + const upperCase = connect('testmu', 'LT://APP1'); + assert.equal(readGeneratedConfig(upperCase.remoteConfigPath).providerApp, 'lt://APP1'); + // Connect verification reads these flags, so they must carry the canonical reference too. + assert.equal(upperCase.flags.providerApp, 'lt://APP1'); assert.equal( readGeneratedConfig(connect('browserstack', 'Bs://abc').remoteConfigPath).providerApp, 'bs://abc', ); - assert.throws( - () => connect('testmu', 'lt://'), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /--provider-app lt:\/\/ has no app id/.test(error.message), - ); + for (const [provider, app] of [ + ['testmu', 'lt://'], + ['testmu', 'lt://a b'], + ['testmu', 'LT://a/b'], + ['browserstack', 'bs://'], + ] as const) { + assert.throws( + () => connect(provider, app), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message.includes(`--provider-app ${app} is not a valid`), + ); + } } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index 5611c86eaa..a9e0f61db2 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,6 +4,10 @@ import { readAwsDeviceFarmRegionFromArn, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; +import { + isBrowserStackAppReference, + isTestMuAppReference, +} from '@agent-device/provider-webdriver/providers'; import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; @@ -55,6 +59,11 @@ export function resolveCloudWebDriverConnectProfile(options: { cwd: options.cwd, env: options.env, flags: options.flags, + // Verification reads these flags; it must see the canonical reference the profile saved, + // not the spelling typed on the command line. + ...(providerConfig.providerApp + ? { extraFlags: { providerApp: providerConfig.providerApp } } + : {}), }); } @@ -99,6 +108,8 @@ type HubProviderProfile = { credentialEnv: readonly [string, string]; /** Scheme of the provider's own app references, e.g. `bs://` or `lt://`. */ appScheme: string; + /** The hub's own reference grammar, checked here so a malformed id fails at connect. */ + isAppReference: (reference: string) => boolean; appHint: string; }; @@ -107,6 +118,7 @@ const BROWSERSTACK_HUB_PROFILE: HubProviderProfile = { label: 'BrowserStack', credentialEnv: ['BROWSERSTACK_USERNAME', 'BROWSERSTACK_ACCESS_KEY'], appScheme: 'bs://', + isAppReference: isBrowserStackAppReference, appHint: '', }; @@ -115,6 +127,7 @@ const TESTMU_HUB_PROFILE: HubProviderProfile = { label: 'TestMu AI', credentialEnv: ['LT_USERNAME', 'LT_ACCESS_KEY'], appScheme: 'lt://', + isAppReference: isTestMuAppReference, appHint: '', }; @@ -175,11 +188,13 @@ function normalizeHubAppReference(hub: HubProviderProfile, app: string, cwd: str if (/^https?:\/\//i.test(app)) return app; // URI schemes are case-insensitive; the hub only matches the lower-case spelling. if (app.slice(0, hub.appScheme.length).toLowerCase() === hub.appScheme) { - const id = app.slice(hub.appScheme.length); - if (id.length > 0) return `${hub.appScheme}${id}`; - throw new AppError('INVALID_ARGS', `${hub.command} --provider-app ${app} has no app id.`, { - hint: `Pass ${hub.appHint}.`, - }); + const reference = `${hub.appScheme}${app.slice(hub.appScheme.length)}`; + if (hub.isAppReference(reference)) return reference; + throw new AppError( + 'INVALID_ARGS', + `${hub.command} --provider-app ${app} is not a valid ${hub.appScheme} app reference.`, + { hint: `Pass ${hub.appHint}.` }, + ); } const resolvedPath = path.resolve(cwd, app); try { From 720e8927cc5e9cc8420c02e5c8315b8bfbb2431c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:55:35 +0530 Subject: [PATCH 8/9] fix(provider-webdriver): refuse a directory before any hosted upload A materialized build that names no uploadable file falls back to its installable path, which for iOS is the extracted .app directory. The deployment runtime handed that to the hub's uploader, and BrowserStack's read it as a file and failed with a raw EISDIR. The deployment runtime now refuses a directory before calling any hub's uploader, with INVALID_ARGS naming the provider and path and a hint to zip the .app or pass the .ipa, .apk, or .aab. Co-Authored-By: Claude Opus 5.5 --- .../src/runtime-deployment.test.ts | 30 +++++++++++++++++++ .../src/runtime-deployment.ts | 17 +++++++++++ 2 files changed, 47 insertions(+) diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index 41aaa508d3..efe9835962 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -125,6 +125,36 @@ test('a hosted upload sends the file the materializer names, else the installabl expect(installApp).toHaveBeenNthCalledWith(1, 'lt://1', expect.any(AbortSignal)); }); +test('a hosted upload refuses a directory typed instead of reading it', async () => { + const tempDir = await mkdtempForTest('agent-device-materialized-directory-'); + try { + const installablePath = path.join(tempDir, 'extracted', 'App.app'); + await fs.mkdir(installablePath, { recursive: true }); + const uploadApp = vi.fn(); + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'browserstack', + uploadApp, + findSessionForDevice: () => activeSession(installApp), + }); + + await expect( + deployment.deployMaterializedApp( + iosDevice, + { artifact: { installablePath, cleanup: async () => {} } }, + new AbortController().signal, + ), + ).rejects.toMatchObject({ + code: 'INVALID_ARGS', + message: `browserstack can only upload an app file, not a directory: ${installablePath}`, + }); + expect(uploadApp).not.toHaveBeenCalled(); + expect(installApp).not.toHaveBeenCalled(); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + test('a provider without an uploader still installs the materialized bundle path', async () => { const installApp = vi.fn(async () => undefined); const deployment = createWebDriverDeploymentRuntime({ diff --git a/packages/provider-webdriver/src/runtime-deployment.ts b/packages/provider-webdriver/src/runtime-deployment.ts index fb2a058b86..2f96f1ace4 100644 --- a/packages/provider-webdriver/src/runtime-deployment.ts +++ b/packages/provider-webdriver/src/runtime-deployment.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs/promises'; import type { ProviderDeviceInstallOptions, ProviderDeviceInstallResult, @@ -155,6 +156,7 @@ async function uploadAppIfNeeded( } const uploadApp = session.prepared.uploadApp ?? options.uploadApp; if (!uploadApp) return undefined; + await assertUploadableFile(options.provider, appPath); return await uploadApp({ provider: options.provider, lease: session.lease, @@ -166,6 +168,21 @@ async function uploadAppIfNeeded( }); } +/** Hosted upload APIs take one file; an extracted `.app` with no declared archive is a directory. */ +async function assertUploadableFile(provider: string, appPath: string): Promise { + const stat = await fs.stat(appPath).catch(() => undefined); + if (!stat || stat.isFile()) return; + throw new AppError( + 'INVALID_ARGS', + `${provider} can only upload an app file, not a directory: ${appPath}`, + { + provider, + appPath, + hint: 'Zip the iOS simulator .app bundle and install the .zip, or install the .ipa, .apk, or .aab.', + }, + ); +} + function deploymentResult( result: ProviderDeviceInstallResult | undefined, fallbackTarget?: string, From 417ff5ba7a40d0f168d7caf9b78057944397a494 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 23:01:22 +0530 Subject: [PATCH 9/9] fix(cli): refuse a non-URL install-from-source source with a typed error install-from-source sends its positional as a URL source. A local path reached the iOS trusted-host check, where `new URL()` threw a TypeError that surfaced as an untyped UNKNOWN "Invalid URL". The CLI now refuses a positional that is not an http(s) URL with INVALID_ARGS and points at `install ` for local builds, and the trusted-host check reports an unparsable source as INVALID_ARGS instead of throwing. Co-Authored-By: Claude Opus 5.5 --- packages/provision-kit/src/install-source.ts | 3 +- src/__tests__/cli-install-from-source.test.ts | 30 +++++++++++++++++++ src/__tests__/install-source.test.ts | 8 +++++ src/commands/management/install.ts | 5 ++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/cli-install-from-source.test.ts diff --git a/packages/provision-kit/src/install-source.ts b/packages/provision-kit/src/install-source.ts index 26119979e9..3daada22f3 100644 --- a/packages/provision-kit/src/install-source.ts +++ b/packages/provision-kit/src/install-source.ts @@ -160,7 +160,8 @@ export async function validateDownloadSourceUrl(parsedUrl: URL): Promise { } export function isTrustedInstallSourceUrl(sourceUrl: string | URL): boolean { - const parsed = sourceUrl instanceof URL ? sourceUrl : new URL(sourceUrl); + const parsed = sourceUrl instanceof URL ? sourceUrl : URL.parse(sourceUrl); + if (!parsed) throw new AppError('INVALID_ARGS', 'Invalid source URL'); const hostname = parsed.hostname.toLowerCase(); if (!hostname) return false; const pathname = parsed.pathname; diff --git a/src/__tests__/cli-install-from-source.test.ts b/src/__tests__/cli-install-from-source.test.ts new file mode 100644 index 0000000000..f99e8b3705 --- /dev/null +++ b/src/__tests__/cli-install-from-source.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { AgentDeviceClient } from '../agent-device-client.ts'; +import { tryRunClientBackedCommand } from '../cli/commands/router.ts'; + +test('install-from-source refuses a local path with a typed error', async () => { + const client = { + apps: { + installFromSource: async () => { + throw new Error('unexpected call'); + }, + }, + } as unknown as AgentDeviceClient; + + await assert.rejects( + () => + tryRunClientBackedCommand({ + command: 'install-from-source', + positionals: ['/abs/path/app.zip'], + flags: { json: false, help: false, version: false }, + client, + }), + (error) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === 'install-from-source must be an http(s) URL: /abs/path/app.zip' && + String(error.details?.hint).includes('install '), + ); +}); diff --git a/src/__tests__/install-source.test.ts b/src/__tests__/install-source.test.ts index 974a6ba509..7059e83d0f 100644 --- a/src/__tests__/install-source.test.ts +++ b/src/__tests__/install-source.test.ts @@ -24,6 +24,7 @@ import { withAppleToolProvider, } from '@agent-device/platform-apple/tool-provider'; import { prepareIosInstallArtifact } from '@agent-device/platform-apple/install-artifact'; +import { AppError } from '@agent-device/kernel/errors'; import { ANDROID_INSTALL_SOURCE_CONTRACT_EVIDENCE } from './install-source.coverage.ts'; import { mkdtempForTest } from './test-utils/tmp-dir.ts'; import * as networkTransport from '@agent-device/provision-kit/install-source-network-transport'; @@ -104,6 +105,13 @@ test('isTrustedInstallSourceUrl recognizes supported artifact services', () => { false, ); assert.equal(isTrustedInstallSourceUrl('https://expo.dev/pricing'), false); + assert.throws( + () => isTrustedInstallSourceUrl('/abs/path/app.zip'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === 'Invalid source URL', + ); }); test('materializeInstallablePath rejects archive extraction when disabled', async () => { diff --git a/src/commands/management/install.ts b/src/commands/management/install.ts index 7155d10e09..015858f4fd 100644 --- a/src/commands/management/install.ts +++ b/src/commands/management/install.ts @@ -204,6 +204,11 @@ function resolveInstallSource(positionals: string[], flags: CliFlags) { } if (githubArtifactSource) return githubArtifactSource; if (configuredSource) return configuredSource; + if (!/^https?:\/\//i.test(url!)) { + throw new AppError('INVALID_ARGS', `install-from-source must be an http(s) URL: ${url}`, { + hint: 'Install a local build with install .', + }); + } return { kind: 'url' as const, url: url!,